1#!/usr/bin/python
2import hashlib
3import optparse
4import os
5import re
6import shlex
7import subprocess
8import sys
9import threading
10import time
11
12TASK_COMPILATION = 'compile'
13TASK_DISABLE_OVERLAYS = 'disable overlays'
14TASK_ENABLE_MULTIPLE_OVERLAYS = 'enable multiple overlays'
15TASK_ENABLE_SINGLE_OVERLAY = 'enable single overlay'
16TASK_FILE_EXISTS_TEST = 'test (file exists)'
17TASK_GREP_IDMAP_TEST = 'test (grep idmap)'
18TASK_MD5_TEST = 'test (md5)'
19TASK_IDMAP_PATH = 'idmap --path'
20TASK_IDMAP_SCAN = 'idmap --scan'
21TASK_INSTRUMENTATION = 'instrumentation'
22TASK_INSTRUMENTATION_TEST = 'test (instrumentation)'
23TASK_MKDIR = 'mkdir'
24TASK_PUSH = 'push'
25TASK_ROOT = 'root'
26TASK_REMOUNT = 'remount'
27TASK_RM = 'rm'
28TASK_SETUP_IDMAP_PATH = 'setup idmap --path'
29TASK_SETUP_IDMAP_SCAN = 'setup idmap --scan'
30TASK_START = 'start'
31TASK_STOP = 'stop'
32
33adb = 'adb'
34
35def _adb_shell(cmd):
36    argv = shlex.split(adb + " shell '" + cmd + "; echo $?'")
37    proc = subprocess.Popen(argv, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
38    (stdout, stderr) = proc.communicate()
39    (stdout, stderr) = (stdout.replace('\r', ''), stderr.replace('\r', ''))
40    tmp = stdout.rsplit('\n', 2)
41    if len(tmp) == 2:
42        stdout == ''
43        returncode = int(tmp[0])
44    else:
45        stdout = tmp[0] + '\n'
46        returncode = int(tmp[1])
47    return returncode, stdout, stderr
48
49class VerbosePrinter:
50    class Ticker(threading.Thread):
51        def _print(self):
52            s = '\r' + self.text + '[' + '.' * self.i + ' ' * (4 - self.i) + ']'
53            sys.stdout.write(s)
54            sys.stdout.flush()
55            self.i = (self.i + 1) % 5
56
57        def __init__(self, cond_var, text):
58            threading.Thread.__init__(self)
59            self.text = text
60            self.setDaemon(True)
61            self.cond_var = cond_var
62            self.running = False
63            self.i = 0
64            self._print()
65            self.running = True
66
67        def run(self):
68            self.cond_var.acquire()
69            while True:
70                self.cond_var.wait(0.25)
71                running = self.running
72                if not running:
73                    break
74                self._print()
75            self.cond_var.release()
76
77        def stop(self):
78            self.cond_var.acquire()
79            self.running = False
80            self.cond_var.notify_all()
81            self.cond_var.release()
82
83    def _start_ticker(self):
84        self.ticker = VerbosePrinter.Ticker(self.cond_var, self.text)
85        self.ticker.start()
86
87    def _stop_ticker(self):
88        self.ticker.stop()
89        self.ticker.join()
90        self.ticker = None
91
92    def _format_begin(self, type, name):
93        N = self.width - len(type) - len(' [    ] ')
94        fmt = '%%s %%-%ds ' % N
95        return fmt % (type, name)
96
97    def __init__(self, use_color):
98        self.cond_var = threading.Condition()
99        self.ticker = None
100        if use_color:
101            self.color_RED = '\033[1;31m'
102            self.color_red = '\033[0;31m'
103            self.color_reset = '\033[0;37m'
104        else:
105            self.color_RED = ''
106            self.color_red = ''
107            self.color_reset = ''
108
109        argv = shlex.split('stty size') # get terminal width
110        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
111        (stdout, stderr) = proc.communicate()
112        if proc.returncode == 0:
113            (h, w) = stdout.split()
114            self.width = int(w)
115        else:
116            self.width = 72 # conservative guesstimate
117
118    def begin(self, type, name):
119        self.text = self._format_begin(type, name)
120        sys.stdout.write(self.text + '[    ]')
121        sys.stdout.flush()
122        self._start_ticker()
123
124    def end_pass(self, type, name):
125        self._stop_ticker()
126        sys.stdout.write('\r' + self.text + '[ OK ]\n')
127        sys.stdout.flush()
128
129    def end_fail(self, type, name, msg):
130        self._stop_ticker()
131        sys.stdout.write('\r' + self.color_RED + self.text + '[FAIL]\n')
132        sys.stdout.write(self.color_red)
133        sys.stdout.write(msg)
134        sys.stdout.write(self.color_reset)
135        sys.stdout.flush()
136
137class QuietPrinter:
138    def begin(self, type, name):
139        pass
140
141    def end_pass(self, type, name):
142        sys.stdout.write('PASS ' + type + ' ' + name + '\n')
143        sys.stdout.flush()
144
145    def end_fail(self, type, name, msg):
146        sys.stdout.write('FAIL ' + type + ' ' + name + '\n')
147        sys.stdout.flush()
148
149class CompilationTask:
150    def __init__(self, makefile):
151        self.makefile = makefile
152
153    def get_type(self):
154        return TASK_COMPILATION
155
156    def get_name(self):
157        return self.makefile
158
159    def execute(self):
160        os.putenv('ONE_SHOT_MAKEFILE', os.getcwd() + "/" + self.makefile)
161        argv = shlex.split('make -C "../../../../../" files')
162        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
163        (stdout, stderr) = proc.communicate()
164        return proc.returncode, stdout, stderr
165
166class InstrumentationTask:
167    def __init__(self, instrumentation_class):
168        self.instrumentation_class = instrumentation_class
169
170    def get_type(self):
171        return TASK_INSTRUMENTATION
172
173    def get_name(self):
174        return self.instrumentation_class
175
176    def execute(self):
177        return _adb_shell('am instrument -r -w -e class %s com.android.overlaytest/android.test.InstrumentationTestRunner' % self.instrumentation_class)
178
179class PushTask:
180    def __init__(self, src, dest):
181        self.src = src
182        self.dest = dest
183
184    def get_type(self):
185        return TASK_PUSH
186
187    def get_name(self):
188        return "%s -> %s" % (self.src, self.dest)
189
190    def execute(self):
191        src = os.getenv('OUT') + "/" + self.src
192        argv = shlex.split(adb + ' push %s %s' % (src, self.dest))
193        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
194        (stdout, stderr) = proc.communicate()
195        return proc.returncode, stdout, stderr
196
197class MkdirTask:
198    def __init__(self, path):
199        self.path = path
200
201    def get_type(self):
202        return TASK_MKDIR
203
204    def get_name(self):
205        return self.path
206
207    def execute(self):
208        return _adb_shell('mkdir -p %s' % self.path)
209
210class RmTask:
211    def __init__(self, path):
212        self.path = path
213
214    def get_type(self):
215        return TASK_RM
216
217    def get_name(self):
218        return self.path
219
220    def execute(self):
221        returncode, stdout, stderr = _adb_shell('ls %s' % self.path)
222        if returncode != 0 and stdout.endswith(': No such file or directory\n'):
223            return 0, "", ""
224        return _adb_shell('rm -r %s' % self.path)
225
226class IdmapPathTask:
227    def __init__(self, path_target_apk, path_overlay_apk, path_idmap):
228        self.path_target_apk = path_target_apk
229        self.path_overlay_apk = path_overlay_apk
230        self.path_idmap = path_idmap
231
232    def get_type(self):
233        return TASK_IDMAP_PATH
234
235    def get_name(self):
236        return self.path_idmap
237
238    def execute(self):
239        return _adb_shell('su system idmap --path "%s" "%s" "%s"' % (self.path_target_apk, self.path_overlay_apk, self.path_idmap))
240
241class IdmapScanTask:
242    def __init__(self, overlay_dir, target_pkg_name, target_pkg, idmap_dir, symlink_dir):
243        self.overlay_dir = overlay_dir
244        self.target_pkg_name = target_pkg_name
245        self.target_pkg = target_pkg
246        self.idmap_dir = idmap_dir
247        self.symlink_dir = symlink_dir
248
249    def get_type(self):
250        return TASK_IDMAP_SCAN
251
252    def get_name(self):
253        return self.target_pkg_name
254
255    def execute(self):
256        return _adb_shell('su system idmap --scan "%s" "%s" "%s" "%s"' % (self.overlay_dir, self.target_pkg_name, self.target_pkg, self.idmap_dir))
257
258class FileExistsTest:
259    def __init__(self, path):
260        self.path = path
261
262    def get_type(self):
263        return TASK_FILE_EXISTS_TEST
264
265    def get_name(self):
266        return self.path
267
268    def execute(self):
269        return _adb_shell('ls %s' % self.path)
270
271class GrepIdmapTest:
272    def __init__(self, path_idmap, pattern, expected_n):
273        self.path_idmap = path_idmap
274        self.pattern = pattern
275        self.expected_n = expected_n
276
277    def get_type(self):
278        return TASK_GREP_IDMAP_TEST
279
280    def get_name(self):
281        return self.pattern
282
283    def execute(self):
284        returncode, stdout, stderr = _adb_shell('idmap --inspect %s' % self.path_idmap)
285        if returncode != 0:
286            return returncode, stdout, stderr
287        all_matches = re.findall('\s' + self.pattern + '$', stdout, flags=re.MULTILINE)
288        if len(all_matches) != self.expected_n:
289            return 1, 'pattern=%s idmap=%s expected=%d found=%d\n' % (self.pattern, self.path_idmap, self.expected_n, len(all_matches)), ''
290        return 0, "", ""
291
292class Md5Test:
293    def __init__(self, path, expected_content):
294        self.path = path
295        self.expected_md5 = hashlib.md5(expected_content).hexdigest()
296
297    def get_type(self):
298        return TASK_MD5_TEST
299
300    def get_name(self):
301        return self.path
302
303    def execute(self):
304        returncode, stdout, stderr = _adb_shell('md5 %s' % self.path)
305        if returncode != 0:
306            return returncode, stdout, stderr
307        actual_md5 = stdout.split()[0]
308        if actual_md5 != self.expected_md5:
309            return 1, 'expected %s, got %s\n' % (self.expected_md5, actual_md5), ''
310        return 0, "", ""
311
312class StartTask:
313    def get_type(self):
314        return TASK_START
315
316    def get_name(self):
317        return ""
318
319    def execute(self):
320        (returncode, stdout, stderr) = _adb_shell('start')
321        if returncode != 0:
322            return returncode, stdout, stderr
323
324        while True:
325            (returncode, stdout, stderr) = _adb_shell('getprop dev.bootcomplete')
326            if returncode != 0:
327                return returncode, stdout, stderr
328            if stdout.strip() == "1":
329                break
330            time.sleep(0.5)
331
332        return 0, "", ""
333
334class StopTask:
335    def get_type(self):
336        return TASK_STOP
337
338    def get_name(self):
339        return ""
340
341    def execute(self):
342        (returncode, stdout, stderr) = _adb_shell('stop')
343        if returncode != 0:
344            return returncode, stdout, stderr
345        return _adb_shell('setprop dev.bootcomplete 0')
346
347class RootTask:
348    def get_type(self):
349        return TASK_ROOT
350
351    def get_name(self):
352        return ""
353
354    def execute(self):
355        (returncode, stdout, stderr) = _adb_shell('getprop service.adb.root 0')
356        if returncode != 0:
357            return returncode, stdout, stderr
358        if stdout.strip() == '1': # already root
359            return 0, "", ""
360
361        argv = shlex.split(adb + ' root')
362        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
363        (stdout, stderr) = proc.communicate()
364        if proc.returncode != 0:
365            return proc.returncode, stdout, stderr
366
367        argv = shlex.split(adb + ' wait-for-device')
368        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
369        (stdout, stderr) = proc.communicate()
370        return proc.returncode, stdout, stderr
371
372class RemountTask:
373    def get_type(self):
374        return TASK_REMOUNT
375
376    def get_name(self):
377        return ""
378
379    def execute(self):
380        argv = shlex.split(adb + ' remount')
381        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
382        (stdout, stderr) = proc.communicate()
383        # adb remount returns 0 even if the operation failed, so check stdout
384        if stdout.startswith('remount failed:'):
385            return 1, stdout, stderr
386        return proc.returncode, stdout, stderr
387
388class CompoundTask:
389    def __init__(self, type, tasks):
390        self.type = type
391        self.tasks = tasks
392
393    def get_type(self):
394        return self.type
395
396    def get_name(self):
397        return ""
398
399    def execute(self):
400        for t in self.tasks:
401            (returncode, stdout, stderr) = t.execute()
402            if returncode != 0:
403                return returncode, stdout, stderr
404        return 0, "", ""
405
406def _create_disable_overlays_task():
407    tasks = [
408        RmTask("/vendor/overlay/framework_a.apk"),
409        RmTask("/vendor/overlay/framework_b.apk"),
410        RmTask("/data/resource-cache/vendor@overlay@framework_a.apk@idmap"),
411        RmTask("/data/resource-cache/vendor@overlay@framework_b.apk@idmap"),
412        RmTask("/vendor/overlay/app_a.apk"),
413        RmTask("/vendor/overlay/app_b.apk"),
414        RmTask("/data/resource-cache/vendor@overlay@app_a.apk@idmap"),
415        RmTask("/data/resource-cache/vendor@overlay@app_b.apk@idmap"),
416    ]
417    return CompoundTask(TASK_DISABLE_OVERLAYS, tasks)
418
419def _create_enable_single_overlay_task():
420    tasks = [
421        _create_disable_overlays_task(),
422        MkdirTask('/system/vendor'),
423        MkdirTask('/vendor/overlay'),
424        PushTask('/data/app/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_a.apk'),
425        PushTask('/data/app/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
426    ]
427    return CompoundTask(TASK_ENABLE_SINGLE_OVERLAY, tasks)
428
429def _create_enable_multiple_overlays_task():
430    tasks = [
431        _create_disable_overlays_task(),
432        MkdirTask('/system/vendor'),
433        MkdirTask('/vendor/overlay'),
434
435        PushTask('/data/app/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_b.apk'),
436        PushTask('/data/app/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
437        PushTask('/data/app/com.android.overlaytest.second_app_overlay.apk', '/vendor/overlay/app_b.apk'),
438    ]
439    return CompoundTask(TASK_ENABLE_MULTIPLE_OVERLAYS, tasks)
440
441def _create_setup_idmap_path_task(idmaps, symlinks):
442    tasks = [
443        _create_enable_single_overlay_task(),
444        RmTask(symlinks),
445        RmTask(idmaps),
446        MkdirTask(idmaps),
447        MkdirTask(symlinks),
448    ]
449    return CompoundTask(TASK_SETUP_IDMAP_PATH, tasks)
450
451def _create_setup_idmap_scan_task(idmaps, symlinks):
452    tasks = [
453        _create_enable_single_overlay_task(),
454        RmTask(symlinks),
455        RmTask(idmaps),
456        MkdirTask(idmaps),
457        MkdirTask(symlinks),
458        _create_enable_multiple_overlays_task(),
459    ]
460    return CompoundTask(TASK_SETUP_IDMAP_SCAN, tasks)
461
462def _handle_instrumentation_task_output(stdout, printer):
463    regex_status_code = re.compile(r'^INSTRUMENTATION_STATUS_CODE: -?(\d+)')
464    regex_name = re.compile(r'^INSTRUMENTATION_STATUS: test=(.*)')
465    regex_begin_stack = re.compile(r'^INSTRUMENTATION_STATUS: stack=(.*)')
466    regex_end_stack = re.compile(r'^$')
467
468    failed_tests = 0
469    current_test = None
470    current_stack = []
471    mode_stack = False
472    for line in stdout.split("\n"):
473        line = line.rstrip() # strip \r from adb output
474        m = regex_status_code.match(line)
475        if m:
476            c = int(m.group(1))
477            if c == 1:
478                printer.begin(TASK_INSTRUMENTATION_TEST, current_test)
479            elif c == 0:
480                printer.end_pass(TASK_INSTRUMENTATION_TEST, current_test)
481            else:
482                failed_tests += 1
483                current_stack.append("\n")
484                msg = "\n".join(current_stack)
485                printer.end_fail(TASK_INSTRUMENTATION_TEST, current_test, msg.rstrip() + '\n')
486            continue
487
488        m = regex_name.match(line)
489        if m:
490            current_test = m.group(1)
491            continue
492
493        m = regex_begin_stack.match(line)
494        if m:
495            mode_stack = True
496            current_stack = []
497            current_stack.append("  " + m.group(1))
498            continue
499
500        m = regex_end_stack.match(line)
501        if m:
502            mode_stack = False
503            continue
504
505        if mode_stack:
506            current_stack.append("    " + line.strip())
507
508    return failed_tests
509
510def _set_adb_device(option, opt, value, parser):
511    global adb
512    if opt == '-d' or opt == '--device':
513        adb = 'adb -d'
514    if opt == '-e' or opt == '--emulator':
515        adb = 'adb -e'
516    if opt == '-s' or opt == '--serial':
517        adb = 'adb -s ' + value
518
519def _create_opt_parser():
520    parser = optparse.OptionParser()
521    parser.add_option('-d', '--device', action='callback', callback=_set_adb_device,
522            help='pass -d to adb')
523    parser.add_option('-e', '--emulator', action='callback', callback=_set_adb_device,
524            help='pass -e to adb')
525    parser.add_option('-s', '--serial', type="str", action='callback', callback=_set_adb_device,
526            help='pass -s <serical> to adb')
527    parser.add_option('-C', '--no-color', action='store_false',
528            dest='use_color', default=True,
529            help='disable color escape sequences in output')
530    parser.add_option('-q', '--quiet', action='store_true',
531            dest='quiet_mode', default=False,
532            help='quiet mode, output only results')
533    parser.add_option('-b', '--no-build', action='store_false',
534            dest='do_build', default=True,
535            help='do not rebuild test projects')
536    parser.add_option('-k', '--continue', action='store_true',
537            dest='do_continue', default=False,
538            help='do not rebuild test projects')
539    parser.add_option('-i', '--test-idmap', action='store_true',
540            dest='test_idmap', default=False,
541            help='run tests for single overlay')
542    parser.add_option('-0', '--test-no-overlay', action='store_true',
543            dest='test_no_overlay', default=False,
544            help='run tests without any overlay')
545    parser.add_option('-1', '--test-single-overlay', action='store_true',
546            dest='test_single_overlay', default=False,
547            help='run tests for single overlay')
548    parser.add_option('-2', '--test-multiple-overlays', action='store_true',
549            dest='test_multiple_overlays', default=False,
550            help='run tests for multiple overlays')
551    return parser
552
553if __name__ == '__main__':
554    opt_parser = _create_opt_parser()
555    opts, args = opt_parser.parse_args(sys.argv[1:])
556    if not opts.test_idmap and not opts.test_no_overlay and not opts.test_single_overlay and not opts.test_multiple_overlays:
557        opts.test_idmap = True
558        opts.test_no_overlay = True
559        opts.test_single_overlay = True
560        opts.test_multiple_overlays = True
561    if len(args) > 0:
562        opt_parser.error("unexpected arguments: %s" % " ".join(args))
563        # will never reach this: opt_parser.error will call sys.exit
564
565    if opts.quiet_mode:
566        printer = QuietPrinter()
567    else:
568        printer = VerbosePrinter(opts.use_color)
569    tasks = []
570
571    # must be in the same directory as this script for compilation tasks to work
572    script = sys.argv[0]
573    dirname = os.path.dirname(script)
574    wd = os.path.realpath(dirname)
575    os.chdir(wd)
576
577    # build test cases
578    if opts.do_build:
579        tasks.append(CompilationTask('OverlayTest/Android.mk'))
580        tasks.append(CompilationTask('OverlayTestOverlay/Android.mk'))
581        tasks.append(CompilationTask('OverlayAppFirst/Android.mk'))
582        tasks.append(CompilationTask('OverlayAppSecond/Android.mk'))
583
584    # remount filesystem, install test project
585    tasks.append(RootTask())
586    tasks.append(RemountTask())
587    tasks.append(PushTask('/system/app/OverlayTest.apk', '/system/app/OverlayTest.apk'))
588
589    # test idmap
590    if opts.test_idmap:
591        idmaps='/data/local/tmp/idmaps'
592        symlinks='/data/local/tmp/symlinks'
593
594        # idmap --path
595        tasks.append(StopTask())
596        tasks.append(_create_setup_idmap_path_task(idmaps, symlinks))
597        tasks.append(StartTask())
598        tasks.append(IdmapPathTask('/vendor/overlay/framework_a.apk', '/system/framework/framework-res.apk', idmaps + '/a.idmap'))
599        tasks.append(FileExistsTest(idmaps + '/a.idmap'))
600        tasks.append(GrepIdmapTest(idmaps + '/a.idmap', 'bool/config_annoy_dianne', 1))
601
602        # idmap --scan
603        idmap = idmaps + '/vendor@overlay@framework_b.apk@idmap'
604        tasks.append(StopTask())
605        tasks.append(_create_setup_idmap_scan_task(idmaps, symlinks))
606        tasks.append(StartTask())
607        tasks.append(IdmapScanTask('/vendor/overlay', 'android', '/system/framework/framework-res.apk', idmaps, symlinks))
608        tasks.append(FileExistsTest(idmap))
609        tasks.append(GrepIdmapTest(idmap, 'bool/config_annoy_dianne', 1))
610
611        # overlays.list
612        overlays_list_path = '/data/resource-cache/overlays.list'
613        expected_content = '''\
614/vendor/overlay/framework_b.apk /data/resource-cache/vendor@overlay@framework_b.apk@idmap
615'''
616        tasks.append(FileExistsTest(overlays_list_path))
617        tasks.append(Md5Test(overlays_list_path, expected_content))
618
619        # idmap cleanup
620        tasks.append(RmTask(symlinks))
621        tasks.append(RmTask(idmaps))
622
623    # test no overlay
624    if opts.test_no_overlay:
625        tasks.append(StopTask())
626        tasks.append(_create_disable_overlays_task())
627        tasks.append(StartTask())
628        tasks.append(InstrumentationTask('com.android.overlaytest.WithoutOverlayTest'))
629
630    # test single overlay
631    if opts.test_single_overlay:
632        tasks.append(StopTask())
633        tasks.append(_create_enable_single_overlay_task())
634        tasks.append(StartTask())
635        tasks.append(InstrumentationTask('com.android.overlaytest.WithOverlayTest'))
636
637    # test multiple overlays
638    if opts.test_multiple_overlays:
639        tasks.append(StopTask())
640        tasks.append(_create_enable_multiple_overlays_task())
641        tasks.append(StartTask())
642        tasks.append(InstrumentationTask('com.android.overlaytest.WithMultipleOverlaysTest'))
643
644    ignored_errors = 0
645    for t in tasks:
646        type = t.get_type()
647        name = t.get_name()
648        if type == TASK_INSTRUMENTATION:
649            # InstrumentationTask will run several tests, but we want it
650            # to appear as if each test was run individually. Calling
651            # "am instrument" with a single test method is prohibitively
652            # expensive, so let's instead post-process the output to
653            # emulate individual calls.
654            retcode, stdout, stderr = t.execute()
655            if retcode != 0:
656                printer.begin(TASK_INSTRUMENTATION, name)
657                printer.end_fail(TASK_INSTRUMENTATION, name, stderr)
658                sys.exit(retcode)
659            retcode = _handle_instrumentation_task_output(stdout, printer)
660            if retcode != 0:
661                if not opts.do_continue:
662                    sys.exit(retcode)
663                else:
664                    ignored_errors += retcode
665        else:
666            printer.begin(type, name)
667            retcode, stdout, stderr = t.execute()
668            if retcode == 0:
669                printer.end_pass(type, name)
670            if retcode != 0:
671                if len(stderr) == 0:
672                    # hope for output from stdout instead (true for eg adb shell rm)
673                    stderr = stdout
674                printer.end_fail(type, name, stderr)
675                if not opts.do_continue:
676                    sys.exit(retcode)
677                else:
678                    ignored_errors += retcode
679    sys.exit(ignored_errors)
680