android.py revision da22befd8006e4927a039c503fb688f3643f07a5 
1#    Copyright 2013-2015 ARM Limited
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15
16
17"""
18Utility functions for working with Android devices through adb.
19
20"""
21# pylint: disable=E1103
22import os
23import time
24import subprocess
25import logging
26import re
27import threading
28import tempfile
29import Queue
30from collections import defaultdict
31
32from devlib.exception import TargetError, HostError, DevlibError
33from devlib.utils.misc import check_output, which, memoized, ABI_MAP
34from devlib.utils.misc import escape_single_quotes, escape_double_quotes
35from devlib import host
36
37
38logger = logging.getLogger('android')
39
40MAX_ATTEMPTS = 5
41AM_START_ERROR = re.compile(r"Error: Activity.*")
42
43# See:
44# http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
45ANDROID_VERSION_MAP = {
46    23: 'MARSHMALLOW',
47    22: 'LOLLYPOP_MR1',
48    21: 'LOLLYPOP',
49    20: 'KITKAT_WATCH',
50    19: 'KITKAT',
51    18: 'JELLY_BEAN_MR2',
52    17: 'JELLY_BEAN_MR1',
53    16: 'JELLY_BEAN',
54    15: 'ICE_CREAM_SANDWICH_MR1',
55    14: 'ICE_CREAM_SANDWICH',
56    13: 'HONEYCOMB_MR2',
57    12: 'HONEYCOMB_MR1',
58    11: 'HONEYCOMB',
59    10: 'GINGERBREAD_MR1',
60    9: 'GINGERBREAD',
61    8: 'FROYO',
62    7: 'ECLAIR_MR1',
63    6: 'ECLAIR_0_1',
64    5: 'ECLAIR',
65    4: 'DONUT',
66    3: 'CUPCAKE',
67    2: 'BASE_1_1',
68    1: 'BASE',
69}
70
71
72# Initialized in functions near the botton of the file
73android_home = None
74platform_tools = None
75adb = None
76aapt = None
77fastboot = None
78
79
80class AndroidProperties(object):
81
82    def __init__(self, text):
83        self._properties = {}
84        self.parse(text)
85
86    def parse(self, text):
87        self._properties = dict(re.findall(r'\[(.*?)\]:\s+\[(.*?)\]', text))
88
89    def iteritems(self):
90        return self._properties.iteritems()
91
92    def __iter__(self):
93        return iter(self._properties)
94
95    def __getattr__(self, name):
96        return self._properties.get(name)
97
98    __getitem__ = __getattr__
99
100
101class AdbDevice(object):
102
103    def __init__(self, name, status):
104        self.name = name
105        self.status = status
106
107    def __cmp__(self, other):
108        if isinstance(other, AdbDevice):
109            return cmp(self.name, other.name)
110        else:
111            return cmp(self.name, other)
112
113    def __str__(self):
114        return 'AdbDevice({}, {})'.format(self.name, self.status)
115
116    __repr__ = __str__
117
118
119class ApkInfo(object):
120
121    version_regex = re.compile(r"name='(?P<name>[^']+)' versionCode='(?P<vcode>[^']+)' versionName='(?P<vname>[^']+)'")
122    name_regex = re.compile(r"name='(?P<name>[^']+)'")
123
124    def __init__(self, path=None):
125        self.path = path
126        self.package = None
127        self.activity = None
128        self.label = None
129        self.version_name = None
130        self.version_code = None
131        self.native_code = None
132        self.parse(path)
133
134    def parse(self, apk_path):
135        _check_env()
136        command = [aapt, 'dump', 'badging', apk_path]
137        logger.debug(' '.join(command))
138        try:
139            output = subprocess.check_output(command, stderr=subprocess.STDOUT)
140        except subprocess.CalledProcessError as e:
141            raise HostError('Error parsing APK file {}. `aapt` says:\n{}'
142                            .format(apk_path, e.output))
143        for line in output.split('\n'):
144            if line.startswith('application-label:'):
145                self.label = line.split(':')[1].strip().replace('\'', '')
146            elif line.startswith('package:'):
147                match = self.version_regex.search(line)
148                if match:
149                    self.package = match.group('name')
150                    self.version_code = match.group('vcode')
151                    self.version_name = match.group('vname')
152            elif line.startswith('launchable-activity:'):
153                match = self.name_regex.search(line)
154                self.activity = match.group('name')
155            elif line.startswith('native-code'):
156                apk_abis = [entry.strip() for entry in line.split(':')[1].split("'") if entry.strip()]
157                mapped_abis = []
158                for apk_abi in apk_abis:
159                    found = False
160                    for abi, architectures in ABI_MAP.iteritems():
161                        if apk_abi in architectures:
162                            mapped_abis.append(abi)
163                            found = True
164                            break
165                    if not found:
166                        mapped_abis.append(apk_abi)
167                self.native_code = mapped_abis
168            else:
169                pass  # not interested
170
171
172class AdbConnection(object):
173
174    # maintains the count of parallel active connections to a device, so that
175    # adb disconnect is not invoked untill all connections are closed
176    active_connections = defaultdict(int)
177    default_timeout = 10
178    ls_command = 'ls'
179
180    @property
181    def name(self):
182        return self.device
183
184    @property
185    @memoized
186    def newline_separator(self):
187        output = adb_command(self.device,
188                             "shell '({}); echo \"\n$?\"'".format(self.ls_command), adb_server=self.adb_server)
189        if output.endswith('\r\n'):
190            return '\r\n'
191        elif output.endswith('\n'):
192            return '\n'
193        else:
194            raise DevlibError("Unknown line ending")
195
196    # Again, we need to handle boards where the default output format from ls is
197    # single column *and* boards where the default output is multi-column.
198    # We need to do this purely because the '-1' option causes errors on older
199    # versions of the ls tool in Android pre-v7.
200    def _setup_ls(self):
201        command = "shell '(ls -1); echo \"\n$?\"'"
202        try:
203            output = adb_command(self.device, command, timeout=self.timeout, adb_server=self.adb_server)
204        except subprocess.CalledProcessError as e:
205            raise HostError(
206                'Failed to set up ls command on Android device. Output:\n'
207                + e.output)
208        lines = output.splitlines()
209        retval = lines[-1].strip()
210        if int(retval) == 0:
211            self.ls_command = 'ls -1'
212        else:
213            self.ls_command = 'ls'
214        logger.debug("ls command is set to {}".format(self.ls_command))
215
216    def __init__(self, device=None, timeout=None, platform=None, adb_server=None):
217        self.timeout = timeout if timeout is not None else self.default_timeout
218        if device is None:
219            device = adb_get_device(timeout=timeout, adb_server=adb_server)
220        self.device = device
221        self.adb_server = adb_server
222        adb_connect(self.device)
223        AdbConnection.active_connections[self.device] += 1
224        self._setup_ls()
225
226    def push(self, source, dest, timeout=None):
227        if timeout is None:
228            timeout = self.timeout
229        command = "push '{}' '{}'".format(source, dest)
230        if not os.path.exists(source):
231            raise HostError('No such file "{}"'.format(source))
232        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
233
234    def pull(self, source, dest, timeout=None):
235        if timeout is None:
236            timeout = self.timeout
237        # Pull all files matching a wildcard expression
238        if os.path.isdir(dest) and \
239           ('*' in source or '?' in source):
240            command = 'shell {} {}'.format(self.ls_command, source)
241            output = adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
242            for line in output.splitlines():
243                command = "pull '{}' '{}'".format(line.strip(), dest)
244                adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
245            return
246        command = "pull '{}' '{}'".format(source, dest)
247        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
248
249    def execute(self, command, timeout=None, check_exit_code=False,
250                as_root=False, strip_colors=True):
251        return adb_shell(self.device, command, timeout, check_exit_code,
252                         as_root, self.newline_separator,adb_server=self.adb_server)
253
254    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
255        return adb_background_shell(self.device, command, stdout, stderr, as_root)
256
257    def close(self):
258        AdbConnection.active_connections[self.device] -= 1
259        if AdbConnection.active_connections[self.device] <= 0:
260            adb_disconnect(self.device)
261            del AdbConnection.active_connections[self.device]
262
263    def cancel_running_command(self):
264        # adbd multiplexes commands so that they don't interfer with each
265        # other, so there is no need to explicitly cancel a running command
266        # before the next one can be issued.
267        pass
268
269
270def fastboot_command(command, timeout=None, device=None):
271    _check_env()
272    target = '-s {}'.format(device) if device else ''
273    full_command = 'fastboot {} {}'.format(target, command)
274    logger.debug(full_command)
275    output, _ = check_output(full_command, timeout, shell=True)
276    return output
277
278
279def fastboot_flash_partition(partition, path_to_image):
280    command = 'flash {} {}'.format(partition, path_to_image)
281    fastboot_command(command)
282
283
284def adb_get_device(timeout=None, adb_server=None):
285    """
286    Returns the serial number of a connected android device.
287
288    If there are more than one device connected to the machine, or it could not
289    find any device connected, :class:`devlib.exceptions.HostError` is raised.
290    """
291    # TODO this is a hacky way to issue a adb command to all listed devices
292
293    # Ensure server is started so the 'daemon started successfully' message
294    # doesn't confuse the parsing below
295    adb_command(None, 'start-server', adb_server=adb_server)
296
297    # The output of calling adb devices consists of a heading line then
298    # a list of the devices sperated by new line
299    # The last line is a blank new line. in otherwords, if there is a device found
300    # then the output length is 2 + (1 for each device)
301    start = time.time()
302    while True:
303        output = adb_command(None, "devices", adb_server=adb_server).splitlines()  # pylint: disable=E1103
304        output_length = len(output)
305        if output_length == 3:
306            # output[1] is the 2nd line in the output which has the device name
307            # Splitting the line by '\t' gives a list of two indexes, which has
308            # device serial in 0 number and device type in 1.
309            return output[1].split('\t')[0]
310        elif output_length > 3:
311            message = '{} Android devices found; either explicitly specify ' +\
312                      'the device you want, or make sure only one is connected.'
313            raise HostError(message.format(output_length - 2))
314        else:
315            if timeout < time.time() - start:
316                raise HostError('No device is connected and available')
317            time.sleep(1)
318
319
320def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
321    _check_env()
322    # Connect is required only for ADB-over-IP
323    if "." not in device:
324        logger.debug('Device connected via USB, connect not required')
325        return
326    tries = 0
327    output = None
328    while tries <= attempts:
329        tries += 1
330        if device:
331            command = 'adb connect {}'.format(device)
332            logger.debug(command)
333            output, _ = check_output(command, shell=True, timeout=timeout)
334        if _ping(device):
335            break
336        time.sleep(10)
337    else:  # did not connect to the device
338        message = 'Could not connect to {}'.format(device or 'a device')
339        if output:
340            message += '; got: "{}"'.format(output)
341        raise HostError(message)
342
343
344def adb_disconnect(device):
345    _check_env()
346    if not device:
347        return
348    if ":" in device and device in adb_list_devices():
349        command = "adb disconnect " + device
350        logger.debug(command)
351        retval = subprocess.call(command, stdout=open(os.devnull, 'wb'), shell=True)
352        if retval:
353            raise TargetError('"{}" returned {}'.format(command, retval))
354
355
356def _ping(device):
357    _check_env()
358    device_string = ' -s {}'.format(device) if device else ''
359    command = "adb{} shell \"ls / > /dev/null\"".format(device_string)
360    logger.debug(command)
361    result = subprocess.call(command, stderr=subprocess.PIPE, shell=True)
362    if not result:
363        return True
364    else:
365        return False
366
367
368def adb_shell(device, command, timeout=None, check_exit_code=False,
369              as_root=False, newline_separator='\r\n', adb_server=None):  # NOQA
370    _check_env()
371    if as_root:
372        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
373    device_part = []
374    if adb_server:
375        device_part = ['-H', adb_server]
376    device_part += ['-s', device] if device else []
377
378    # On older combinations of ADB/Android versions, the adb host command always
379    # exits with 0 if it was able to run the command on the target, even if the
380    # command failed (https://code.google.com/p/android/issues/detail?id=3254).
381    # Homogenise this behaviour by running the command then echoing the exit
382    # code.
383    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
384    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
385    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
386    raw_output, error = check_output(actual_command, timeout, shell=False)
387    if raw_output:
388        try:
389            output, exit_code, _ = raw_output.rsplit(newline_separator, 2)
390        except ValueError:
391            exit_code, _ = raw_output.rsplit(newline_separator, 1)
392            output = ''
393    else:  # raw_output is empty
394        exit_code = '969696'  # just because
395        output = ''
396
397    if check_exit_code:
398        exit_code = exit_code.strip()
399        re_search = AM_START_ERROR.findall('{}\n{}'.format(output, error))
400        if exit_code.isdigit():
401            if int(exit_code):
402                message = ('Got exit code {}\nfrom target command: {}\n'
403                           'STDOUT: {}\nSTDERR: {}')
404                raise TargetError(message.format(exit_code, command, output, error))
405            elif re_search:
406                message = 'Could not start activity; got the following:\n{}'
407                raise TargetError(message.format(re_search[0]))
408        else:  # not all digits
409            if re_search:
410                message = 'Could not start activity; got the following:\n{}'
411                raise TargetError(message.format(re_search[0]))
412            else:
413                message = 'adb has returned early; did not get an exit code. '\
414                          'Was kill-server invoked?\nOUTPUT:\n-----\n{}\n'\
415                          '-----\nERROR:\n-----\n{}\n-----'
416                raise TargetError(message.format(raw_output, error))
417
418    return output
419
420
421def adb_background_shell(device, command,
422                         stdout=subprocess.PIPE,
423                         stderr=subprocess.PIPE,
424                         as_root=False):
425    """Runs the sepcified command in a subprocess, returning the the Popen object."""
426    _check_env()
427    if as_root:
428        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
429    device_string = ' -s {}'.format(device) if device else ''
430    full_command = 'adb{} shell "{}"'.format(device_string, escape_double_quotes(command))
431    logger.debug(full_command)
432    return subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
433
434
435def adb_list_devices(adb_server=None):
436    output = adb_command(None, 'devices',adb_server=adb_server)
437    devices = []
438    for line in output.splitlines():
439        parts = [p.strip() for p in line.split()]
440        if len(parts) == 2:
441            devices.append(AdbDevice(*parts))
442    return devices
443
444
445def get_adb_command(device, command, timeout=None,adb_server=None):
446    _check_env()
447    device_string = ""
448    if adb_server != None:
449        device_string = ' -H {}'.format(adb_server)
450    device_string += ' -s {}'.format(device) if device else ''
451    return "adb{} {}".format(device_string, command)
452
453def adb_command(device, command, timeout=None,adb_server=None):
454    full_command = get_adb_command(device, command, timeout, adb_server)
455    logger.debug(full_command)
456    output, _ = check_output(full_command, timeout, shell=True)
457    return output
458
459def grant_app_permissions(target, package):
460    """
461    Grant an app all the permissions it may ask for
462    """
463    dumpsys = target.execute('dumpsys package {}'.format(package))
464
465    permissions = re.search(
466        'requested permissions:\s*(?P<permissions>(android.permission.+\s*)+)', dumpsys
467    )
468    if permissions is None:
469        return
470    permissions = permissions.group('permissions').replace(" ", "").splitlines()
471
472    for permission in permissions:
473        try:
474            target.execute('pm grant {} {}'.format(package, permission))
475        except TargetError:
476            logger.debug('Cannot grant {}'.format(permission))
477
478
479# Messy environment initialisation stuff...
480
481class _AndroidEnvironment(object):
482
483    def __init__(self):
484        self.android_home = None
485        self.platform_tools = None
486        self.adb = None
487        self.aapt = None
488        self.fastboot = None
489
490
491def _initialize_with_android_home(env):
492    logger.debug('Using ANDROID_HOME from the environment.')
493    env.android_home = android_home
494    env.platform_tools = os.path.join(android_home, 'platform-tools')
495    os.environ['PATH'] = env.platform_tools + os.pathsep + os.environ['PATH']
496    _init_common(env)
497    return env
498
499
500def _initialize_without_android_home(env):
501    adb_full_path = which('adb')
502    if adb_full_path:
503        env.adb = 'adb'
504    else:
505        raise HostError('ANDROID_HOME is not set and adb is not in PATH. '
506                        'Have you installed Android SDK?')
507    logger.debug('Discovering ANDROID_HOME from adb path.')
508    env.platform_tools = os.path.dirname(adb_full_path)
509    env.android_home = os.path.dirname(env.platform_tools)
510    _init_common(env)
511    return env
512
513
514def _init_common(env):
515    logger.debug('ANDROID_HOME: {}'.format(env.android_home))
516    build_tools_directory = os.path.join(env.android_home, 'build-tools')
517    if not os.path.isdir(build_tools_directory):
518        msg = '''ANDROID_HOME ({}) does not appear to have valid Android SDK install
519                 (cannot find build-tools)'''
520        raise HostError(msg.format(env.android_home))
521    versions = os.listdir(build_tools_directory)
522    for version in reversed(sorted(versions)):
523        aapt_path = os.path.join(build_tools_directory, version, 'aapt')
524        if os.path.isfile(aapt_path):
525            logger.debug('Using aapt for version {}'.format(version))
526            env.aapt = aapt_path
527            break
528    else:
529        raise HostError('aapt not found. Please make sure at least one Android '
530                        'platform is installed.')
531
532
533def _check_env():
534    global android_home, platform_tools, adb, aapt  # pylint: disable=W0603
535    if not android_home:
536        android_home = os.getenv('ANDROID_HOME')
537        if android_home:
538            _env = _initialize_with_android_home(_AndroidEnvironment())
539        else:
540            _env = _initialize_without_android_home(_AndroidEnvironment())
541        android_home = _env.android_home
542        platform_tools = _env.platform_tools
543        adb = _env.adb
544        aapt = _env.aapt
545
546class LogcatMonitor(threading.Thread):
547    """
548    Helper class for monitoring Anroid's logcat
549
550    :param target: Android target to monitor
551    :type target: :class:`AndroidTarget`
552
553                    device. Logcat entries that don't match any will not be
554                    seen. If omitted, all entries will be sent to host.
555    :type regexps: list(str)
556    """
557
558    FLUSH_SIZE = 1000
559
560    @property
561    def logfile(self):
562        return self._logfile
563
564    def __init__(self, target, regexps=None):
565        super(LogcatMonitor, self).__init__()
566
567        self.target = target
568
569        self._started = threading.Event()
570        self._stopped = threading.Event()
571        self._match_found = threading.Event()
572
573        self._sought = None
574        self._found = None
575
576        self._lines = Queue.Queue()
577        self._datalock = threading.Lock()
578        self._regexps = regexps
579
580    def start(self, outfile=None):
581        """
582        Start logcat and begin monitoring
583
584        :param outfile: Optional path to file to store all logcat entries
585        :type outfile: str
586        """
587        if outfile:
588            self._logfile = outfile
589        else:
590            fd, self._logfile = tempfile.mkstemp()
591            os.close(fd)
592        logger.debug('Logging to {}'.format(self._logfile))
593
594        super(LogcatMonitor, self).start()
595
596    def run(self):
597        self.target.clear_logcat()
598
599        logcat_cmd = 'logcat'
600
601        # Join all requested regexps with an 'or'
602        if self._regexps:
603            regexp = '{}'.format('|'.join(self._regexps))
604            if len(self._regexps) > 1:
605                regexp = '({})'.format(regexp)
606            logcat_cmd = '{} -e "{}"'.format(logcat_cmd, regexp)
607
608        logger.debug('logcat command ="{}"'.format(logcat_cmd))
609        self._logcat = self.target.background(logcat_cmd)
610
611        self._started.set()
612
613        while not self._stopped.is_set():
614            line = self._logcat.stdout.readline(1024)
615            if line:
616                self._add_line(line)
617
618    def stop(self):
619        if not self.is_alive():
620            logger.warning('LogcatMonitor.stop called before start')
621            return
622
623        # Make sure we've started before we try to kill anything
624        self._started.wait()
625
626        # Kill the underlying logcat process
627        # This will unblock self._logcat.stdout.readline()
628        host.kill_children(self._logcat.pid)
629        self._logcat.kill()
630
631        self._stopped.set()
632        self.join()
633
634        self._flush_lines()
635
636    def _add_line(self, line):
637        self._lines.put(line)
638
639        if self._sought and re.match(self._sought, line):
640            self._found = line
641            self._match_found.set()
642
643        if self._lines.qsize() >= self.FLUSH_SIZE:
644            self._flush_lines()
645
646    def _flush_lines(self):
647        with self._datalock:
648            with open(self._logfile, 'a') as fh:
649                while not self._lines.empty():
650                    fh.write(self._lines.get())
651
652    def clear_log(self):
653        with self._datalock:
654            while not self._lines.empty():
655                self._lines.get()
656
657            with open(self._logfile, 'w') as fh:
658                pass
659
660    def get_log(self):
661        """
662        Return the list of lines found by the monitor
663        """
664        self._flush_lines()
665
666        with self._datalock:
667            with open(self._logfile, 'r') as fh:
668                res = [line for line in fh]
669
670        return res
671
672    def search(self, regexp):
673        """
674        Search a line that matches a regexp in the logcat log
675        Return immediatly
676        """
677        res = []
678
679        self._flush_lines()
680
681        with self._datalock:
682            with open(self._logfile, 'r') as fh:
683                for line in fh:
684                    if re.match(regexp, line):
685                        res.append(line)
686
687        return res
688
689    def wait_for(self, regexp, timeout=30):
690        """
691        Search a line that matches a regexp in the logcat log
692        Wait for it to appear if it's not found
693
694        :param regexp: regexp to search
695        :type regexp: str
696
697        :param timeout: Timeout in seconds, before rasing RuntimeError.
698                        ``None`` means wait indefinitely
699        :type timeout: number
700
701        :returns: List of matched strings
702        """
703        res = self.search(regexp)
704
705        # Found some matches, return them
706        # Also return if thread not running
707        if len(res) > 0 or not self.is_alive():
708            return res
709
710        # Did not find any match, wait for one to pop up
711        self._sought = regexp
712        found = self._match_found.wait(timeout)
713        self._match_found.clear()
714        self._sought = None
715
716        if found:
717            return [self._found]
718        else:
719            raise RuntimeError('Logcat monitor timeout ({}s)'.format(timeout))
720