android.py revision fb5a260f4b4879503bd5385f96a97612b0e49d13 
1#    Copyright 2013-2015 ARM Limited
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15
16
17"""
18Utility functions for working with Android devices through adb.
19
20"""
21# pylint: disable=E1103
22import os
23import time
24import subprocess
25import logging
26import re
27import threading
28import tempfile
29import Queue
30from collections import defaultdict
31
32from devlib.exception import TargetError, HostError, DevlibError
33from devlib.utils.misc import check_output, which, memoized, ABI_MAP
34from devlib.utils.misc import escape_single_quotes, escape_double_quotes
35from devlib import host
36
37
38logger = logging.getLogger('android')
39
40MAX_ATTEMPTS = 5
41AM_START_ERROR = re.compile(r"Error: Activity.*")
42
43# See:
44# http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
45ANDROID_VERSION_MAP = {
46    23: 'MARSHMALLOW',
47    22: 'LOLLYPOP_MR1',
48    21: 'LOLLYPOP',
49    20: 'KITKAT_WATCH',
50    19: 'KITKAT',
51    18: 'JELLY_BEAN_MR2',
52    17: 'JELLY_BEAN_MR1',
53    16: 'JELLY_BEAN',
54    15: 'ICE_CREAM_SANDWICH_MR1',
55    14: 'ICE_CREAM_SANDWICH',
56    13: 'HONEYCOMB_MR2',
57    12: 'HONEYCOMB_MR1',
58    11: 'HONEYCOMB',
59    10: 'GINGERBREAD_MR1',
60    9: 'GINGERBREAD',
61    8: 'FROYO',
62    7: 'ECLAIR_MR1',
63    6: 'ECLAIR_0_1',
64    5: 'ECLAIR',
65    4: 'DONUT',
66    3: 'CUPCAKE',
67    2: 'BASE_1_1',
68    1: 'BASE',
69}
70
71
72# Initialized in functions near the botton of the file
73android_home = None
74platform_tools = None
75adb = None
76aapt = None
77fastboot = None
78
79
80class AndroidProperties(object):
81
82    def __init__(self, text):
83        self._properties = {}
84        self.parse(text)
85
86    def parse(self, text):
87        self._properties = dict(re.findall(r'\[(.*?)\]:\s+\[(.*?)\]', text))
88
89    def iteritems(self):
90        return self._properties.iteritems()
91
92    def __iter__(self):
93        return iter(self._properties)
94
95    def __getattr__(self, name):
96        return self._properties.get(name)
97
98    __getitem__ = __getattr__
99
100
101class AdbDevice(object):
102
103    def __init__(self, name, status):
104        self.name = name
105        self.status = status
106
107    def __cmp__(self, other):
108        if isinstance(other, AdbDevice):
109            return cmp(self.name, other.name)
110        else:
111            return cmp(self.name, other)
112
113    def __str__(self):
114        return 'AdbDevice({}, {})'.format(self.name, self.status)
115
116    __repr__ = __str__
117
118
119class ApkInfo(object):
120
121    version_regex = re.compile(r"name='(?P<name>[^']+)' versionCode='(?P<vcode>[^']+)' versionName='(?P<vname>[^']+)'")
122    name_regex = re.compile(r"name='(?P<name>[^']+)'")
123
124    def __init__(self, path=None):
125        self.path = path
126        self.package = None
127        self.activity = None
128        self.label = None
129        self.version_name = None
130        self.version_code = None
131        self.native_code = None
132        self.parse(path)
133
134    def parse(self, apk_path):
135        _check_env()
136        command = [aapt, 'dump', 'badging', apk_path]
137        logger.debug(' '.join(command))
138        output = subprocess.check_output(command)
139        for line in output.split('\n'):
140            if line.startswith('application-label:'):
141                self.label = line.split(':')[1].strip().replace('\'', '')
142            elif line.startswith('package:'):
143                match = self.version_regex.search(line)
144                if match:
145                    self.package = match.group('name')
146                    self.version_code = match.group('vcode')
147                    self.version_name = match.group('vname')
148            elif line.startswith('launchable-activity:'):
149                match = self.name_regex.search(line)
150                self.activity = match.group('name')
151            elif line.startswith('native-code'):
152                apk_abis = [entry.strip() for entry in line.split(':')[1].split("'") if entry.strip()]
153                mapped_abis = []
154                for apk_abi in apk_abis:
155                    found = False
156                    for abi, architectures in ABI_MAP.iteritems():
157                        if apk_abi in architectures:
158                            mapped_abis.append(abi)
159                            found = True
160                            break
161                    if not found:
162                        mapped_abis.append(apk_abi)
163                self.native_code = mapped_abis
164            else:
165                pass  # not interested
166
167
168class AdbConnection(object):
169
170    # maintains the count of parallel active connections to a device, so that
171    # adb disconnect is not invoked untill all connections are closed
172    active_connections = defaultdict(int)
173    default_timeout = 10
174    ls_command = 'ls'
175
176    @property
177    def name(self):
178        return self.device
179
180    @property
181    @memoized
182    def newline_separator(self):
183        output = adb_command(self.device,
184                             "shell '({}); echo \"\n$?\"'".format(self.ls_command), adb_server=self.adb_server)
185        if output.endswith('\r\n'):
186            return '\r\n'
187        elif output.endswith('\n'):
188            return '\n'
189        else:
190            raise DevlibError("Unknown line ending")
191
192    # Again, we need to handle boards where the default output format from ls is
193    # single column *and* boards where the default output is multi-column.
194    # We need to do this purely because the '-1' option causes errors on older
195    # versions of the ls tool in Android pre-v7.
196    def _setup_ls(self):
197        command = "shell '(ls -1); echo \"\n$?\"'"
198        try:
199            output = adb_command(self.device, command, timeout=self.timeout, adb_server=self.adb_server)
200        except subprocess.CalledProcessError as e:
201            raise HostError(
202                'Failed to set up ls command on Android device. Output:\n'
203                + e.output)
204        lines = output.splitlines()
205        retval = lines[-1].strip()
206        if int(retval) == 0:
207            self.ls_command = 'ls -1'
208        else:
209            self.ls_command = 'ls'
210        logger.debug("ls command is set to {}".format(self.ls_command))
211
212    def __init__(self, device=None, timeout=None, platform=None, adb_server=None):
213        self.timeout = timeout if timeout is not None else self.default_timeout
214        if device is None:
215            device = adb_get_device(timeout=timeout, adb_server=adb_server)
216        self.device = device
217        self.adb_server = adb_server
218        adb_connect(self.device)
219        AdbConnection.active_connections[self.device] += 1
220        self._setup_ls()
221
222    def push(self, source, dest, timeout=None):
223        if timeout is None:
224            timeout = self.timeout
225        command = "push '{}' '{}'".format(source, dest)
226        if not os.path.exists(source):
227            raise HostError('No such file "{}"'.format(source))
228        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
229
230    def pull(self, source, dest, timeout=None):
231        if timeout is None:
232            timeout = self.timeout
233        # Pull all files matching a wildcard expression
234        if os.path.isdir(dest) and \
235           ('*' in source or '?' in source):
236            command = 'shell {} {}'.format(self.ls_command, source)
237            output = adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
238            for line in output.splitlines():
239                command = "pull '{}' '{}'".format(line.strip(), dest)
240                adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
241            return
242        command = "pull '{}' '{}'".format(source, dest)
243        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
244
245    def execute(self, command, timeout=None, check_exit_code=False,
246                as_root=False, strip_colors=True):
247        return adb_shell(self.device, command, timeout, check_exit_code,
248                         as_root, self.newline_separator,adb_server=self.adb_server)
249
250    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
251        return adb_background_shell(self.device, command, stdout, stderr, as_root)
252
253    def close(self):
254        AdbConnection.active_connections[self.device] -= 1
255        if AdbConnection.active_connections[self.device] <= 0:
256            adb_disconnect(self.device)
257            del AdbConnection.active_connections[self.device]
258
259    def cancel_running_command(self):
260        # adbd multiplexes commands so that they don't interfer with each
261        # other, so there is no need to explicitly cancel a running command
262        # before the next one can be issued.
263        pass
264
265
266def fastboot_command(command, timeout=None, device=None):
267    _check_env()
268    target = '-s {}'.format(device) if device else ''
269    full_command = 'fastboot {} {}'.format(target, command)
270    logger.debug(full_command)
271    output, _ = check_output(full_command, timeout, shell=True)
272    return output
273
274
275def fastboot_flash_partition(partition, path_to_image):
276    command = 'flash {} {}'.format(partition, path_to_image)
277    fastboot_command(command)
278
279
280def adb_get_device(timeout=None, adb_server=None):
281    """
282    Returns the serial number of a connected android device.
283
284    If there are more than one device connected to the machine, or it could not
285    find any device connected, :class:`devlib.exceptions.HostError` is raised.
286    """
287    # TODO this is a hacky way to issue a adb command to all listed devices
288
289    # The output of calling adb devices consists of a heading line then
290    # a list of the devices sperated by new line
291    # The last line is a blank new line. in otherwords, if there is a device found
292    # then the output length is 2 + (1 for each device)
293    start = time.time()
294    while True:
295        output = adb_command(None, "devices", adb_server=adb_server).splitlines()  # pylint: disable=E1103
296        output_length = len(output)
297        if output_length == 3:
298            # output[1] is the 2nd line in the output which has the device name
299            # Splitting the line by '\t' gives a list of two indexes, which has
300            # device serial in 0 number and device type in 1.
301            return output[1].split('\t')[0]
302        elif output_length > 3:
303            message = '{} Android devices found; either explicitly specify ' +\
304                      'the device you want, or make sure only one is connected.'
305            raise HostError(message.format(output_length - 2))
306        else:
307            if timeout < time.time() - start:
308                raise HostError('No device is connected and available')
309            time.sleep(1)
310
311
312def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
313    _check_env()
314    # Connect is required only for ADB-over-IP
315    if "." not in device:
316        logger.debug('Device connected via USB, connect not required')
317        return
318    tries = 0
319    output = None
320    while tries <= attempts:
321        tries += 1
322        if device:
323            command = 'adb connect {}'.format(device)
324            logger.debug(command)
325            output, _ = check_output(command, shell=True, timeout=timeout)
326        if _ping(device):
327            break
328        time.sleep(10)
329    else:  # did not connect to the device
330        message = 'Could not connect to {}'.format(device or 'a device')
331        if output:
332            message += '; got: "{}"'.format(output)
333        raise HostError(message)
334
335
336def adb_disconnect(device):
337    _check_env()
338    if not device:
339        return
340    if ":" in device and device in adb_list_devices():
341        command = "adb disconnect " + device
342        logger.debug(command)
343        retval = subprocess.call(command, stdout=open(os.devnull, 'wb'), shell=True)
344        if retval:
345            raise TargetError('"{}" returned {}'.format(command, retval))
346
347
348def _ping(device):
349    _check_env()
350    device_string = ' -s {}'.format(device) if device else ''
351    command = "adb{} shell \"ls / > /dev/null\"".format(device_string)
352    logger.debug(command)
353    result = subprocess.call(command, stderr=subprocess.PIPE, shell=True)
354    if not result:
355        return True
356    else:
357        return False
358
359
360def adb_shell(device, command, timeout=None, check_exit_code=False,
361              as_root=False, newline_separator='\r\n', adb_server=None):  # NOQA
362    _check_env()
363    if as_root:
364        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
365    device_part = []
366    if adb_server:
367        device_part = ['-H', adb_server]
368    device_part += ['-s', device] if device else []
369
370    # On older combinations of ADB/Android versions, the adb host command always
371    # exits with 0 if it was able to run the command on the target, even if the
372    # command failed (https://code.google.com/p/android/issues/detail?id=3254).
373    # Homogenise this behaviour by running the command then echoing the exit
374    # code.
375    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
376    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
377    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
378    raw_output, error = check_output(actual_command, timeout, shell=False)
379    if raw_output:
380        try:
381            output, exit_code, _ = raw_output.rsplit(newline_separator, 2)
382        except ValueError:
383            exit_code, _ = raw_output.rsplit(newline_separator, 1)
384            output = ''
385    else:  # raw_output is empty
386        exit_code = '969696'  # just because
387        output = ''
388
389    if check_exit_code:
390        exit_code = exit_code.strip()
391        re_search = AM_START_ERROR.findall('{}\n{}'.format(output, error))
392        if exit_code.isdigit():
393            if int(exit_code):
394                message = ('Got exit code {}\nfrom target command: {}\n'
395                           'STDOUT: {}\nSTDERR: {}')
396                raise TargetError(message.format(exit_code, command, output, error))
397            elif re_search:
398                message = 'Could not start activity; got the following:\n{}'
399                raise TargetError(message.format(re_search[0]))
400        else:  # not all digits
401            if re_search:
402                message = 'Could not start activity; got the following:\n{}'
403                raise TargetError(message.format(re_search[0]))
404            else:
405                message = 'adb has returned early; did not get an exit code. '\
406                          'Was kill-server invoked?\nOUTPUT:\n-----\n{}\n'\
407                          '-----\nERROR:\n-----\n{}\n-----'
408                raise TargetError(message.format(raw_output, error))
409
410    return output
411
412
413def adb_background_shell(device, command,
414                         stdout=subprocess.PIPE,
415                         stderr=subprocess.PIPE,
416                         as_root=False):
417    """Runs the sepcified command in a subprocess, returning the the Popen object."""
418    _check_env()
419    if as_root:
420        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
421    device_string = ' -s {}'.format(device) if device else ''
422    full_command = 'adb{} shell "{}"'.format(device_string, escape_double_quotes(command))
423    logger.debug(full_command)
424    return subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
425
426
427def adb_list_devices(adb_server=None):
428    output = adb_command(None, 'devices',adb_server=adb_server)
429    devices = []
430    for line in output.splitlines():
431        parts = [p.strip() for p in line.split()]
432        if len(parts) == 2:
433            devices.append(AdbDevice(*parts))
434    return devices
435
436
437def adb_command(device, command, timeout=None,adb_server=None):
438    _check_env()
439    device_string = ""
440    if adb_server != None:
441        device_string = ' -H {}'.format(adb_server)
442    device_string += ' -s {}'.format(device) if device else ''
443    full_command = "adb{} {}".format(device_string, command)
444    logger.debug(full_command)
445    output, _ = check_output(full_command, timeout, shell=True)
446    return output
447
448def grant_app_permissions(target, package):
449    """
450    Grant an app all the permissions it may ask for
451    """
452    dumpsys = target.execute('dumpsys package {}'.format(package))
453
454    permissions = re.search(
455        'requested permissions:\s*(?P<permissions>(android.permission.+\s*)+)', dumpsys
456    )
457    if permissions is None:
458        return
459    permissions = permissions.group('permissions').replace(" ", "").splitlines()
460
461    for permission in permissions:
462        try:
463            target.execute('pm grant {} {}'.format(package, permission))
464        except TargetError:
465            logger.debug('Cannot grant {}'.format(permission))
466
467
468# Messy environment initialisation stuff...
469
470class _AndroidEnvironment(object):
471
472    def __init__(self):
473        self.android_home = None
474        self.platform_tools = None
475        self.adb = None
476        self.aapt = None
477        self.fastboot = None
478
479
480def _initialize_with_android_home(env):
481    logger.debug('Using ANDROID_HOME from the environment.')
482    env.android_home = android_home
483    env.platform_tools = os.path.join(android_home, 'platform-tools')
484    os.environ['PATH'] = env.platform_tools + os.pathsep + os.environ['PATH']
485    _init_common(env)
486    return env
487
488
489def _initialize_without_android_home(env):
490    adb_full_path = which('adb')
491    if adb_full_path:
492        env.adb = 'adb'
493    else:
494        raise HostError('ANDROID_HOME is not set and adb is not in PATH. '
495                        'Have you installed Android SDK?')
496    logger.debug('Discovering ANDROID_HOME from adb path.')
497    env.platform_tools = os.path.dirname(adb_full_path)
498    env.android_home = os.path.dirname(env.platform_tools)
499    _init_common(env)
500    return env
501
502
503def _init_common(env):
504    logger.debug('ANDROID_HOME: {}'.format(env.android_home))
505    build_tools_directory = os.path.join(env.android_home, 'build-tools')
506    if not os.path.isdir(build_tools_directory):
507        msg = '''ANDROID_HOME ({}) does not appear to have valid Android SDK install
508                 (cannot find build-tools)'''
509        raise HostError(msg.format(env.android_home))
510    versions = os.listdir(build_tools_directory)
511    for version in reversed(sorted(versions)):
512        aapt_path = os.path.join(build_tools_directory, version, 'aapt')
513        if os.path.isfile(aapt_path):
514            logger.debug('Using aapt for version {}'.format(version))
515            env.aapt = aapt_path
516            break
517    else:
518        raise HostError('aapt not found. Please make sure at least one Android '
519                        'platform is installed.')
520
521
522def _check_env():
523    global android_home, platform_tools, adb, aapt  # pylint: disable=W0603
524    if not android_home:
525        android_home = os.getenv('ANDROID_HOME')
526        if android_home:
527            _env = _initialize_with_android_home(_AndroidEnvironment())
528        else:
529            _env = _initialize_without_android_home(_AndroidEnvironment())
530        android_home = _env.android_home
531        platform_tools = _env.platform_tools
532        adb = _env.adb
533        aapt = _env.aapt
534
535class LogcatMonitor(threading.Thread):
536    """
537    Helper class for monitoring Anroid's logcat
538
539    :param target: Android target to monitor
540    :type target: :class:`AndroidTarget`
541
542                    device. Logcat entries that don't match any will not be
543                    seen. If omitted, all entries will be sent to host.
544    :type regexps: list(str)
545    """
546
547    FLUSH_SIZE = 1000
548
549    @property
550    def logfile(self):
551        return self._logfile
552
553    def __init__(self, target, regexps=None):
554        super(LogcatMonitor, self).__init__()
555
556        self.target = target
557
558        self._stopped = threading.Event()
559        self._match_found = threading.Event()
560
561        self._sought = None
562        self._found = None
563
564        self._lines = Queue.Queue()
565        self._datalock = threading.Lock()
566        self._regexps = regexps
567
568    def start(self, outfile=None):
569        """
570        Start logcat and begin monitoring
571
572        :param outfile: Optional path to file to store all logcat entries
573        :type outfile: str
574        """
575        if outfile:
576            self._logfile = outfile
577        else:
578            fd, self._logfile = tempfile.mkstemp()
579            os.close(fd)
580        logger.debug('Logging to {}'.format(self._logfile))
581
582        super(LogcatMonitor, self).start()
583
584    def run(self):
585        self.target.clear_logcat()
586
587        logcat_cmd = 'logcat'
588
589        # Join all requested regexps with an 'or'
590        if self._regexps:
591            regexp = '{}'.format('|'.join(self._regexps))
592            if len(self._regexps) > 1:
593                regexp = '({})'.format(regexp)
594            logcat_cmd = '{} -e "{}"'.format(logcat_cmd, regexp)
595
596        logger.debug('logcat command ="{}"'.format(logcat_cmd))
597        self._logcat = self.target.background(logcat_cmd)
598
599        while not self._stopped.is_set():
600            line = self._logcat.stdout.readline(1024)
601            if line:
602                self._add_line(line)
603
604    def stop(self):
605        # Kill the underlying logcat process
606        # This will unblock self._logcat.stdout.readline()
607        host.kill_children(self._logcat.pid)
608        self._logcat.kill()
609
610        self._stopped.set()
611        self.join()
612
613        self._flush_lines()
614
615    def _add_line(self, line):
616        self._lines.put(line)
617
618        if self._sought and re.match(self._sought, line):
619            self._found = line
620            self._match_found.set()
621
622        if self._lines.qsize() >= self.FLUSH_SIZE:
623            self._flush_lines()
624
625    def _flush_lines(self):
626        with self._datalock:
627            with open(self._logfile, 'a') as fh:
628                while not self._lines.empty():
629                    fh.write(self._lines.get())
630
631    def clear_log(self):
632        with self._datalock:
633            while not self._lines.empty():
634                self._lines.get()
635
636            with open(self._logfile, 'w') as fh:
637                pass
638
639    def get_log(self):
640        """
641        Return the list of lines found by the monitor
642        """
643        self._flush_lines()
644
645        with self._datalock:
646            with open(self._logfile, 'r') as fh:
647                res = [line for line in fh]
648
649        return res
650
651    def search(self, regexp):
652        """
653        Search a line that matches a regexp in the logcat log
654        Return immediatly
655        """
656        res = []
657
658        self._flush_lines()
659
660        with self._datalock:
661            with open(self._logfile, 'r') as fh:
662                for line in fh:
663                    if re.match(regexp, line):
664                        res.append(line)
665
666        return res
667
668    def wait_for(self, regexp, timeout=30):
669        """
670        Search a line that matches a regexp in the logcat log
671        Wait for it to appear if it's not found
672
673        :param regexp: regexp to search
674        :type regexp: str
675
676        :param timeout: Timeout in seconds, before rasing RuntimeError.
677                        ``None`` means wait indefinitely
678        :type timeout: number
679
680        :returns: List of matched strings
681        """
682        res = self.search(regexp)
683
684        # Found some matches, return them
685        # Also return if thread not running
686        if len(res) > 0 or not self.is_alive():
687            return res
688
689        # Did not find any match, wait for one to pop up
690        self._sought = regexp
691        found = self._match_found.wait(timeout)
692        self._match_found.clear()
693        self._sought = None
694
695        if found:
696            return [self._found]
697        else:
698            raise RuntimeError('Logcat monitor timeout ({}s)'.format(timeout))
699