1#    Copyright 2013-2015 ARM Limited
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15
16
17"""
18Utility functions for working with Android devices through adb.
19
20"""
21# pylint: disable=E1103
22import os
23import pexpect
24import time
25import subprocess
26import logging
27import re
28import threading
29import tempfile
30import Queue
31from collections import defaultdict
32
33from devlib.exception import TargetError, HostError, DevlibError
34from devlib.utils.misc import check_output, which, memoized, ABI_MAP
35from devlib.utils.misc import escape_single_quotes, escape_double_quotes
36from devlib import host
37
38
39logger = logging.getLogger('android')
40
41MAX_ATTEMPTS = 5
42AM_START_ERROR = re.compile(r"Error: Activity.*")
43
44# See:
45# http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
46ANDROID_VERSION_MAP = {
47    23: 'MARSHMALLOW',
48    22: 'LOLLYPOP_MR1',
49    21: 'LOLLYPOP',
50    20: 'KITKAT_WATCH',
51    19: 'KITKAT',
52    18: 'JELLY_BEAN_MR2',
53    17: 'JELLY_BEAN_MR1',
54    16: 'JELLY_BEAN',
55    15: 'ICE_CREAM_SANDWICH_MR1',
56    14: 'ICE_CREAM_SANDWICH',
57    13: 'HONEYCOMB_MR2',
58    12: 'HONEYCOMB_MR1',
59    11: 'HONEYCOMB',
60    10: 'GINGERBREAD_MR1',
61    9: 'GINGERBREAD',
62    8: 'FROYO',
63    7: 'ECLAIR_MR1',
64    6: 'ECLAIR_0_1',
65    5: 'ECLAIR',
66    4: 'DONUT',
67    3: 'CUPCAKE',
68    2: 'BASE_1_1',
69    1: 'BASE',
70}
71
72
73# Initialized in functions near the botton of the file
74android_home = None
75platform_tools = None
76adb = None
77aapt = None
78fastboot = None
79
80
81class AndroidProperties(object):
82
83    def __init__(self, text):
84        self._properties = {}
85        self.parse(text)
86
87    def parse(self, text):
88        self._properties = dict(re.findall(r'\[(.*?)\]:\s+\[(.*?)\]', text))
89
90    def iteritems(self):
91        return self._properties.iteritems()
92
93    def __iter__(self):
94        return iter(self._properties)
95
96    def __getattr__(self, name):
97        return self._properties.get(name)
98
99    __getitem__ = __getattr__
100
101
102class AdbDevice(object):
103
104    def __init__(self, name, status):
105        self.name = name
106        self.status = status
107
108    def __cmp__(self, other):
109        if isinstance(other, AdbDevice):
110            return cmp(self.name, other.name)
111        else:
112            return cmp(self.name, other)
113
114    def __str__(self):
115        return 'AdbDevice({}, {})'.format(self.name, self.status)
116
117    __repr__ = __str__
118
119
120class ApkInfo(object):
121
122    version_regex = re.compile(r"name='(?P<name>[^']+)' versionCode='(?P<vcode>[^']+)' versionName='(?P<vname>[^']+)'")
123    name_regex = re.compile(r"name='(?P<name>[^']+)'")
124
125    def __init__(self, path=None):
126        self.path = path
127        self.package = None
128        self.activity = None
129        self.label = None
130        self.version_name = None
131        self.version_code = None
132        self.native_code = None
133        self.parse(path)
134
135    def parse(self, apk_path):
136        _check_env()
137        command = [aapt, 'dump', 'badging', apk_path]
138        logger.debug(' '.join(command))
139        try:
140            output = subprocess.check_output(command, stderr=subprocess.STDOUT)
141        except subprocess.CalledProcessError as e:
142            raise HostError('Error parsing APK file {}. `aapt` says:\n{}'
143                            .format(apk_path, e.output))
144        for line in output.split('\n'):
145            if line.startswith('application-label:'):
146                self.label = line.split(':')[1].strip().replace('\'', '')
147            elif line.startswith('package:'):
148                match = self.version_regex.search(line)
149                if match:
150                    self.package = match.group('name')
151                    self.version_code = match.group('vcode')
152                    self.version_name = match.group('vname')
153            elif line.startswith('launchable-activity:'):
154                match = self.name_regex.search(line)
155                self.activity = match.group('name')
156            elif line.startswith('native-code'):
157                apk_abis = [entry.strip() for entry in line.split(':')[1].split("'") if entry.strip()]
158                mapped_abis = []
159                for apk_abi in apk_abis:
160                    found = False
161                    for abi, architectures in ABI_MAP.iteritems():
162                        if apk_abi in architectures:
163                            mapped_abis.append(abi)
164                            found = True
165                            break
166                    if not found:
167                        mapped_abis.append(apk_abi)
168                self.native_code = mapped_abis
169            else:
170                pass  # not interested
171
172
173class AdbConnection(object):
174
175    # maintains the count of parallel active connections to a device, so that
176    # adb disconnect is not invoked untill all connections are closed
177    active_connections = defaultdict(int)
178    default_timeout = 10
179    ls_command = 'ls'
180
181    @property
182    def name(self):
183        return self.device
184
185    @property
186    @memoized
187    def newline_separator(self):
188        output = adb_command(self.device,
189                             "shell '({}); echo \"\n$?\"'".format(self.ls_command), adb_server=self.adb_server)
190        if output.endswith('\r\n'):
191            return '\r\n'
192        elif output.endswith('\n'):
193            return '\n'
194        else:
195            raise DevlibError("Unknown line ending")
196
197    # Again, we need to handle boards where the default output format from ls is
198    # single column *and* boards where the default output is multi-column.
199    # We need to do this purely because the '-1' option causes errors on older
200    # versions of the ls tool in Android pre-v7.
201    def _setup_ls(self):
202        command = "shell '(ls -1); echo \"\n$?\"'"
203        try:
204            output = adb_command(self.device, command, timeout=self.timeout, adb_server=self.adb_server)
205        except subprocess.CalledProcessError as e:
206            raise HostError(
207                'Failed to set up ls command on Android device. Output:\n'
208                + e.output)
209        lines = output.splitlines()
210        retval = lines[-1].strip()
211        if int(retval) == 0:
212            self.ls_command = 'ls -1'
213        else:
214            self.ls_command = 'ls'
215        logger.debug("ls command is set to {}".format(self.ls_command))
216
217    def __init__(self, device=None, timeout=None, platform=None, adb_server=None):
218        self.timeout = timeout if timeout is not None else self.default_timeout
219        if device is None:
220            device = adb_get_device(timeout=timeout, adb_server=adb_server)
221        self.device = device
222        self.adb_server = adb_server
223        adb_connect(self.device)
224        AdbConnection.active_connections[self.device] += 1
225        self._setup_ls()
226
227    def push(self, source, dest, timeout=None):
228        if timeout is None:
229            timeout = self.timeout
230        command = "push '{}' '{}'".format(source, dest)
231        if not os.path.exists(source):
232            raise HostError('No such file "{}"'.format(source))
233        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
234
235    def pull(self, source, dest, timeout=None):
236        if timeout is None:
237            timeout = self.timeout
238        # Pull all files matching a wildcard expression
239        if os.path.isdir(dest) and \
240           ('*' in source or '?' in source):
241            command = 'shell {} {}'.format(self.ls_command, source)
242            output = adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
243            for line in output.splitlines():
244                command = "pull '{}' '{}'".format(line.strip(), dest)
245                adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
246            return
247        command = "pull '{}' '{}'".format(source, dest)
248        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
249
250    def execute(self, command, timeout=None, check_exit_code=False,
251                as_root=False, strip_colors=True):
252        return adb_shell(self.device, command, timeout, check_exit_code,
253                         as_root, self.newline_separator,adb_server=self.adb_server)
254
255    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
256        return adb_background_shell(self.device, command, stdout, stderr, as_root)
257
258    def close(self):
259        AdbConnection.active_connections[self.device] -= 1
260        if AdbConnection.active_connections[self.device] <= 0:
261            adb_disconnect(self.device)
262            del AdbConnection.active_connections[self.device]
263
264    def cancel_running_command(self):
265        # adbd multiplexes commands so that they don't interfer with each
266        # other, so there is no need to explicitly cancel a running command
267        # before the next one can be issued.
268        pass
269
270
271def fastboot_command(command, timeout=None, device=None):
272    _check_env()
273    target = '-s {}'.format(device) if device else ''
274    full_command = 'fastboot {} {}'.format(target, command)
275    logger.debug(full_command)
276    output, _ = check_output(full_command, timeout, shell=True)
277    return output
278
279
280def fastboot_flash_partition(partition, path_to_image):
281    command = 'flash {} {}'.format(partition, path_to_image)
282    fastboot_command(command)
283
284
285def adb_get_device(timeout=None, adb_server=None):
286    """
287    Returns the serial number of a connected android device.
288
289    If there are more than one device connected to the machine, or it could not
290    find any device connected, :class:`devlib.exceptions.HostError` is raised.
291    """
292    # TODO this is a hacky way to issue a adb command to all listed devices
293
294    # Ensure server is started so the 'daemon started successfully' message
295    # doesn't confuse the parsing below
296    adb_command(None, 'start-server', adb_server=adb_server)
297
298    # The output of calling adb devices consists of a heading line then
299    # a list of the devices sperated by new line
300    # The last line is a blank new line. in otherwords, if there is a device found
301    # then the output length is 2 + (1 for each device)
302    start = time.time()
303    while True:
304        output = adb_command(None, "devices", adb_server=adb_server).splitlines()  # pylint: disable=E1103
305        output_length = len(output)
306        if output_length == 3:
307            # output[1] is the 2nd line in the output which has the device name
308            # Splitting the line by '\t' gives a list of two indexes, which has
309            # device serial in 0 number and device type in 1.
310            return output[1].split('\t')[0]
311        elif output_length > 3:
312            message = '{} Android devices found; either explicitly specify ' +\
313                      'the device you want, or make sure only one is connected.'
314            raise HostError(message.format(output_length - 2))
315        else:
316            if timeout < time.time() - start:
317                raise HostError('No device is connected and available')
318            time.sleep(1)
319
320
321def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
322    _check_env()
323    # Connect is required only for ADB-over-IP
324    if "." not in device:
325        logger.debug('Device connected via USB, connect not required')
326        return
327    tries = 0
328    output = None
329    while tries <= attempts:
330        tries += 1
331        if device:
332            command = 'adb connect {}'.format(device)
333            logger.debug(command)
334            output, _ = check_output(command, shell=True, timeout=timeout)
335        if _ping(device):
336            break
337        time.sleep(10)
338    else:  # did not connect to the device
339        message = 'Could not connect to {}'.format(device or 'a device')
340        if output:
341            message += '; got: "{}"'.format(output)
342        raise HostError(message)
343
344
345def adb_disconnect(device):
346    _check_env()
347    if not device:
348        return
349    if ":" in device and device in adb_list_devices():
350        command = "adb disconnect " + device
351        logger.debug(command)
352        retval = subprocess.call(command, stdout=open(os.devnull, 'wb'), shell=True)
353        if retval:
354            raise TargetError('"{}" returned {}'.format(command, retval))
355
356
357def _ping(device):
358    _check_env()
359    device_string = ' -s {}'.format(device) if device else ''
360    command = "adb{} shell \"ls / > /dev/null\"".format(device_string)
361    logger.debug(command)
362    result = subprocess.call(command, stderr=subprocess.PIPE, shell=True)
363    if not result:
364        return True
365    else:
366        return False
367
368
369def adb_shell(device, command, timeout=None, check_exit_code=False,
370              as_root=False, newline_separator='\r\n', adb_server=None):  # NOQA
371    _check_env()
372    if as_root:
373        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
374    device_part = []
375    if adb_server:
376        device_part = ['-H', adb_server]
377    device_part += ['-s', device] if device else []
378
379    # On older combinations of ADB/Android versions, the adb host command always
380    # exits with 0 if it was able to run the command on the target, even if the
381    # command failed (https://code.google.com/p/android/issues/detail?id=3254).
382    # Homogenise this behaviour by running the command then echoing the exit
383    # code.
384    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
385    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
386    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
387    raw_output, error = check_output(actual_command, timeout, shell=False)
388    if raw_output:
389        try:
390            output, exit_code, _ = raw_output.rsplit(newline_separator, 2)
391        except ValueError:
392            exit_code, _ = raw_output.rsplit(newline_separator, 1)
393            output = ''
394    else:  # raw_output is empty
395        exit_code = '969696'  # just because
396        output = ''
397
398    if check_exit_code:
399        exit_code = exit_code.strip()
400        re_search = AM_START_ERROR.findall('{}\n{}'.format(output, error))
401        if exit_code.isdigit():
402            if int(exit_code):
403                message = ('Got exit code {}\nfrom target command: {}\n'
404                           'STDOUT: {}\nSTDERR: {}')
405                raise TargetError(message.format(exit_code, command, output, error))
406            elif re_search:
407                message = 'Could not start activity; got the following:\n{}'
408                raise TargetError(message.format(re_search[0]))
409        else:  # not all digits
410            if re_search:
411                message = 'Could not start activity; got the following:\n{}'
412                raise TargetError(message.format(re_search[0]))
413            else:
414                message = 'adb has returned early; did not get an exit code. '\
415                          'Was kill-server invoked?\nOUTPUT:\n-----\n{}\n'\
416                          '-----\nERROR:\n-----\n{}\n-----'
417                raise TargetError(message.format(raw_output, error))
418
419    return output
420
421
422def adb_background_shell(device, command,
423                         stdout=subprocess.PIPE,
424                         stderr=subprocess.PIPE,
425                         as_root=False):
426    """Runs the sepcified command in a subprocess, returning the the Popen object."""
427    _check_env()
428    if as_root:
429        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
430    device_string = ' -s {}'.format(device) if device else ''
431    full_command = 'adb{} shell "{}"'.format(device_string, escape_double_quotes(command))
432    logger.debug(full_command)
433    return subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
434
435
436def adb_list_devices(adb_server=None):
437    output = adb_command(None, 'devices',adb_server=adb_server)
438    devices = []
439    for line in output.splitlines():
440        parts = [p.strip() for p in line.split()]
441        if len(parts) == 2:
442            devices.append(AdbDevice(*parts))
443    return devices
444
445
446def get_adb_command(device, command, timeout=None,adb_server=None):
447    _check_env()
448    device_string = ""
449    if adb_server != None:
450        device_string = ' -H {}'.format(adb_server)
451    device_string += ' -s {}'.format(device) if device else ''
452    return "adb{} {}".format(device_string, command)
453
454def adb_command(device, command, timeout=None,adb_server=None):
455    full_command = get_adb_command(device, command, timeout, adb_server)
456    logger.debug(full_command)
457    output, _ = check_output(full_command, timeout, shell=True)
458    return output
459
460def grant_app_permissions(target, package):
461    """
462    Grant an app all the permissions it may ask for
463    """
464    dumpsys = target.execute('dumpsys package {}'.format(package))
465
466    permissions = re.search(
467        'requested permissions:\s*(?P<permissions>(android.permission.+\s*)+)', dumpsys
468    )
469    if permissions is None:
470        return
471    permissions = permissions.group('permissions').replace(" ", "").splitlines()
472
473    for permission in permissions:
474        try:
475            target.execute('pm grant {} {}'.format(package, permission))
476        except TargetError:
477            logger.debug('Cannot grant {}'.format(permission))
478
479
480# Messy environment initialisation stuff...
481
482class _AndroidEnvironment(object):
483
484    def __init__(self):
485        self.android_home = None
486        self.platform_tools = None
487        self.adb = None
488        self.aapt = None
489        self.fastboot = None
490
491
492def _initialize_with_android_home(env):
493    logger.debug('Using ANDROID_HOME from the environment.')
494    env.android_home = android_home
495    env.platform_tools = os.path.join(android_home, 'platform-tools')
496    os.environ['PATH'] = env.platform_tools + os.pathsep + os.environ['PATH']
497    _init_common(env)
498    return env
499
500
501def _initialize_without_android_home(env):
502    adb_full_path = which('adb')
503    if adb_full_path:
504        env.adb = 'adb'
505    else:
506        raise HostError('ANDROID_HOME is not set and adb is not in PATH. '
507                        'Have you installed Android SDK?')
508    logger.debug('Discovering ANDROID_HOME from adb path.')
509    env.platform_tools = os.path.dirname(adb_full_path)
510    env.android_home = os.path.dirname(env.platform_tools)
511    try:
512        _init_common(env)
513    except:
514        env.aapt = which('aapt')
515        if env.aapt:
516            logger.info('Using aapt: ' + env.aapt)
517        else:
518            raise RuntimeError('aapt not found, try setting ANDROID_HOME to \
519                                Android SDK or run LISA from android environment')
520    return env
521
522
523def _init_common(env):
524    logger.debug('ANDROID_HOME: {}'.format(env.android_home))
525    build_tools_directory = os.path.join(env.android_home, 'build-tools')
526    if not os.path.isdir(build_tools_directory):
527        msg = '''ANDROID_HOME ({}) does not appear to have valid Android SDK install
528                 (cannot find build-tools)'''
529        raise HostError(msg.format(env.android_home))
530    versions = os.listdir(build_tools_directory)
531    for version in reversed(sorted(versions)):
532        aapt_path = os.path.join(build_tools_directory, version, 'aapt')
533        if os.path.isfile(aapt_path):
534            logger.debug('Using aapt for version {}'.format(version))
535            env.aapt = aapt_path
536            break
537    else:
538        raise HostError('aapt not found. Please make sure at least one Android '
539                        'platform is installed.')
540
541
542def _check_env():
543    global android_home, platform_tools, adb, aapt  # pylint: disable=W0603
544    if not android_home:
545        android_home = os.getenv('ANDROID_HOME')
546        if android_home:
547            _env = _initialize_with_android_home(_AndroidEnvironment())
548        else:
549            _env = _initialize_without_android_home(_AndroidEnvironment())
550        android_home = _env.android_home
551        platform_tools = _env.platform_tools
552        adb = _env.adb
553        aapt = _env.aapt
554
555class LogcatMonitor(object):
556    """
557    Helper class for monitoring Anroid's logcat
558
559    :param target: Android target to monitor
560    :type target: :class:`AndroidTarget`
561
562    :param regexps: List of uncompiled regular expressions to filter on the
563                    device. Logcat entries that don't match any will not be
564                    seen. If omitted, all entries will be sent to host.
565    :type regexps: list(str)
566    """
567
568    @property
569    def logfile(self):
570        return self._logfile
571
572    def __init__(self, target, regexps=None):
573        super(LogcatMonitor, self).__init__()
574
575        self.target = target
576        self._regexps = regexps
577
578    def start(self, outfile=None):
579        """
580        Start logcat and begin monitoring
581
582        :param outfile: Optional path to file to store all logcat entries
583        :type outfile: str
584        """
585        if outfile:
586            self._logfile = open(outfile, 'w')
587        else:
588            self._logfile = tempfile.NamedTemporaryFile()
589
590        self.target.clear_logcat()
591
592        logcat_cmd = 'logcat'
593
594        # Join all requested regexps with an 'or'
595        if self._regexps:
596            regexp = '{}'.format('|'.join(self._regexps))
597            if len(self._regexps) > 1:
598                regexp = '({})'.format(regexp)
599            logcat_cmd = '{} -e "{}"'.format(logcat_cmd, regexp)
600
601        logcat_cmd = get_adb_command(self.target.conn.device, logcat_cmd)
602
603        logger.debug('logcat command ="{}"'.format(logcat_cmd))
604        self._logcat = pexpect.spawn(logcat_cmd, logfile=self._logfile)
605
606    def stop(self):
607        self._logcat.terminate()
608        self._logfile.close()
609
610    def get_log(self):
611        """
612        Return the list of lines found by the monitor
613        """
614        with open(self._logfile.name) as fh:
615            return [line for line in fh]
616
617    def clear_log(self):
618        with open(self._logfile.name, 'w') as fh:
619            pass
620
621    def search(self, regexp):
622        """
623        Search a line that matches a regexp in the logcat log
624        Return immediatly
625        """
626        return [line for line in self.get_log() if re.match(regexp, line)]
627
628    def wait_for(self, regexp, timeout=30):
629        """
630        Search a line that matches a regexp in the logcat log
631        Wait for it to appear if it's not found
632
633        :param regexp: regexp to search
634        :type regexp: str
635
636        :param timeout: Timeout in seconds, before rasing RuntimeError.
637                        ``None`` means wait indefinitely
638        :type timeout: number
639
640        :returns: List of matched strings
641        """
642        log = self.get_log()
643        res = [line for line in log if re.match(regexp, line)]
644
645        # Found some matches, return them
646        if len(res) > 0:
647            return res
648
649        # Store the number of lines we've searched already, so we don't have to
650        # re-grep them after 'expect' returns
651        next_line_num = len(log)
652
653        try:
654            self._logcat.expect(regexp, timeout=timeout)
655        except pexpect.TIMEOUT:
656            raise RuntimeError('Logcat monitor timeout ({}s)'.format(timeout))
657
658        return [line for line in self.get_log()[next_line_num:]
659                if re.match(regexp, line)]
660