autoupdater.py revision 458bf9ef5cf16bf88b306844a54d073c9cde5856 
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import httplib
6import logging
7import multiprocessing
8import os
9import re
10import urlparse
11
12from autotest_lib.client.common_lib import error, global_config
13
14# Local stateful update path is relative to the CrOS source directory.
15LOCAL_STATEFUL_UPDATE_PATH = 'src/platform/dev/stateful_update'
16LOCAL_CHROOT_STATEFUL_UPDATE_PATH = '/usr/bin/stateful_update'
17REMOTE_STATEUL_UPDATE_PATH = '/usr/local/bin/stateful_update'
18STATEFUL_UPDATE = '/tmp/stateful_update'
19UPDATER_BIN = '/usr/bin/update_engine_client'
20UPDATER_IDLE = 'UPDATE_STATUS_IDLE'
21UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT'
22UPDATED_MARKER = '/var/run/update_engine_autoupdate_completed'
23UPDATER_LOGS = '/var/log/messages /var/log/update_engine'
24
25
26class ChromiumOSError(error.InstallError):
27    """Generic error for ChromiumOS-specific exceptions."""
28    pass
29
30
31class RootFSUpdateError(ChromiumOSError):
32    """Raised when the RootFS fails to update."""
33    pass
34
35
36class StatefulUpdateError(ChromiumOSError):
37    """Raised when the stateful partition fails to update."""
38    pass
39
40
41def url_to_version(update_url):
42    """Return the version based on update_url.
43
44    @param update_url: url to the image to update to.
45
46    """
47    # The Chrome OS version is generally the last element in the URL. The only
48    # exception is delta update URLs, which are rooted under the version; e.g.,
49    # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to
50    # strip off the au section of the path before reading the version.
51    return re.sub('/au/.*', '',
52                  urlparse.urlparse(update_url).path).split('/')[-1].strip()
53
54
55def url_to_image_name(update_url):
56    """Return the image name based on update_url.
57
58    From a URL like:
59        http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
60    return lumpy-release/R27-3837.0.0
61
62    @param update_url: url to the image to update to.
63    @returns a string representing the image name in the update_url.
64
65    """
66    return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
67
68
69class ChromiumOSUpdater():
70    """Helper class used to update DUT with image of desired version."""
71    KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3}
72    KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5}
73
74
75    def __init__(self, update_url, host=None, local_devserver=False):
76        self.host = host
77        self.update_url = update_url
78        self._update_error_queue = multiprocessing.Queue(2)
79        self.local_devserver = local_devserver
80        if not local_devserver:
81          self.update_version = url_to_version(update_url)
82        else:
83          self.update_version = None
84
85    def check_update_status(self):
86        """Return current status from update-engine."""
87        update_status = self._run(
88            '%s -status 2>&1 | grep CURRENT_OP' % UPDATER_BIN)
89        return update_status.stdout.strip().split('=')[-1]
90
91
92    def reset_update_engine(self):
93        """Restarts the update-engine service."""
94        self._run('rm -f %s' % UPDATED_MARKER)
95        try:
96            self._run('initctl stop update-engine')
97        except error.AutoservRunError:
98            logging.warn('Stopping update-engine service failed. Already dead?')
99        self._run('initctl start update-engine')
100
101        if self.check_update_status() != UPDATER_IDLE:
102            raise ChromiumOSError('%s is not in an installable state' %
103                                  self.host.hostname)
104
105
106    def _run(self, cmd, *args, **kwargs):
107        """Abbreviated form of self.host.run(...)"""
108        return self.host.run(cmd, *args, **kwargs)
109
110
111    def rootdev(self, options=''):
112        """Returns the stripped output of rootdev <options>.
113
114        @param options: options to run rootdev.
115
116        """
117        return self._run('rootdev %s' % options).stdout.strip()
118
119
120    def get_kernel_state(self):
121        """Returns the (<active>, <inactive>) kernel state as a pair."""
122        active_root = int(re.findall('\d+\Z', self.rootdev('-s'))[0])
123        if active_root == self.KERNEL_A['root']:
124            return self.KERNEL_A, self.KERNEL_B
125        elif active_root == self.KERNEL_B['root']:
126            return self.KERNEL_B, self.KERNEL_A
127        else:
128            raise ChromiumOSError('Encountered unknown root partition: %s' %
129                                  active_root)
130
131
132    def _cgpt(self, flag, kernel, dev='$(rootdev -s -d)'):
133        """Return numeric cgpt value for the specified flag, kernel, device. """
134        return int(self._run('cgpt show -n -i %d %s %s' % (
135            kernel['kernel'], flag, dev)).stdout.strip())
136
137
138    def get_kernel_priority(self, kernel):
139        """Return numeric priority for the specified kernel.
140
141        @param kernel: information of the given kernel, KERNEL_A or KERNEL_B.
142
143        """
144        return self._cgpt('-P', kernel)
145
146
147    def get_kernel_success(self, kernel):
148        """Return boolean success flag for the specified kernel.
149
150        @param kernel: information of the given kernel, KERNEL_A or KERNEL_B.
151
152        """
153        return self._cgpt('-S', kernel) != 0
154
155
156    def get_kernel_tries(self, kernel):
157        """Return tries count for the specified kernel.
158
159        @param kernel: information of the given kernel, KERNEL_A or KERNEL_B.
160
161        """
162        return self._cgpt('-T', kernel)
163
164
165    def get_stateful_update_script(self):
166        """Returns the path to the stateful update script on the target."""
167        # We attempt to load the local stateful update path in 3 different
168        # ways. First we use the location specified in the autotest global
169        # config. If this doesn't exist, we attempt to use the Chromium OS
170        # Chroot path to the installed script. If all else fails, we use the
171        # stateful update script on the host.
172        stateful_update_path = os.path.join(
173                global_config.global_config.get_config_value(
174                        'CROS', 'source_tree', default=''),
175                LOCAL_STATEFUL_UPDATE_PATH)
176
177        if not os.path.exists(stateful_update_path):
178            logging.warn('Could not find Chrome OS source location for '
179                         'stateful_update script at %s, falling back to chroot '
180                         'copy.', stateful_update_path)
181            stateful_update_path = LOCAL_CHROOT_STATEFUL_UPDATE_PATH
182
183        if not os.path.exists(stateful_update_path):
184            logging.warn('Could not chroot stateful_update script, falling '
185                         'back on client copy.')
186            statefuldev_script = REMOTE_STATEUL_UPDATE_PATH
187        else:
188            self.host.send_file(
189                    stateful_update_path, STATEFUL_UPDATE, delete_dest=True)
190            statefuldev_script = STATEFUL_UPDATE
191
192        return statefuldev_script
193
194
195    def reset_stateful_partition(self):
196        """Clear any pending stateful update request."""
197        statefuldev_cmd = [self.get_stateful_update_script()]
198        statefuldev_cmd += ['--stateful_change=reset', '2>&1']
199        # This shouldn't take any time at all.
200        self._run(' '.join(statefuldev_cmd), timeout=10)
201
202
203    def revert_boot_partition(self):
204        """Revert the boot partition."""
205        part = self.rootdev('-s')
206        logging.warn('Reverting update; Boot partition will be %s', part)
207        return self._run('/postinst %s 2>&1' % part)
208
209
210    def trigger_update(self):
211        """Triggers a background update on a test image.
212
213        @raise RootFSUpdateError if anything went wrong.
214
215        """
216        autoupdate_cmd = '%s --check_for_update --omaha_url=%s' % (
217            UPDATER_BIN, self.update_url)
218        logging.info('triggering update via: %s', autoupdate_cmd)
219        try:
220            # This should return immediately, hence the short timeout.
221            self._run(autoupdate_cmd, timeout=10)
222        except error.AutoservRunError, e:
223            raise RootFSUpdateError('update triggering failed on %s: %s' %
224                                    (self.host.hostname, str(e)))
225
226
227    def _update_root(self):
228        logging.info('Updating root partition...')
229
230        # Run update_engine using the specified URL.
231        try:
232            autoupdate_cmd = '%s --update --omaha_url=%s 2>&1' % (
233                UPDATER_BIN, self.update_url)
234            self._run(autoupdate_cmd, timeout=900)
235        except error.AutoservRunError:
236            update_error = RootFSUpdateError('update-engine failed on %s' %
237                                             self.host.hostname)
238            self._update_error_queue.put(update_error)
239            raise update_error
240
241        # Check that the installer completed as expected.
242        status = self.check_update_status()
243        if status != UPDATER_NEED_REBOOT:
244            update_error = RootFSUpdateError('update-engine error on %s: %s' %
245                                             (self.host.hostname, status))
246            self._update_error_queue.put(update_error)
247            raise update_error
248
249
250    def update_stateful(self, clobber=True):
251        """Updates the stateful partition.
252
253        @param clobber: If True, a clean stateful installation.
254        """
255        logging.info('Updating stateful partition...')
256        # For production devservers we create a static tree of payloads rooted
257        # at archive.
258        if not self.local_devserver:
259          statefuldev_url = self.update_url.replace('update',
260                                                    'static/archive')
261        else:
262          statefuldev_url = self.update_url.replace('update',
263                                                    'static')
264
265        # Attempt stateful partition update; this must succeed so that the newly
266        # installed host is testable after update.
267        statefuldev_cmd = [self.get_stateful_update_script(), statefuldev_url]
268        if clobber:
269            statefuldev_cmd.append('--stateful_change=clean')
270
271        statefuldev_cmd.append('2>&1')
272        try:
273            self._run(' '.join(statefuldev_cmd), timeout=600)
274        except error.AutoservRunError:
275            update_error = StatefulUpdateError('stateful_update failed on %s' %
276                                               self.host.hostname)
277            self._update_error_queue.put(update_error)
278            raise update_error
279
280
281    def run_update(self, force_update, update_root=True):
282        """Update the DUT with image of specific version.
283
284        @param force_update: True to update DUT even if it's running the same
285            version already.
286        @param update_root: True to force a kernel update. If it's False and
287            force_update is True, stateful update will be used to clean up
288            the DUT.
289
290        """
291        booted_version = self.get_build_id()
292        if (self.check_version() and not force_update):
293            logging.info('System is already up to date. Skipping update.')
294            return False
295
296        if self.update_version:
297            logging.info('Updating from version %s to %s.',
298                         booted_version, self.update_version)
299
300        # Check that Dev Server is accepting connections (from autoserv's host).
301        # If we can't talk to it, the machine host probably can't either.
302        auserver_host = urlparse.urlparse(self.update_url)[1]
303        try:
304            httplib.HTTPConnection(auserver_host).connect()
305        except IOError:
306            raise ChromiumOSError(
307                'Update server at %s not available' % auserver_host)
308
309        logging.info('Installing from %s to %s', self.update_url,
310                     self.host.hostname)
311
312        # Reset update state.
313        self.reset_update_engine()
314        self.reset_stateful_partition()
315
316        try:
317            updaters = [
318                multiprocessing.process.Process(target=self._update_root),
319                multiprocessing.process.Process(target=self.update_stateful)
320                ]
321            if not update_root:
322                logging.info('Root update is skipped.')
323                updaters = updaters[1:]
324
325            # Run the updaters in parallel.
326            for updater in updaters: updater.start()
327            for updater in updaters: updater.join()
328
329            # Re-raise the first error that occurred.
330            if not self._update_error_queue.empty():
331                update_error = self._update_error_queue.get()
332                self.revert_boot_partition()
333                self.reset_stateful_partition()
334                raise update_error
335
336            logging.info('Update complete.')
337            return True
338        except:
339            # Collect update engine logs in the event of failure.
340            if self.host.job:
341                logging.info('Collecting update engine logs...')
342                self.host.get_file(
343                    UPDATER_LOGS, self.host.job.sysinfo.sysinfodir,
344                    preserve_perm=False)
345            raise
346
347
348    def check_version(self):
349        """Check the image running in DUT has the desired version.
350
351        @returns: True if the DUT's image version matches the version that
352            the autoupdater tries to update to.
353
354        """
355        booted_version = self.get_build_id()
356        return (self.update_version and
357                self.update_version.endswith(booted_version))
358
359
360    def check_version_to_confirm_install(self):
361        """Check image running in DUT has the desired version to be installed.
362
363        The method should not be used to check if DUT needs to have a full
364        reimage. Only use it to confirm a image is installed.
365
366        The method is designed to verify version for following 4 scenarios with
367        samples of version to update to and expected booted version:
368        1. trybot paladin build.
369        update version: trybot-lumpy-paladin/R27-3837.0.0-b123
370        booted version: 3837.0.2013_03_21_1340
371
372        2. trybot release build.
373        update version: trybot-lumpy-release/R27-3837.0.0-b456
374        booted version: 3837.0.0
375
376        3. buildbot official release build.
377        update version: lumpy-release/R27-3837.0.0
378        booted version: 3837.0.0
379
380        4. non-official paladin rc build.
381        update version: lumpy-paladin/R27-3878.0.0-rc7
382        booted version: 3837.0.0-rc7
383
384        5. chrome-perf build.
385        update version: lumpy-chrome-perf/R28-3837.0.0-b2996
386        booted version: 3837.0.0
387
388        6. pgo-generate build.
389        update version: lumpy-release-pgo-generate/R28-3837.0.0-b2996
390        booted version: 3837.0.0-pgo-generate
391
392        When we are checking if a DUT needs to do a full install, we should NOT
393        use this method to check if the DUT is running the same version, since
394        it may return false positive for a DUT running trybot paladin build to
395        be updated to another trybot paladin build.
396
397        TODO: This logic has a bug if a trybot paladin build failed to be
398        installed in a DUT running an older trybot paladin build with same
399        platform number, but different build number (-b###). So to conclusively
400        determine if a tryjob paladin build is imaged successfully, we may need
401        to find out the date string from update url.
402
403        @returns: True if the DUT's image version (without the date string if
404            the image is a trybot build), matches the version that the
405            autoupdater is trying to update to.
406
407        """
408        # In the local_devserver case, we can't know the expected
409        # build, so just pass.
410        if not self.update_version:
411            return True
412
413        # Always try the default check_version method first, this prevents
414        # any backward compatibility issue.
415        if self.check_version():
416            return True
417
418        # Remove R#- and -b# at the end of build version
419        stripped_version = re.sub(r'(R\d+-|-b\d+)', '', self.update_version)
420
421        booted_version = self.get_build_id()
422
423        is_trybot_paladin_build = re.match(r'.+trybot-.+-paladin',
424                                           self.update_url)
425
426        # Replace date string with 0 in booted_version
427        booted_version_no_date = re.sub(r'\d{4}_\d{2}_\d{2}_\d+', '0',
428                                        booted_version)
429        has_date_string = booted_version != booted_version_no_date
430
431        is_pgo_generate_build = re.match(r'.+-pgo-generate',
432                                           self.update_url)
433
434        # Remove |-pgo-generate| in booted_version
435        booted_version_no_pgo = booted_version.replace('-pgo-generate', '')
436        has_pgo_generate = booted_version != booted_version_no_pgo
437
438        if is_trybot_paladin_build:
439            if not has_date_string:
440                logging.error('A trybot paladin build is expected. Version ' +
441                              '"%s" is not a paladin build.', booted_version)
442                return False
443            return stripped_version == booted_version_no_date
444        elif is_pgo_generate_build:
445            if not has_pgo_generate:
446                logging.error('A pgo-generate build is expected. Version ' +
447                              '"%s" is not a pgo-generate build.',
448                              booted_version)
449                return False
450            return stripped_version == booted_version_no_pgo
451        else:
452            if has_date_string:
453                logging.error('Unexpected date found in a non trybot paladin' +
454                              ' build.')
455                return False
456            # Versioned build, i.e., rc or release build.
457            return stripped_version == booted_version
458
459
460    def get_build_id(self):
461        """Pulls the CHROMEOS_RELEASE_VERSION string from /etc/lsb-release."""
462        return self._run('grep CHROMEOS_RELEASE_VERSION'
463                         ' /etc/lsb-release').stdout.split('=')[1].strip()
464