autoupdate_EndToEndTest.py revision 1192c41145b533f4138434bfa7e6ba2af9f75330
10ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 20ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold# Use of this source code is governed by a BSD-style license that can be 30ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold# found in the LICENSE file. 40ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 52f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosaimport collections 60ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldimport json 70ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldimport logging 86c55bdb98e967675456a71a0971b81058536cac8Chris Sosaimport os 9a0261611f9be78ee9d456fd50bea81e9eda07c17Gilad Arnoldimport socket 1015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnoldimport time 110ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldimport urllib2 120ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldimport urlparse 1303901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold 146c55bdb98e967675456a71a0971b81058536cac8Chris Sosafrom autotest_lib.client.bin import utils as client_utils 156c55bdb98e967675456a71a0971b81058536cac8Chris Sosafrom autotest_lib.client.common_lib import error, global_config 160ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldfrom autotest_lib.client.common_lib.cros import autoupdater, dev_server 17368abdf47d544aca4adbaf19d112e3457e056d3dPrathmesh Prabhufrom autotest_lib.server import autotest, hosts, test 182f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosafrom autotest_lib.server.cros.dynamic_suite import tools 190ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 200ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 210ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnolddef _wait(secs, desc=None): 220ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Emits a log message and sleeps for a given number of seconds.""" 230338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold msg = 'Waiting %s seconds' % secs 240ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold if desc: 250ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold msg += ' (%s)' % desc 260ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold logging.info(msg) 270ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold time.sleep(secs) 280ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 290ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 30ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosaclass ExpectedUpdateEventChainFailed(error.TestFail): 31ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa """Raised if we fail to receive an expected event in a chain.""" 32ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 33ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 340338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold# Update event types. 350338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_TYPE_DOWNLOAD_COMPLETE = '1' 360338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_TYPE_INSTALL_COMPLETE = '2' 370338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_TYPE_UPDATE_COMPLETE = '3' 380338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_TYPE_DOWNLOAD_STARTED = '13' 390338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_TYPE_DOWNLOAD_FINISHED = '14' 40fac9a5d238337da91939328386c2fa4fdfb6d957Alex DeymoEVENT_TYPE_REBOOTED_AFTER_UPDATE = '54' 410338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 420338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold# Update event results. 430338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_RESULT_ERROR = '0' 440338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_RESULT_SUCCESS = '1' 450338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_RESULT_SUCCESS_REBOOT = '2' 460338ff3b754d52b41ca2d0432164a757daa1112dGilad ArnoldEVENT_RESULT_UPDATE_DEFERRED = '9' 470338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 48a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold# Omaha event types/results, from update_engine/omaha_request_action.h 49a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold# These are stored in dict form in order to easily print out the keys. 50a0ca5707ed10a6575ed290f341294331455f7769Gilad ArnoldEVENT_TYPE_DICT = { 51a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_TYPE_DOWNLOAD_COMPLETE: 'download_complete', 52a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_TYPE_INSTALL_COMPLETE: 'install_complete', 53a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_TYPE_UPDATE_COMPLETE: 'update_complete', 54a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_TYPE_DOWNLOAD_STARTED: 'download_started', 55fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo EVENT_TYPE_DOWNLOAD_FINISHED: 'download_finished', 56fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo EVENT_TYPE_REBOOTED_AFTER_UPDATE: 'rebooted_after_update' 57a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold} 58a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 59a0ca5707ed10a6575ed290f341294331455f7769Gilad ArnoldEVENT_RESULT_DICT = { 60a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_RESULT_ERROR: 'error', 61a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_RESULT_SUCCESS: 'success', 62a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_RESULT_SUCCESS_REBOOT: 'success_reboot', 63a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold EVENT_RESULT_UPDATE_DEFERRED: 'update_deferred' 64a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold} 65a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 660338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 67fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnolddef snippet(text): 68fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold """Returns the text with start/end snip markers around it. 69fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold 70fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold @param text: The snippet text. 71fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold 72fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold @return The text with start/end snip markers around it. 73fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold """ 74fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold snip = '---8<---' * 10 75fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold start = '-- START -' 76fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold end = '-- END -' 77fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold return ('%s%s\n%s\n%s%s' % 78fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold (start, snip[len(start):], text, end, snip[len(end):])) 79fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold 80fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold 810ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldclass ExpectedUpdateEvent(object): 82248108c1c925791d926105f42212b1033213a4dcGilad Arnold """Defines an expected event in an update process.""" 8345f02ae47134953169805d281992c9edf0019250Chris Sosa 840338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold _ATTR_NAME_DICT_MAP = { 85a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'event_type': EVENT_TYPE_DICT, 86a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'event_result': EVENT_RESULT_DICT, 870338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold } 880338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 89a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold _VALID_TYPES = set(EVENT_TYPE_DICT.keys()) 90a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold _VALID_RESULTS = set(EVENT_RESULT_DICT.keys()) 9145f02ae47134953169805d281992c9edf0019250Chris Sosa 920ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def __init__(self, event_type=None, event_result=None, version=None, 93248108c1c925791d926105f42212b1033213a4dcGilad Arnold previous_version=None, on_error=None): 94248108c1c925791d926105f42212b1033213a4dcGilad Arnold """Initializes an event expectation. 95248108c1c925791d926105f42212b1033213a4dcGilad Arnold 96248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param event_type: Expected event type. 97248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param event_result: Expected event result code. 98248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param version: Expected reported image version. 99248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param previous_version: Expected reported previous image version. 100248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param on_error: This is either an object to be returned when a received 101248108c1c925791d926105f42212b1033213a4dcGilad Arnold event mismatches the expectation, or a callable used 102248108c1c925791d926105f42212b1033213a4dcGilad Arnold for generating one. In the latter case, takes as 103248108c1c925791d926105f42212b1033213a4dcGilad Arnold input two attribute dictionaries (expected and actual) 104248108c1c925791d926105f42212b1033213a4dcGilad Arnold and an iterable of mismatched keys. If None, a generic 105248108c1c925791d926105f42212b1033213a4dcGilad Arnold message is returned. 106248108c1c925791d926105f42212b1033213a4dcGilad Arnold """ 1070338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold if event_type and event_type not in self._VALID_TYPES: 10845f02ae47134953169805d281992c9edf0019250Chris Sosa raise ValueError('event_type %s is not valid.' % event_type) 10945f02ae47134953169805d281992c9edf0019250Chris Sosa 1100338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold if event_result and event_result not in self._VALID_RESULTS: 11145f02ae47134953169805d281992c9edf0019250Chris Sosa raise ValueError('event_result %s is not valid.' % event_result) 11245f02ae47134953169805d281992c9edf0019250Chris Sosa 1130ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold self._expected_attrs = { 1140ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 'event_type': event_type, 1150ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 'event_result': event_result, 1160ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 'version': version, 1170ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 'previous_version': previous_version, 1180ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold } 119248108c1c925791d926105f42212b1033213a4dcGilad Arnold self._on_error = on_error 1200ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1210ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1220338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold @staticmethod 1230338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold def _attr_val_str(attr_val, helper_dict, default=None): 1240338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold """Returns an enriched attribute value string, or default.""" 1250338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold if not attr_val: 1260338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold return default 1270338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1280338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold s = str(attr_val) 1290338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold if helper_dict: 1300338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold s += ':%s' % helper_dict.get(attr_val, 'unknown') 1310338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1320338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold return s 1330338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1340338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1350338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold def _attr_name_and_values(self, attr_name, expected_attr_val, 1360338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold actual_attr_val=None): 1370338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold """Returns an attribute name, expected and actual value strings. 1380338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1390338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold This will return (name, expected, actual); the returned value for 1400338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold actual will be None if its respective input is None/empty. 1410338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1420338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold """ 1430338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold helper_dict = self._ATTR_NAME_DICT_MAP.get(attr_name) 1440338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold expected_attr_val_str = self._attr_val_str(expected_attr_val, 1450338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold helper_dict, 1460338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold default='any') 1470338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold actual_attr_val_str = self._attr_val_str(actual_attr_val, helper_dict) 1480338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1490338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold return attr_name, expected_attr_val_str, actual_attr_val_str 1500338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1510338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 152248108c1c925791d926105f42212b1033213a4dcGilad Arnold def _attrs_to_str(self, attrs_dict): 1530338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold return ' '.join(['%s=%s' % 1540338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold self._attr_name_and_values(attr_name, attr_val)[0:2] 155248108c1c925791d926105f42212b1033213a4dcGilad Arnold for attr_name, attr_val in attrs_dict.iteritems()]) 156248108c1c925791d926105f42212b1033213a4dcGilad Arnold 157248108c1c925791d926105f42212b1033213a4dcGilad Arnold 158248108c1c925791d926105f42212b1033213a4dcGilad Arnold def __str__(self): 159248108c1c925791d926105f42212b1033213a4dcGilad Arnold return self._attrs_to_str(self._expected_attrs) 1600ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1610ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1620ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def verify(self, actual_event): 1630ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Verify the attributes of an actual event. 1640ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 165ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa @param actual_event: a dictionary containing event attributes 1660ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 167248108c1c925791d926105f42212b1033213a4dcGilad Arnold @return An error message, or None if all attributes as expected. 1680ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1690ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 170248108c1c925791d926105f42212b1033213a4dcGilad Arnold mismatched_attrs = [ 171248108c1c925791d926105f42212b1033213a4dcGilad Arnold attr_name for attr_name, expected_attr_val 172248108c1c925791d926105f42212b1033213a4dcGilad Arnold in self._expected_attrs.iteritems() 173248108c1c925791d926105f42212b1033213a4dcGilad Arnold if (expected_attr_val and 174248108c1c925791d926105f42212b1033213a4dcGilad Arnold not self._verify_attr(attr_name, expected_attr_val, 175248108c1c925791d926105f42212b1033213a4dcGilad Arnold actual_event.get(attr_name)))] 176248108c1c925791d926105f42212b1033213a4dcGilad Arnold if not mismatched_attrs: 177248108c1c925791d926105f42212b1033213a4dcGilad Arnold return None 178248108c1c925791d926105f42212b1033213a4dcGilad Arnold if callable(self._on_error): 179248108c1c925791d926105f42212b1033213a4dcGilad Arnold return self._on_error(self._expected_attrs, actual_event, 180248108c1c925791d926105f42212b1033213a4dcGilad Arnold mismatched_attrs) 181248108c1c925791d926105f42212b1033213a4dcGilad Arnold if self._on_error is None: 182248108c1c925791d926105f42212b1033213a4dcGilad Arnold return ('Received event (%s) does not match expectation (%s)' % 183248108c1c925791d926105f42212b1033213a4dcGilad Arnold (self._attrs_to_str(actual_event), self)) 184248108c1c925791d926105f42212b1033213a4dcGilad Arnold return self._on_error 1850ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1860ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1870ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def _verify_attr(self, attr_name, expected_attr_val, actual_attr_val): 1880ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Verifies that an actual log event attributes matches expected on. 1890ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1900ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @param attr_name: name of the attribute to verify 1910ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @param expected_attr_val: expected attribute value 1920ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @param actual_attr_val: actual attribute value 1930ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1940ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @return True if actual value is present and matches, False otherwise. 1950ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1960ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 19745f02ae47134953169805d281992c9edf0019250Chris Sosa # None values are assumed to be missing and non-matching. 198f014ab424450fd595c347d905ac06ccf3d6faaddGilad Arnold if actual_attr_val is None: 1990338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.error('No value found for %s (expected %s)', 2000338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold *self._attr_name_and_values(attr_name, 2010338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold expected_attr_val)[0:2]) 20245f02ae47134953169805d281992c9edf0019250Chris Sosa return False 20345f02ae47134953169805d281992c9edf0019250Chris Sosa 20416e76893c34739649a751f42b3881b57620d665cGilad Arnold # We allow expected version numbers (e.g. 2940.0.0) to be contained in 20516e76893c34739649a751f42b3881b57620d665cGilad Arnold # actual values (2940.0.0-a1); this is necessary for the test to pass 20616e76893c34739649a751f42b3881b57620d665cGilad Arnold # with developer / non-release images. 20716e76893c34739649a751f42b3881b57620d665cGilad Arnold if (actual_attr_val == expected_attr_val or 20816e76893c34739649a751f42b3881b57620d665cGilad Arnold ('version' in attr_name and expected_attr_val in actual_attr_val)): 20916e76893c34739649a751f42b3881b57620d665cGilad Arnold return True 2100ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 21116e76893c34739649a751f42b3881b57620d665cGilad Arnold return False 2120ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2130ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 214248108c1c925791d926105f42212b1033213a4dcGilad Arnold def get_attrs(self): 215248108c1c925791d926105f42212b1033213a4dcGilad Arnold """Returns a dictionary of expected attributes.""" 216248108c1c925791d926105f42212b1033213a4dcGilad Arnold return dict(self._expected_attrs) 217248108c1c925791d926105f42212b1033213a4dcGilad Arnold 218248108c1c925791d926105f42212b1033213a4dcGilad Arnold 2190ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldclass ExpectedUpdateEventChain(object): 2200ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Defines a chain of expected update events.""" 221248108c1c925791d926105f42212b1033213a4dcGilad Arnold def __init__(self): 222fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo self._expected_events_chain = [] 2230ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2240ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 225fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo def add_event(self, expected_events, timeout, on_timeout=None): 226248108c1c925791d926105f42212b1033213a4dcGilad Arnold """Adds an expected event to the chain. 227248108c1c925791d926105f42212b1033213a4dcGilad Arnold 228fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo @param expected_events: The ExpectedEvent, or a list thereof, to wait 229fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo for. If a list is passed, it will wait for *any* 230fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo of the provided events, but only one of those. 231248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param timeout: A timeout (in seconds) to wait for the event. 232248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param on_timeout: An error string to use if the event times out. If 233248108c1c925791d926105f42212b1033213a4dcGilad Arnold None, a generic message is used. 2340ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 235fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo if isinstance(expected_events, ExpectedUpdateEvent): 23616e76893c34739649a751f42b3881b57620d665cGilad Arnold expected_events = [expected_events] 237fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo self._expected_events_chain.append( 238fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo (expected_events, timeout, on_timeout)) 2390ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2400ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 241cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @staticmethod 242fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo def _format_event_with_timeout(expected_events, timeout): 243cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold """Returns a string representation of the event, with timeout.""" 244248108c1c925791d926105f42212b1033213a4dcGilad Arnold until = 'within %s seconds' % timeout if timeout else 'indefinitely' 245fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo return '%s, %s' % (' OR '.join(map(str, expected_events)), until) 2460ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2470ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2480ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def __str__(self): 2490ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold return ('[%s]' % 2500ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold ', '.join( 251fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo [self._format_event_with_timeout(expected_events, timeout) 252fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo for expected_events, timeout, _ 253fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo in self._expected_events_chain])) 2540ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2550ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2560ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def __repr__(self): 257fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo return str(self._expected_events_chain) 2580ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2590ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2600ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def verify(self, get_next_event): 2610ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Verifies that an actual stream of events complies. 2620ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2630ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @param get_next_event: a function returning the next event 2640ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 265ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @raises ExpectedUpdateEventChainFailed if we failed to verify an event. 2660ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2670ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 268fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo for expected_events, timeout, on_timeout in self._expected_events_chain: 2690338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Expecting %s', 270fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo self._format_event_with_timeout(expected_events, 271248108c1c925791d926105f42212b1033213a4dcGilad Arnold timeout)) 272248108c1c925791d926105f42212b1033213a4dcGilad Arnold err_msg = self._verify_event_with_timeout( 273fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo expected_events, timeout, on_timeout, get_next_event) 274248108c1c925791d926105f42212b1033213a4dcGilad Arnold if err_msg is not None: 275248108c1c925791d926105f42212b1033213a4dcGilad Arnold logging.error('Failed expected event: %s', err_msg) 276248108c1c925791d926105f42212b1033213a4dcGilad Arnold raise ExpectedUpdateEventChainFailed(err_msg) 2770ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2780ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 279cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @staticmethod 280fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo def _verify_event_with_timeout(expected_events, timeout, on_timeout, 281248108c1c925791d926105f42212b1033213a4dcGilad Arnold get_next_event): 2820ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Verify an expected event occurs within a given timeout. 2830ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 284fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo @param expected_events: the list of possible events expected next 285248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param timeout: specified in seconds 286248108c1c925791d926105f42212b1033213a4dcGilad Arnold @param on_timeout: A string to return if timeout occurs, or None. 2870ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @param get_next_event: function returning the next event in a stream 2880ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 289248108c1c925791d926105f42212b1033213a4dcGilad Arnold @return None if event complies, an error string otherwise. 2900ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 2910ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 2920ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold base_timestamp = curr_timestamp = time.time() 2930ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold expired_timestamp = base_timestamp + timeout 2940ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold while curr_timestamp <= expired_timestamp: 2950ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold new_event = get_next_event() 2960ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold if new_event: 2970338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Event received after %s seconds', 2980338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold round(curr_timestamp - base_timestamp, 1)) 29916e76893c34739649a751f42b3881b57620d665cGilad Arnold results = [event.verify(new_event) for event in expected_events] 30016e76893c34739649a751f42b3881b57620d665cGilad Arnold return None if None in results else ' AND '.join(results) 3010ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3020ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold # No new events, sleep for one second only (so we don't miss 3030ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold # events at the end of the allotted timeout). 3040ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold time.sleep(1) 3050ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold curr_timestamp = time.time() 3060ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3070338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.error('Timeout expired') 308248108c1c925791d926105f42212b1033213a4dcGilad Arnold if on_timeout is None: 309248108c1c925791d926105f42212b1033213a4dcGilad Arnold return ('Waiting for event %s timed out after %d seconds' % 310fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo (' OR '.join(map(str, expected_events)), timeout)) 311248108c1c925791d926105f42212b1033213a4dcGilad Arnold return on_timeout 3120ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3130ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3140ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldclass UpdateEventLogVerifier(object): 3150ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Verifies update event chains on a devserver update log.""" 31603901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold def __init__(self, event_log_url, url_request_timeout=None): 3170ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold self._event_log_url = event_log_url 31803901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold self._url_request_timeout = url_request_timeout 3190ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold self._event_log = [] 3200ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold self._num_consumed_events = 0 3210ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3220ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 323fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo def verify_expected_events_chain(self, expected_event_chain): 324ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa """Verify a given event chain. 325ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa 326ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa @param expected_event_chain: instance of expected event chain. 327ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 328ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @raises ExpectedUpdateEventChainFailed if we failed to verify the an 329ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa event. 330ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa """ 331ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa expected_event_chain.verify(self._get_next_log_event) 3320ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3330ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3340ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def _get_next_log_event(self): 3350ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Returns the next event in an event log. 3360ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3370ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold Uses the URL handed to it during initialization to obtain the host log 3380ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold from a devserver. If new events are encountered, the first of them is 3390ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold consumed and returned. 3400ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3410ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @return The next new event in the host log, as reported by devserver; 34203901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold None if no such event was found or an error occurred. 3430ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3440ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 3450ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold # (Re)read event log from devserver, if necessary. 3460ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold if len(self._event_log) <= self._num_consumed_events: 34703901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold try: 34803901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold if self._url_request_timeout: 34903901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold conn = urllib2.urlopen(self._event_log_url, 35003901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold timeout=self._url_request_timeout) 35103901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold else: 35203901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold conn = urllib2.urlopen(self._event_log_url) 35303901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold except urllib2.URLError, e: 3540338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.warning('Failed to read event log url: %s', e) 35503901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold return None 356a0261611f9be78ee9d456fd50bea81e9eda07c17Gilad Arnold except socket.timeout, e: 357a0261611f9be78ee9d456fd50bea81e9eda07c17Gilad Arnold logging.warning('Timed out reading event log url: %s', e) 358a0261611f9be78ee9d456fd50bea81e9eda07c17Gilad Arnold return None 35903901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold 3600ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold event_log_resp = conn.read() 3610ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold conn.close() 3620ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold self._event_log = json.loads(event_log_resp) 3630ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 36403901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold # Return next new event, if one is found. 3650ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold if len(self._event_log) > self._num_consumed_events: 3667572b05fd3356b2ddf6866958e80c7885e44647bGilad Arnold new_event = { 3677572b05fd3356b2ddf6866958e80c7885e44647bGilad Arnold key: str(val) for key, val 3687572b05fd3356b2ddf6866958e80c7885e44647bGilad Arnold in self._event_log[self._num_consumed_events].iteritems() 3697572b05fd3356b2ddf6866958e80c7885e44647bGilad Arnold } 3700ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold self._num_consumed_events += 1 3710338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Consumed new event: %s', new_event) 3720ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold return new_event 3730ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3740ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 375793344b359304c31d78197788b55cfbbe2636025Chris Sosaclass OmahaDevserverFailedToStart(error.TestError): 376793344b359304c31d78197788b55cfbbe2636025Chris Sosa """Raised when a omaha devserver fails to start.""" 377793344b359304c31d78197788b55cfbbe2636025Chris Sosa 378793344b359304c31d78197788b55cfbbe2636025Chris Sosa 3790ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnoldclass OmahaDevserver(object): 3800ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Spawns a test-private devserver instance.""" 381793344b359304c31d78197788b55cfbbe2636025Chris Sosa # How long to wait for a devserver to start. 38252c35724507ec105053d2af792fe161a627e05c1Alex Deymo _WAIT_FOR_DEVSERVER_STARTED_SECONDS = 30 383793344b359304c31d78197788b55cfbbe2636025Chris Sosa 384260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold # How long to sleep (seconds) between checks to see if a devserver is up. 38503901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold _WAIT_SLEEP_INTERVAL = 1 3860ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3876f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold # Max devserver execution time (seconds); used with timeout(1) to ensure we 3886f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold # don't have defunct instances hogging the system. 3896f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold _DEVSERVER_TIMELIMIT_SECONDS = 12 * 60 * 60 390793344b359304c31d78197788b55cfbbe2636025Chris Sosa 391793344b359304c31d78197788b55cfbbe2636025Chris Sosa 392260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def __init__(self, omaha_host, devserver_dir, update_payload_staged_url): 3930ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """Starts a private devserver instance, operating at Omaha capacity. 3940ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3950ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @param omaha_host: host address where the devserver is spawned. 39603901089c0db27508a6b2ff61ae8b75eba77cf37Gilad Arnold @param devserver_dir: path to the devserver source directory 397ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @param update_payload_staged_url: URL to provision for update requests. 3980ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 3990ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 400ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa if not update_payload_staged_url: 4010338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold raise error.TestError('Missing update payload url') 4020ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 4036c55bdb98e967675456a71a0971b81058536cac8Chris Sosa self._omaha_host = omaha_host 404260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_pid = 0 405260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_port = 0 # Determined later from devserver portfile. 4066c55bdb98e967675456a71a0971b81058536cac8Chris Sosa self._devserver_dir = devserver_dir 407ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa self._update_payload_staged_url = update_payload_staged_url 4086c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 4096c55bdb98e967675456a71a0971b81058536cac8Chris Sosa self._devserver_ssh = hosts.SSHHost(self._omaha_host, 4106c55bdb98e967675456a71a0971b81058536cac8Chris Sosa user=os.environ['USER']) 411260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 4123563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa # Temporary files for various devserver outputs. 4133563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_logfile = None 414238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo self._devserver_stdoutfile = None 4153563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_portfile = None 4163563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_pidfile = None 4173563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_static_dir = None 4183563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 4193563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 4203563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa def _cleanup_devserver_files(self): 4213563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa """Cleans up the temporary devserver files.""" 422238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo for filename in (self._devserver_logfile, self._devserver_stdoutfile, 423238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo self._devserver_portfile, self._devserver_pidfile): 424238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo if filename: 425238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo self._devserver_ssh.run('rm -f %s' % filename, 426238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo ignore_status=True) 4273563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 4283563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa if self._devserver_static_dir: 4293563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_ssh.run('rm -rf %s' % self._devserver_static_dir, 4303563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa ignore_status=True) 4313563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 432260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 4333563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa def _create_tempfile_on_devserver(self, label, dir=False): 4343563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa """Creates a temporary file/dir on the devserver and returns its path. 435260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 436260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold @param label: Identifier for the file context (string, no whitespaces). 4373563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa @param dir: If True, create a directory instead of a file. 438260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 439260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold @raises test.TestError: If we failed to invoke mktemp on the server. 440260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold @raises OmahaDevserverFailedToStart: If tempfile creation failed. 441260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """ 442260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold remote_cmd = 'mktemp --tmpdir devserver-%s.XXXXXX' % label 4433563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa if dir: 4443563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa remote_cmd += ' --directory' 4453563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 446260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold try: 447260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold result = self._devserver_ssh.run(remote_cmd, ignore_status=True) 448260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold except error.AutoservRunError as e: 449260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._log_and_raise_remote_ssh_error(e) 450260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold if result.exit_status != 0: 451260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold raise OmahaDevserverFailedToStart( 452260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 'Could not create a temporary %s file on the devserver, ' 453a7412a90ffefac2b6314726b284af1b31d6bd797Alex Deymo 'error output: "%s"' % (label, result.stderr)) 454260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return result.stdout.strip() 455260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 456260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold @staticmethod 457260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _log_and_raise_remote_ssh_error(e): 458260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Logs failure to ssh remote, then raises a TestError.""" 459260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.debug('Failed to ssh into the devserver: %s', e) 460260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.error('If you are running this locally it means you did not ' 461260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 'configure ssh correctly.') 462260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold raise error.TestError('Failed to ssh into the devserver: %s' % e) 463260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 464260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 465260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _read_int_from_devserver_file(self, filename): 466260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Reads and returns an integer value from a file on the devserver.""" 467260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return int(self._get_devserver_file_content(filename).strip()) 4680ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 4696c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 470793344b359304c31d78197788b55cfbbe2636025Chris Sosa def _wait_for_devserver_to_start(self): 471793344b359304c31d78197788b55cfbbe2636025Chris Sosa """Waits until the devserver starts within the time limit. 472793344b359304c31d78197788b55cfbbe2636025Chris Sosa 473260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold Infers and sets the devserver PID and serving port. 474260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 475793344b359304c31d78197788b55cfbbe2636025Chris Sosa Raises: 476793344b359304c31d78197788b55cfbbe2636025Chris Sosa OmahaDevserverFailedToStart: If the time limit is reached and we 477793344b359304c31d78197788b55cfbbe2636025Chris Sosa cannot connect to the devserver. 478793344b359304c31d78197788b55cfbbe2636025Chris Sosa """ 479260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold # Compute the overall timeout. 480260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold deadline = time.time() + self._WAIT_FOR_DEVSERVER_STARTED_SECONDS 481260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 482260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold # First, wait for port file to be filled and determine the server port. 48394b9ad4983cc735e162a3549c1628d08241b0d57Gilad Arnold logging.warning('Waiting for devserver to start up.') 484260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold while time.time() < deadline: 485260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold try: 486260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_pid = self._read_int_from_devserver_file( 487260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_pidfile) 488260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_port = self._read_int_from_devserver_file( 489260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_portfile) 490260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.info('Devserver pid is %d, serving on port %d', 491260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_pid, self._devserver_port) 492260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold break 493260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold except Exception: # Couldn't read file or corrupt content. 494260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold time.sleep(self._WAIT_SLEEP_INTERVAL) 495260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold else: 49652c35724507ec105053d2af792fe161a627e05c1Alex Deymo try: 49752c35724507ec105053d2af792fe161a627e05c1Alex Deymo self._devserver_ssh.run_output('uptime') 49852c35724507ec105053d2af792fe161a627e05c1Alex Deymo except error.AutoservRunError as e: 49952c35724507ec105053d2af792fe161a627e05c1Alex Deymo logging.debug('Failed to run uptime on the devserver: %s', e) 50094b9ad4983cc735e162a3549c1628d08241b0d57Gilad Arnold raise OmahaDevserverFailedToStart( 501260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 'The test failed to find the pid/port of the omaha ' 50252c35724507ec105053d2af792fe161a627e05c1Alex Deymo 'devserver after %d seconds. Check the dumped devserver ' 50352c35724507ec105053d2af792fe161a627e05c1Alex Deymo 'logs and devserver load for more information.' % 50452c35724507ec105053d2af792fe161a627e05c1Alex Deymo self._WAIT_FOR_DEVSERVER_STARTED_SECONDS) 505260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 506260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold # Check that the server is reponsding to network requests. 507260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.warning('Waiting for devserver to accept network requests.') 508260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold url = 'http://%s' % self.get_netloc() 509260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold while time.time() < deadline: 5109e2c98d8d0c61a816e736a387591950af9505504xixuan if dev_server.ImageServer.devserver_healthy(url, timeout_min=0.1): 511260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold break 512793344b359304c31d78197788b55cfbbe2636025Chris Sosa 513793344b359304c31d78197788b55cfbbe2636025Chris Sosa # TODO(milleral): Refactor once crbug.com/221626 is resolved. 514793344b359304c31d78197788b55cfbbe2636025Chris Sosa time.sleep(self._WAIT_SLEEP_INTERVAL) 515793344b359304c31d78197788b55cfbbe2636025Chris Sosa else: 516793344b359304c31d78197788b55cfbbe2636025Chris Sosa raise OmahaDevserverFailedToStart( 517793344b359304c31d78197788b55cfbbe2636025Chris Sosa 'The test failed to establish a connection to the omaha ' 518793344b359304c31d78197788b55cfbbe2636025Chris Sosa 'devserver it set up on port %d. Check the dumped ' 519260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 'devserver logs for more information.' % 520260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_port) 521793344b359304c31d78197788b55cfbbe2636025Chris Sosa 522793344b359304c31d78197788b55cfbbe2636025Chris Sosa 5236c55bdb98e967675456a71a0971b81058536cac8Chris Sosa def start_devserver(self): 524793344b359304c31d78197788b55cfbbe2636025Chris Sosa """Starts the devserver and confirms it is up. 525793344b359304c31d78197788b55cfbbe2636025Chris Sosa 526793344b359304c31d78197788b55cfbbe2636025Chris Sosa Raises: 527260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold test.TestError: If we failed to spawn the remote devserver. 528793344b359304c31d78197788b55cfbbe2636025Chris Sosa OmahaDevserverFailedToStart: If the time limit is reached and we 529793344b359304c31d78197788b55cfbbe2636025Chris Sosa cannot connect to the devserver. 5306c55bdb98e967675456a71a0971b81058536cac8Chris Sosa """ 5312f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa update_payload_url_base, update_payload_path = self._split_url( 532ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa self._update_payload_staged_url) 5333563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 5343563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa # Allocate temporary files for various server outputs. 5353563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_logfile = self._create_tempfile_on_devserver('log') 536238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo self._devserver_stdoutfile = self._create_tempfile_on_devserver( 537238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo 'stdout') 5383563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_portfile = self._create_tempfile_on_devserver('port') 5393563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_pidfile = self._create_tempfile_on_devserver('pid') 5403563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._devserver_static_dir = self._create_tempfile_on_devserver( 5413563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 'static', dir=True) 5423563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa 5436f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold # Invoke the Omaha/devserver on the remote server. Will attempt to kill 5446f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold # it with a SIGTERM after a predetermined timeout has elapsed, followed 5456f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold # by SIGKILL if not dead within 30 seconds from the former signal. 5460ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold cmdlist = [ 5476f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold 'timeout', '-s', 'TERM', '-k', '30', 5486f36e954e1d73e961b8d47ebf4ec7d267c4307f4Gilad Arnold str(self._DEVSERVER_TIMELIMIT_SECONDS), 5496c55bdb98e967675456a71a0971b81058536cac8Chris Sosa '%s/devserver.py' % self._devserver_dir, 5500ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold '--payload=%s' % update_payload_path, 551260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold '--port=0', 552260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold '--pidfile=%s' % self._devserver_pidfile, 553260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold '--portfile=%s' % self._devserver_portfile, 554260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold '--logfile=%s' % self._devserver_logfile, 5550ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold '--remote_payload', 5560ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold '--urlbase=%s' % update_payload_url_base, 5570ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold '--max_updates=1', 5580ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold '--host_log', 5593563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa '--static_dir=%s' % self._devserver_static_dir, 5601fd5e3042c6d8de3bdbb8cc169b5e0c5bd61af93Kevin Cernekee '--critical_update', 5610ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold ] 562238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo remote_cmd = '( %s ) </dev/null >%s 2>&1 &' % ( 563238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo ' '.join(cmdlist), self._devserver_stdoutfile) 5646c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 565260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.info('Starting devserver with %r', remote_cmd) 566260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold try: 567260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_ssh.run_output(remote_cmd) 568260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold except error.AutoservRunError as e: 569260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._log_and_raise_remote_ssh_error(e) 5706c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 571260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold try: 572260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._wait_for_devserver_to_start() 573260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold except OmahaDevserverFailedToStart: 574260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._kill_remote_process() 575260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._dump_devserver_log() 576299e7788f7054dccab196dd7274f75ad41b66606Alex Deymo self._cleanup_devserver_files() 577260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold raise 57838ba6b79b19b5bdd9cfe71b26efd0c267768527aGilad Arnold 5796c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 580260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _kill_remote_process(self): 581260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Kills the devserver and verifies it's down; clears the remote pid.""" 582260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def devserver_down(): 583cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold """Ensure that the devserver process is down.""" 584260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return not self._remote_process_alive() 5856c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 586260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold if devserver_down(): 5876c55bdb98e967675456a71a0971b81058536cac8Chris Sosa return 5886c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 589260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold for signal in 'SIGTERM', 'SIGKILL': 590260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold remote_cmd = 'kill -s %s %s' % (signal, self._devserver_pid) 591260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_ssh.run(remote_cmd) 592260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold try: 593260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold client_utils.poll_for_condition( 594260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold devserver_down, sleep_interval=1, desc='devserver down') 595260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold break 596260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold except client_utils.TimeoutError: 597260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.warning('Could not kill devserver with %s.', signal) 598260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold else: 599260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold logging.warning('Failed to kill devserver, giving up.') 600260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 601260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._devserver_pid = None 6026c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 6036c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 604260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _remote_process_alive(self): 605260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Tests whether the remote devserver process is running.""" 606260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold if not self._devserver_pid: 607260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return False 608260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold remote_cmd = 'test -e /proc/%s' % self._devserver_pid 609260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold result = self._devserver_ssh.run(remote_cmd, ignore_status=True) 610260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return result.exit_status == 0 6116c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 6126c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 6136c55bdb98e967675456a71a0971b81058536cac8Chris Sosa def get_netloc(self): 6146c55bdb98e967675456a71a0971b81058536cac8Chris Sosa """Returns the netloc (host:port) of the devserver.""" 615260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold if not (self._devserver_pid and self._devserver_port): 6160338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold raise error.TestError('No running omaha/devserver') 6176c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 618260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return '%s:%s' % (self._omaha_host, self._devserver_port) 6190ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 6206c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 6212f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa def get_update_url(self): 6222f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """Returns the update_url you can use to update via this server.""" 6232f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa return urlparse.urlunsplit(('http', self.get_netloc(), '/update', 6242f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa '', '')) 6252f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 6262f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 627260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _get_devserver_file_content(self, filename): 628260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Returns the content of a file on the devserver.""" 6291391ae87d9941601dfc53377e2c187edcb0071dcGilad Arnold return self._devserver_ssh.run_output('cat %s' % filename, 6301391ae87d9941601dfc53377e2c187edcb0071dcGilad Arnold stdout_tee=None) 631260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 632260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 633260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _get_devserver_log(self): 634260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Obtain the devserver output.""" 635260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold return self._get_devserver_file_content(self._devserver_logfile) 636260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 637260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold 638238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo def _get_devserver_stdout(self): 639238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo """Obtain the devserver output in stdout and stderr.""" 640238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo return self._get_devserver_file_content(self._devserver_stdoutfile) 641238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo 642238cff65701d1ea7ea9458baa7148f5a76f9b183Alex Deymo 643260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def _dump_devserver_log(self, logging_level=logging.ERROR): 6443563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa """Dump the devserver log to the autotest log. 64519426059c61c1b318b5d4bd04d70dacb63ec827dChris Sosa 64619426059c61c1b318b5d4bd04d70dacb63ec827dChris Sosa @param logging_level: logging level (from logging) to log the output. 64719426059c61c1b318b5d4bd04d70dacb63ec827dChris Sosa """ 6481391ae87d9941601dfc53377e2c187edcb0071dcGilad Arnold logging.log(logging_level, "Devserver stdout and stderr:\n" + 6491391ae87d9941601dfc53377e2c187edcb0071dcGilad Arnold snippet(self._get_devserver_stdout())) 6501391ae87d9941601dfc53377e2c187edcb0071dcGilad Arnold logging.log(logging_level, "Devserver log file:\n" + 6511391ae87d9941601dfc53377e2c187edcb0071dcGilad Arnold snippet(self._get_devserver_log())) 6520ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 6530ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 6540ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @staticmethod 6550ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold def _split_url(url): 6562f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """Splits a URL into the URL base and path.""" 6570ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold split_url = urlparse.urlsplit(url) 6580ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold url_base = urlparse.urlunsplit( 6592f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa (split_url.scheme, split_url.netloc, '', '', '')) 6602f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa url_path = split_url.path 6612f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa return url_base, url_path.lstrip('/') 6620ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 6630ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 664260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold def stop_devserver(self): 665260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold """Kill remote process and wait for it to die, dump its output.""" 6666c55bdb98e967675456a71a0971b81058536cac8Chris Sosa if not self._devserver_pid: 667ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa logging.error('No running omaha/devserver.') 668ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa return 6696c55bdb98e967675456a71a0971b81058536cac8Chris Sosa 6700338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Killing omaha/devserver') 671260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._kill_remote_process() 67219426059c61c1b318b5d4bd04d70dacb63ec827dChris Sosa logging.debug('Final devserver log before killing') 673260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold self._dump_devserver_log(logging.DEBUG) 6743563042207623fbb9db02018156e8b5fa0e4c2c3Chris Sosa self._cleanup_devserver_files() 6750ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 6760ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 677f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnoldclass TestPlatform(object): 678f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """An interface and factory for platform-dependent functionality.""" 67915384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 6802f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # Named tuple containing urls for staged urls needed for test. 6812f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # source_url: url to find the update payload for the source image. 6822f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # source_stateful_url: url to find the stateful payload for the source 6832f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # image. 6842f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # target_url: url to find the update payload for the target image. 6852f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # target_stateful_url: url to find the stateful payload for the target 6862f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa # image. 687f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold StagedURLs = collections.namedtuple( 688f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 'StagedURLs', 6892f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa ['source_url', 'source_stateful_url', 'target_url', 6902f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 'target_stateful_url']) 6912f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 6920ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 693f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def __init__(self): 694f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold assert False, 'Cannot instantiate this interface' 695f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 696f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 697f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @staticmethod 698f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def create(host): 699f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Returns a TestPlatform implementation based on the host type. 700f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 701f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold *DO NOT* override this method. 702f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 703f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param host: a host object representing the DUT 704f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 705f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return A TestPlatform implementation. 706f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 707f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold os_type = host.get_os_type() 708585cbd6a5cc32db99c6745aa4333471b6b095b43Keith Haddow if os_type in ('cros', 'moblab'): 709f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return ChromiumOSTestPlatform(host) 710f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 711f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise error.TestError('Unknown OS type reported by host: %s' % os_type) 712f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 7130ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 714f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def initialize(self, autotest_devserver, devserver_dir): 715f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Initialize the object. 7160ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 717f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param autotest_devserver: Instance of client.common_lib.dev_server to 718f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold use to reach the devserver instance for this 719f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold build. 720f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param devserver_dir: Path to devserver source tree. 7210ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 722f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 7230ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 7240ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 725f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def prep_artifacts(self, test_conf): 726f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Prepares update artifacts for the test. 72709706f18ea3878f1b173a63fdc92f2fc6450bb4dGilad Arnold 728f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold The test config must include 'source_payload_uri' and 729f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 'target_payload_uri'. In addition, it may include platform-specific 730f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold values as determined by the test control file. 73109706f18ea3878f1b173a63fdc92f2fc6450bb4dGilad Arnold 732f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param test_conf: Dictionary containing the test configuration. 73309706f18ea3878f1b173a63fdc92f2fc6450bb4dGilad Arnold 734f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return A tuple of staged URLs. 735f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 736f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @raise error.TestError on failure. 73709706f18ea3878f1b173a63fdc92f2fc6450bb4dGilad Arnold """ 738f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 73909706f18ea3878f1b173a63fdc92f2fc6450bb4dGilad Arnold 74009706f18ea3878f1b173a63fdc92f2fc6450bb4dGilad Arnold 741f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def reboot_device(self): 742f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Reboots the device.""" 743f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 744f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 745f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 746f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def prep_device_for_update(self, source_release): 747f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Prepares the device for update. 748f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 7497ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold @param source_release: Source release version (string), or None. 750f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 751f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @raise error.TestError on failure. 752f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 753f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 754f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 755f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 756f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def get_active_slot(self): 757f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Returns the active boot slot of the device.""" 758f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 759f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 760f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 761f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def start_update_perf(self, bindir): 762f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Starts performance monitoring (if available). 763f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 764f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param bindir: Directory containing test binary files. 765f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 766f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 767f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 768f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 769f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def stop_update_perf(self): 770f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Stops performance monitoring and returns data (if available). 771f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 772f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return Dictionary containing performance attributes. 773f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 774f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 775f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 776f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 777f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def trigger_update(self, omaha_devserver): 778f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Kicks off an update. 779f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 780f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param omaha_devserver: OmahaDevserver instance. 781f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 782f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 783f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 784f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 785f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def finalize_update(self): 786f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Performs post-update procedures.""" 787f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 788f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 789f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 790f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def get_update_log(self, num_lines): 791f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Returns the update log. 792f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 793f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param num_lines: Number of log lines to return (tail), zero for all. 794f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 795f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return String containing the last |num_lines| from the update log. 796f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 797f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 798f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 799f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 800f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def check_device_after_update(self, target_release): 801f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Runs final sanity checks. 802f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 8037ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold @param target_release: Target release version (string), or None. 804f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 805f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @raise error.TestError on failure. 806f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 807f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise NotImplementedError 808f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 809f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 81034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar def oobe_triggers_update(self): 81134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar """Returns True if this host has an OOBE flow during which 81234618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar it will perform an update check and perhaps an update. 81334618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar One example of such a flow is Hands-Off Zero-Touch Enrollment. 81434618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 81534618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar @return Boolean indicating whether the DUT's OOBE triggers an update. 81634618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar """ 81734618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar raise NotImplementedError 81834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 81934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 82034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar def verify_version(self, version): 82134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar """Compares the OS version on the DUT with the provided version. 82234618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 82334618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar @param version: The version to compare with (string). 82434618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar @raise error.TestFail if the versions differ. 82534618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar """ 82634618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar actual_version = self._host.get_release_version() 82734618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar if actual_version != version: 82834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar err_msg = 'Failed to verify OS version. Expected %s, was %s' % ( 82934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar version, actual_version) 83034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar logging.error(err_msg) 83134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar raise error.TestFail(err_msg) 83234618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 83334618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 834f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnoldclass ChromiumOSTestPlatform(TestPlatform): 835f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """A TestPlatform implementation for Chromium OS.""" 836f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 837f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _STATEFUL_UPDATE_FILENAME = 'stateful.tgz' 838f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 839f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def __init__(self, host): 840f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._host = host 841f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._autotest_devserver = None 842f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._devserver_dir = None 843f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._staged_urls = None 844f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._perf_mon_pid = None 845f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 846f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 847f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _stage_payload(self, devserver_label, filename, archive_url=None): 8482f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """Stage the given payload onto the devserver. 8490ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 8502f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa Works for either a stateful or full/delta test payload. Expects the 8512f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa gs_path or a combo of devserver_label + filename. 8520ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 8532f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa @param devserver_label: The build name e.g. x86-mario-release/<version>. 8542f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa If set, assumes default gs archive bucket and 8552f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa requires filename to be specified. 8562f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa @param filename: In conjunction with devserver_label, if just specifying 8572f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa the devserver label name, this is which file are you 8582f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa downloading. 85915384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold @param archive_url: An optional GS archive location, if not using the 86015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold devserver's default. 861ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa 8620ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @return URL of the staged payload on the server. 8630ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 8640ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold @raise error.TestError if there's a problem with staging. 8650ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 8660ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 8672f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa try: 868f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._autotest_devserver.stage_artifacts( 86972f48be6c3f7e30edc6ff9a785f9c4cb53521af4Don Garrett image=devserver_label, files=[filename], 87072f48be6c3f7e30edc6ff9a785f9c4cb53521af4Don Garrett archive_url=archive_url) 871f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return self._autotest_devserver.get_staged_file_url(filename, 872f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver_label) 8732f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa except dev_server.DevServerException, e: 8740338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold raise error.TestError('Failed to stage payload: %s' % e) 8750ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 8760ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 877f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _stage_payload_by_uri(self, payload_uri): 87815384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold """Stage a payload based on its GS URI. 87915384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 88015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold This infers the build's label, filename and GS archive from the 88115384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold provided GS URI. 88215384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 88315384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold @param payload_uri: The full GS URI of the payload. 88415384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 88515384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold @return URL of the staged payload on the server. 88615384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 88715384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold @raise error.TestError if there's a problem with staging. 88815384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 88915384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold """ 89015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold archive_url, _, filename = payload_uri.rpartition('/') 89115384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold devserver_label = urlparse.urlsplit(archive_url).path.strip('/') 89294694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold return self._stage_payload(devserver_label, filename, 89394694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold archive_url=archive_url) 89415384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 89515384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 896cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @staticmethod 897cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold def _payload_to_update_url(payload_url): 8982f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """Given a update or stateful payload url, returns the update url.""" 8997231260da421abdf5ceceff6ab60155936dca21fChris Sosa # We want to transform it to the correct omaha url which is 9007231260da421abdf5ceceff6ab60155936dca21fChris Sosa # <hostname>/update/...LABEL. 9012f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa base_url = payload_url.rpartition('/')[0] 9022f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa return base_url.replace('/static/', '/update/') 9032f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 9042f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 90515384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold def _get_stateful_uri(self, build_uri): 90615384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold """Returns a complete GS URI of a stateful update given a build path.""" 90715384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold return '/'.join([build_uri.rstrip('/'), self._STATEFUL_UPDATE_FILENAME]) 90815384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 90915384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 91015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold def _payload_to_stateful_uri(self, payload_uri): 91115384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold """Given a payload GS URI, returns the corresponding stateful URI.""" 91215384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold build_uri = payload_uri.rpartition('/')[0] 913d8b9093f06cb882f7f46ca2f51d06630c9004f6dGwendal Grignou if build_uri.endswith('payloads'): 914d8b9093f06cb882f7f46ca2f51d06630c9004f6dGwendal Grignou build_uri = build_uri.rpartition('/')[0] 91515384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold return self._get_stateful_uri(build_uri) 9162f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 9172f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 918f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _update_via_test_payloads(self, omaha_host, payload_url, stateful_url, 919f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold clobber): 920cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold """Given the following update and stateful urls, update the DUT. 921cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold 922cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold Only updates the rootfs/stateful if the respective url is provided. 923cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold 924cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @param omaha_host: If updating rootfs, redirect updates through this 925cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold host. Should be None iff payload_url is None. 926cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @param payload_url: If set, the specified url to find the update 927cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold payload. 928cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @param stateful_url: If set, the specified url to find the stateful 929cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold payload. 930cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @param clobber: If True, do a clean install of stateful. 931cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold """ 932cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold def perform_update(url, is_stateful): 933cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold """Perform a rootfs/stateful update using given URL. 934cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold 935cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @param url: URL to update from. 936cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold @param is_stateful: Whether this is a stateful or rootfs update. 937cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold """ 938cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold if url: 939cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold updater = autoupdater.ChromiumOSUpdater(url, host=self._host) 940cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold if is_stateful: 941cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold updater.update_stateful(clobber=clobber) 942cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold else: 9439e30d201214e586d185b497dea9a13a7e8d0495eGilad Arnold updater.update_image() 944cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold 945cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold # We create a OmahaDevserver to redirect blah.bin to update/. This 946cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold # allows us to use any payload filename to serve an update. 947cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold temp_devserver = None 948cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold try: 9496273f5a63be4a2695b19cc76189607def69f5b27Gwendal Grignou if omaha_host: 950cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold temp_devserver = OmahaDevserver( 9516273f5a63be4a2695b19cc76189607def69f5b27Gwendal Grignou omaha_host, self._devserver_dir, 9526273f5a63be4a2695b19cc76189607def69f5b27Gwendal Grignou payload_url or stateful_url) 953cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold temp_devserver.start_devserver() 9546273f5a63be4a2695b19cc76189607def69f5b27Gwendal Grignou if payload_url: 955cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold payload_url = temp_devserver.get_update_url() 956cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold 957cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold stateful_url = self._payload_to_update_url(stateful_url) 958cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold 959cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold perform_update(payload_url, False) 960cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold perform_update(stateful_url, True) 961cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold finally: 962cfa14a695654bc9cbfe9b9ce5868d669d0c57e0fGilad Arnold if temp_devserver: 963260957d6079ee23ec6c9857a95fa183a78741b0bGilad Arnold temp_devserver.stop_devserver() 9642f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 9652f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 966f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _install_source_version(self, devserver_hostname, image_url, 967f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold stateful_url): 9682f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """Prepare the specified host with the image given by the urls. 9692f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 970ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @param devserver_hostname: If updating rootfs, redirect updates 971ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa through this host. Should be None iff 972ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa image_url is None. 9732f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa @param image_url: If set, the specified url to find the source image 9742f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa or full payload for the source image. 9752f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa @param stateful_url: If set, the specified url to find the stateful 9762f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa payload. 9772f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """ 97803286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou # Reboot to get us into a clean state. 97903286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou self._host.reboot() 9804cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold try: 9814cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # Since we are installing the source image of the test, clobber 9824cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # stateful. 983f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._update_via_test_payloads(devserver_hostname, image_url, 98494694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold stateful_url, True) 9851b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold except OmahaDevserverFailedToStart as e: 9861b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold logging.fatal('Failed to start private devserver for installing ' 9871b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'the source image (%s) on the DUT', image_url) 9881b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold raise error.TestError( 9891b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'Failed to start private devserver for installing the ' 9901b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'source image on the DUT: %s' % e) 9911b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold except error.AutoservRunError as e: 99203286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou logging.fatal('Error re-imaging the DUT with ' 99303286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou 'the source image from %s', image_url) 99403286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou raise error.TestError('Failed to install ' 99503286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou 'the source image DUT: %s' % e) 99603286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou self._host.reboot() 99703286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou 99803286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou # If powerwashed, need to reinstall stateful_url 99903286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou if not self._host.check_rsync(): 100003286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou logging.warn('Device has been powerwashed, need to reinstall ' 100103286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou 'stateful from %s', stateful_url) 100203286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou self._update_via_test_payloads(devserver_hostname, None, 100303286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou stateful_url, True) 100403286f0f93ff59fc1744099378201f85db9c1883Gwendal Grignou self._host.reboot() 1005f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 1006f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 1007f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _stage_artifacts_onto_devserver(self, test_conf): 10082f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa """Stages artifacts that will be used by the test onto the devserver. 10092f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 10102f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa @param test_conf: a dictionary containing test configuration values 10112f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 1012f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return a StagedURLs tuple containing the staged urls. 1013f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa """ 10140338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Staging images onto autotest devserver (%s)', 1015f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._autotest_devserver.url()) 1016f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 101715384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold staged_source_url = None 101815384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold staged_source_stateful_url = None 10199cd891a18dcdeb1dd5f544322dd64f8f1845c6a6Gilad Arnold try: 10209cd891a18dcdeb1dd5f544322dd64f8f1845c6a6Gilad Arnold source_payload_uri = test_conf['source_payload_uri'] 10219cd891a18dcdeb1dd5f544322dd64f8f1845c6a6Gilad Arnold except KeyError: 10229cd891a18dcdeb1dd5f544322dd64f8f1845c6a6Gilad Arnold # TODO(garnold) Remove legacy key support once control files on all 10239cd891a18dcdeb1dd5f544322dd64f8f1845c6a6Gilad Arnold # release branches have caught up. 10249cd891a18dcdeb1dd5f544322dd64f8f1845c6a6Gilad Arnold source_payload_uri = test_conf['source_image_uri'] 10254cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold if source_payload_uri: 102694694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold staged_source_url = self._stage_payload_by_uri(source_payload_uri) 10274cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold 10284cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # In order to properly install the source image using a full 10294cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # payload we'll also need the stateful update that comes with it. 10304cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # In general, tests may have their source artifacts in a different 10314cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # location than their payloads. This is determined by whether or 10324cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # not the source_archive_uri attribute is set; if it isn't set, 10334cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # then we derive it from the dirname of the source payload. 10344cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold source_archive_uri = test_conf.get('source_archive_uri') 10354cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold if source_archive_uri: 10364cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold source_stateful_uri = self._get_stateful_uri(source_archive_uri) 10372f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa else: 10384cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold source_stateful_uri = self._payload_to_stateful_uri( 10394cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold source_payload_uri) 1040fea7e766e59f8356dad6f70e46c0ee25997863a5Gilad Arnold 10414cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold staged_source_stateful_url = self._stage_payload_by_uri( 104294694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold source_stateful_uri) 104315384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 10444cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # Log source image URLs. 10454cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold logging.info('Source full payload from %s staged at %s', 10464cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold source_payload_uri, staged_source_url) 10474cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold if staged_source_stateful_url: 10484cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold logging.info('Source stateful update from %s staged at %s', 10494cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold source_stateful_uri, staged_source_stateful_url) 105015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 105115384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold target_payload_uri = test_conf['target_payload_uri'] 105294694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold staged_target_url = self._stage_payload_by_uri(target_payload_uri) 105315384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold target_stateful_uri = None 10540e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold staged_target_stateful_url = None 105515384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold target_archive_uri = test_conf.get('target_archive_uri') 10560e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold if target_archive_uri: 10570e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold target_stateful_uri = self._get_stateful_uri(target_archive_uri) 10582f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa else: 10590e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold # Attempt to get the job_repo_url to find the stateful payload for 10600e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold # the target image. 1061368abdf47d544aca4adbaf19d112e3457e056d3dPrathmesh Prabhu info = self._host.host_info_store.get() 1062368abdf47d544aca4adbaf19d112e3457e056d3dPrathmesh Prabhu job_repo_url = info.attributes.get( 1063368abdf47d544aca4adbaf19d112e3457e056d3dPrathmesh Prabhu self._host.job_repo_url_attribute, '') 1064d8b9093f06cb882f7f46ca2f51d06630c9004f6dGwendal Grignou if not job_repo_url: 106515384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold target_stateful_uri = self._payload_to_stateful_uri( 106615384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold target_payload_uri) 10670e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold else: 10680e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold _, devserver_label = tools.get_devserver_build_from_package_url( 10690e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold job_repo_url) 10700e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold staged_target_stateful_url = self._stage_payload( 107194694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold devserver_label, self._STATEFUL_UPDATE_FILENAME) 1072f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 10730e18ad5f71d48432747733e10aad1e1cf5d93113Gilad Arnold if not staged_target_stateful_url and target_stateful_uri: 107415384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold staged_target_stateful_url = self._stage_payload_by_uri( 107594694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold target_stateful_uri) 10762f1ae9f1f2464ebbeb226a6669fa90ef211a179cChris Sosa 1077fea7e766e59f8356dad6f70e46c0ee25997863a5Gilad Arnold # Log target payload URLs. 107815384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold logging.info('%s test payload from %s staged at %s', 107915384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold test_conf['update_type'], target_payload_uri, 108015384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold staged_target_url) 10810338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Target stateful update from %s staged at %s', 108215384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold target_stateful_uri or 'standard location', 108315384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold staged_target_stateful_url) 108415384a4ded97f171906a72106791d19e96e68fa7Gilad Arnold 1085f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return self.StagedURLs(staged_source_url, staged_source_stateful_url, 1086f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold staged_target_url, staged_target_stateful_url) 1087f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 1088f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 1089c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen def _run_login_test(self, release_string): 10901192c41145b533f4138434bfa7e6ba2af9f75330David Haddock """Runs login_LoginSuccess test on the DUT.""" 10917ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold if not release_string: 10927ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold logging.info('No release provided, skipping login test.') 10931192c41145b533f4138434bfa7e6ba2af9f75330David Haddock else: 1094c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen logging.info('Attempting to login (release %s).', release_string) 1095c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen client_at = autotest.Autotest(self._host) 10960c88356d674fbc2545000483603ea3bdb9d13529David Haddock client_at.run_test('login_LoginSuccess', arc_mode='enabled') 1097c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen 1098c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen 1099f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _start_perf_mon(self, bindir): 110031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen """Starts monitoring performance and resource usage on a DUT. 110131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 110231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen Call _stop_perf_mon() with the returned PID to stop monitoring 110331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen and collect the results. 110431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 1105f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param bindir: Directoy containing monitoring script. 1106f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1107f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return The PID of the newly created DUT monitoring process. 110831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen """ 110931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # We can't assume much about the source image so we copy the 111031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # performance monitoring script to the DUT directly. 1111f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold path = os.path.join(bindir, 'update_engine_performance_monitor.py') 111231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen self._host.send_file(path, '/tmp') 111331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen cmd = 'python /tmp/update_engine_performance_monitor.py --start-bg' 111431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen return int(self._host.run(cmd).stdout) 111531887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 111631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 111731887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen def _stop_perf_mon(self, perf_mon_pid): 111831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen """Stops monitoring performance and resource usage on a DUT. 111931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 112031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen @param perf_mon_pid: the PID returned from _start_perf_mon(). 112131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 1122f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @return Dictionary containing performance attributes, or None if 1123f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold unavailable. 112431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen """ 112531887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Gracefully handle problems with performance monitoring by 112631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # just returning None. 112731887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen try: 112831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen cmd = ('python /tmp/update_engine_performance_monitor.py ' 112931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen '--stop-bg=%d') % perf_mon_pid 113031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen perf_json_txt = self._host.run(cmd).stdout 113131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen return json.loads(perf_json_txt) 113231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen except Exception as e: 113331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen logging.warning('Failed to parse output from ' 113431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 'update_engine_performance_monitor.py: %s', e) 113531887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen return None 113631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 113731887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 1138f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Interface overrides. 1139f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # 1140f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def initialize(self, autotest_devserver, devserver_dir): 1141f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._autotest_devserver = autotest_devserver 1142f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._devserver_dir = devserver_dir 1143f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1144f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1145f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def reboot_device(self): 1146f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._host.reboot() 1147f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1148f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1149f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def prep_artifacts(self, test_conf): 1150f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._staged_urls = self._stage_artifacts_onto_devserver(test_conf) 1151f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return self._staged_urls 1152f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1153f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1154f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def prep_device_for_update(self, source_release): 1155f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Install the source version onto the DUT. 1156f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold if self._staged_urls.source_url: 11579b976d77f77c94733ab711c15db4e08528926d46Gilad Arnold logging.info('Installing a source image on the DUT') 1158f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver_hostname = urlparse.urlparse( 1159f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._autotest_devserver.url()).hostname 1160f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._install_source_version(devserver_hostname, 1161f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._staged_urls.source_url, 1162f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._staged_urls.source_stateful_url) 1163f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1164f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Make sure we can login before the update. 1165f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._run_login_test(source_release) 1166f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1167f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1168f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def get_active_slot(self): 1169f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return self._host.run('rootdev -s').stdout.strip() 1170f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1171f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1172f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def start_update_perf(self, bindir): 1173f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold if self._perf_mon_pid is None: 1174f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._perf_mon_pid = self._start_perf_mon(bindir) 1175f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1176f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1177f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def stop_update_perf(self): 1178f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold perf_data = None 1179f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold if self._perf_mon_pid is not None: 1180f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold perf_data = self._stop_perf_mon(self._perf_mon_pid) 1181f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._perf_mon_pid = None 1182f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1183f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return perf_data 1184f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1185f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1186f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def trigger_update(self, omaha_devserver): 1187f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold updater = autoupdater.ChromiumOSUpdater( 1188f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold omaha_devserver.get_update_url(), host=self._host) 1189f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold updater.trigger_update() 1190f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1191f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1192f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def finalize_update(self): 1193f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._update_via_test_payloads( 119494694ec92c9e2c4d316aed34453ef75c21444c61Gilad Arnold None, None, self._staged_urls.target_stateful_url, False) 1195f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1196f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1197f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def get_update_log(self, num_lines): 1198f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold return self._host.run_output( 1199fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold 'tail -n %d /var/log/update_engine.log' % num_lines, 1200fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold stdout_tee=None) 1201f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1202f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1203f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def check_device_after_update(self, target_release): 1204f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Make sure we can login after update. 1205f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._run_login_test(target_release) 1206f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1207f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 120834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar def oobe_triggers_update(self): 120934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar return self._host.oobe_triggers_update() 121034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 121134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 1212f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnoldclass autoupdate_EndToEndTest(test.test): 1213f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Complete update test between two Chrome OS releases. 1214f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1215f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold Performs an end-to-end test of updating a ChromeOS device from one version 1216f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold to another. The test performs the following steps: 1217f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1218f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1. Stages the source (full) and target update payload on the central 1219f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver. 1220f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 2. Spawns a private Omaha-like devserver instance, configured to return 1221f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold the target (update) payload URL in response for an update check. 1222f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 3. Reboots the DUT. 1223f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 4. Installs a source image on the DUT (if provided) and reboots to it. 1224f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 5. Triggers an update check at the DUT. 1225f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 6. Watches as the DUT obtains an update and applies it. 1226f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 7. Reboots and repeats steps 5-6, ensuring that the next update check 1227f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold shows the new image version. 1228f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1229f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold Some notes on naming: 1230f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver: Refers to a machine running the Chrome OS Update Devserver. 1231f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold autotest_devserver: An autotest wrapper to interact with a devserver. 1232f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold Can be used to stage artifacts to a devserver. While 1233f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold this can also be used to update a machine, we do not 1234f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold use it for that purpose in this test as we manage 1235f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold updates with out own devserver instances (see below). 1236f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold omaha_devserver: This test's wrapper of a devserver running for the 1237f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold purposes of emulating omaha. This test controls the 1238f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold lifetime of this devserver instance and is separate 1239f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold from the autotest lab's devserver's instances which are 1240f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold only used for staging and hosting artifacts (because they 1241f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold scale). These are run on the same machines as the actual 1242f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold autotest devservers which are used for staging but on 1243f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold different ports. 1244f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold *staged_url's: In this case staged refers to the fact that these items 1245f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold are available to be downloaded statically from these urls 1246f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold e.g. 'localhost:8080/static/my_file.gz'. These are usually 1247f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold given after staging an artifact using a autotest_devserver 1248f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold though they can be re-created given enough assumptions. 1249f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold *update_url's: Urls refering to the update RPC on a given omaha devserver. 1250f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold Since we always use an instantiated omaha devserver to run 1251f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold updates, these will always reference an existing instance 1252f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold of an omaha devserver that we just created for the purposes 1253f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold of updating. 1254f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver_hostname: At the start of each test, we choose a devserver 1255f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold machine in the lab for the test. We use the devserver 1256f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold instance there (access by autotest_devserver) to stage 1257f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold artifacts. However, we also use the same host to start 1258f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold omaha devserver instances for updating machines with 1259f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold (that reference the staged paylaods on the autotest 1260f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver instance). This hostname refers to that 1261f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold machine we are using (since it's always the same for 1262f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold both staging/omaha'ing). 1263f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1264f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """ 1265f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold version = 1 1266f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1267f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Timeout periods, given in seconds. 1268f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_AFTER_SHUTDOWN_SECONDS = 10 1269f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_AFTER_UPDATE_SECONDS = 20 1270f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_FOR_USB_INSTALL_SECONDS = 4 * 60 1271f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_FOR_MP_RECOVERY_SECONDS = 8 * 60 1272f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_FOR_INITIAL_UPDATE_CHECK_SECONDS = 12 * 60 1273f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # TODO(sosa): Investigate why this needs to be so long (this used to be 1274f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # 120 and regressed). 1275f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_FOR_DOWNLOAD_STARTED_SECONDS = 4 * 60 1276f3c5dcd63105ae2f98a9dbb4da1cdbeacd49e832Grant Grundler # See https://crbug.com/731214 before changing WAIT_FOR_DOWNLOAD 1277f3c5dcd63105ae2f98a9dbb4da1cdbeacd49e832Grant Grundler _WAIT_FOR_DOWNLOAD_COMPLETED_SECONDS = 20 * 60 1278f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_FOR_UPDATE_COMPLETED_SECONDS = 4 * 60 1279f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _WAIT_FOR_UPDATE_CHECK_AFTER_REBOOT_SECONDS = 15 * 60 1280f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold _DEVSERVER_HOSTLOG_REQUEST_TIMEOUT_SECONDS = 30 1281f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1282a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold # Logs and their whereabouts. 1283a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold _WHERE_UPDATE_LOG = ('update_engine log (in sysinfo or on the DUT, also ' 1284a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'included in the test log)') 1285a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold _WHERE_OMAHA_LOG = 'Omaha-devserver log (included in the test log)' 1286a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1287f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1288f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def initialize(self): 1289f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Sets up variables that will be used by test.""" 1290f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._host = None 1291f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._omaha_devserver = None 1292a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._source_image_installed = False 1293f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1294f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._devserver_dir = global_config.global_config.get_config_value( 1295f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 'CROS', 'devserver_dir', default=None) 1296f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold if self._devserver_dir is None: 1297f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold raise error.TestError( 1298f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 'Path to devserver source tree not provided; please define ' 1299f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 'devserver_dir under [CROS] in your shadow_config.ini') 1300f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1301f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1302f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def cleanup(self): 1303f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Kill the omaha devserver if it's still around.""" 1304f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold if self._omaha_devserver: 1305f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._omaha_devserver.stop_devserver() 1306f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1307f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._omaha_devserver = None 1308f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1309f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1310f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def _dump_update_engine_log(self, test_platform): 1311f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold """Dumps relevant AU error log.""" 1312f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold try: 1313fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold error_log = test_platform.get_update_log(80) 1314fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold logging.error('Dumping snippet of update_engine log:\n%s', 1315fffb6d7274294d9a57523fb7535dd627d0378e5fGilad Arnold snippet(error_log)) 1316f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold except Exception: 1317f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Mute any exceptions we get printing debug logs. 1318f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold pass 1319f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1320f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 132131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen def _report_perf_data(self, perf_data): 132231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen """Reports performance and resource data. 132331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 1324f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold Currently, performance attributes are expected to include 'rss_peak' 1325f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold (peak memory usage in bytes). 1326f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1327f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param perf_data: A dictionary containing performance attributes. 132831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen """ 132931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen rss_peak = perf_data.get('rss_peak') 133031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen if rss_peak: 133131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen rss_peak_kib = rss_peak / 1024 133231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen logging.info('Peak memory (RSS) usage on DUT: %d KiB', rss_peak_kib) 133331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen self.output_perf_value(description='mem_usage_peak', 133431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen value=int(rss_peak_kib), 133531887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen units='KiB', 133631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen higher_is_better=False) 133731887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen else: 133831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen logging.warning('No rss_peak key in JSON returned by ' 133931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 'update_engine_performance_monitor.py') 134031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 134131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 1342a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold def _error_initial_check(self, expected, actual, mismatched_attrs): 1343a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'version' in mismatched_attrs: 1344a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold err_msg = ('Initial update check was received but the reported ' 1345a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'version is different from what was expected.') 1346a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if self._source_image_installed: 1347a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold err_msg += (' The source payload we installed was probably ' 1348a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'incorrect or corrupt.') 1349a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold else: 1350a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold err_msg += (' The DUT is probably not running the correct ' 1351a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'source image.') 1352a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return err_msg 1353a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1354a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return 'A test bug occurred; inspect the test log.' 1355a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1356a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1357a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold def _error_intermediate(self, expected, actual, mismatched_attrs, action, 1358a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold problem): 1359a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'event_result' in mismatched_attrs: 13600ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold event_result = actual.get('event_result') 13610ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold reported = (('different than expected (%s)' % 13620ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold EVENT_RESULT_DICT[event_result]) 13630ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold if event_result else 'missing') 13640ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold return ('The updater reported result code is %s. This could be an ' 13650ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'updater bug or a connectivity problem; check the %s. For ' 13660ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'a detailed log of update events, check the %s.' % 13670ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold (reported, self._WHERE_UPDATE_LOG, self._WHERE_OMAHA_LOG)) 1368a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'event_type' in mismatched_attrs: 13690ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold event_type = actual.get('event_type') 13700ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold reported = ('different (%s)' % EVENT_TYPE_DICT[event_type] 13710ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold if event_type else 'missing') 13720ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold return ('Expected the updater to %s (%s) but received event type ' 13730ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'is %s. This could be an updater %s; check the ' 1374a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold '%s. For a detailed log of update events, check the %s.' % 13750ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold (action, EVENT_TYPE_DICT[expected['event_type']], reported, 13760ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold problem, self._WHERE_UPDATE_LOG, self._WHERE_OMAHA_LOG)) 1377a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'version' in mismatched_attrs: 1378a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return ('The updater reported an unexpected version despite ' 1379a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'previously reporting the correct one. This is most likely ' 1380a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'a bug in update engine; check the %s.' % 1381a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WHERE_UPDATE_LOG) 1382a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1383a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return 'A test bug occurred; inspect the test log.' 1384a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1385a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1386a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold def _error_download_started(self, expected, actual, mismatched_attrs): 1387a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return self._error_intermediate(expected, actual, mismatched_attrs, 1388a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'begin downloading', 1389a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'bug, crash or provisioning error') 1390a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1391a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1392a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold def _error_download_finished(self, expected, actual, mismatched_attrs): 1393a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return self._error_intermediate(expected, actual, mismatched_attrs, 1394a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'finish downloading', 'bug or crash') 1395a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1396a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1397a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold def _error_update_complete(self, expected, actual, mismatched_attrs): 1398a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return self._error_intermediate(expected, actual, mismatched_attrs, 1399a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'complete the update', 'bug or crash') 1400a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1401a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 140216e76893c34739649a751f42b3881b57620d665cGilad Arnold def _error_reboot_after_update(self, expected, actual, mismatched_attrs): 1403a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'event_result' in mismatched_attrs: 14040ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold event_result = actual.get('event_result') 14050ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold reported = ('different (%s)' % EVENT_RESULT_DICT[event_result] 14060ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold if event_result else 'missing') 1407a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return ('The updater was expected to reboot (%s) but reported ' 14080ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'result code is %s. This could be a failure to reboot, an ' 14090ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'updater bug or a connectivity problem; check the %s and ' 14100ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'the system log. For a detailed log of update events, ' 14110ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'check the %s.' % 14120ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold (EVENT_RESULT_DICT[expected['event_result']], reported, 1413a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WHERE_UPDATE_LOG, self._WHERE_OMAHA_LOG)) 1414a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'event_type' in mismatched_attrs: 14150ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold event_type = actual.get('event_type') 14160ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold reported = ('different (%s)' % EVENT_TYPE_DICT[event_type] 14170ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold if event_type else 'missing') 1418a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return ('Expected to successfully reboot into the new image (%s) ' 14190ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'but received event type is %s. This probably means that ' 14200ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'the new image failed to verify after reboot, possibly ' 14210ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'because the payload is corrupt. This might also be an ' 14220ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'updater bug or crash; check the %s. For a detailed log of ' 14230ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold 'update events, check the %s.' % 14240ba4fcd882643427524c35522bf77959b57c6f48Gilad Arnold (EVENT_TYPE_DICT[expected['event_type']], reported, 1425a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WHERE_UPDATE_LOG, self._WHERE_OMAHA_LOG)) 1426a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'version' in mismatched_attrs: 1427a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return ('The DUT rebooted after the update but reports a different ' 1428a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'image version than the one expected. This probably means ' 1429a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'that the payload we applied was incorrect or corrupt.') 1430a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if 'previous_version' in mismatched_attrs: 1431a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return ('The DUT rebooted after the update and reports the ' 1432a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'expected version. However, it reports a previous version ' 1433a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'that is different from the one previously reported. This ' 1434a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'is most likely a bug in update engine; check the %s.' % 1435a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WHERE_UPDATE_LOG) 1436a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1437a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return 'A test bug occurred; inspect the test log.' 1438a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1439a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1440a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold def _timeout_err(self, desc, timeout, event_type=None): 1441a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold if event_type is not None: 1442a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold desc += ' (%s)' % EVENT_TYPE_DICT[event_type] 1443a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold return ('Failed to receive %s within %d seconds. This could be a ' 1444a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'problem with the updater or a connectivity issue. For more ' 1445a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'details, check the %s.' % 1446a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold (desc, timeout, self._WHERE_UPDATE_LOG)) 1447a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1448a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 1449f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold def run_update_test(self, test_platform, test_conf): 1450ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa """Runs the actual update test once preconditions are met. 14510ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1452f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold @param test_platform: TestPlatform implementation. 1453ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @param test_conf: A dictionary containing test configuration values 14540ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1455ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @raises ExpectedUpdateEventChainFailed if we failed to verify an update 1456ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa event. 14570ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold """ 1458c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen 1459f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Record the active root partition. 1460f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold source_active_slot = test_platform.get_active_slot() 1461f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold logging.info('Source active slot: %s', source_active_slot) 14620ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 14637ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold source_release = test_conf['source_release'] 14647ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold target_release = test_conf['target_release'] 14657ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold 146631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Start the performance monitoring process on the DUT. 1467f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.start_update_perf(self.bindir) 146831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen try: 146931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Trigger an update. 1470f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.trigger_update(self._omaha_devserver) 14710ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 147231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Track update progress. 147331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen omaha_netloc = self._omaha_devserver.get_netloc() 147431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen omaha_hostlog_url = urlparse.urlunsplit( 147531887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen ['http', omaha_netloc, '/api/hostlog', 147631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 'ip=' + self._host.ip, '']) 147731887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen logging.info('Polling update progress from omaha/devserver: %s', 147831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen omaha_hostlog_url) 147931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen log_verifier = UpdateEventLogVerifier( 148031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen omaha_hostlog_url, 148131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen self._DEVSERVER_HOSTLOG_REQUEST_TIMEOUT_SECONDS) 148231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 148331887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Verify chain of events in a successful update process. 1484248108c1c925791d926105f42212b1033213a4dcGilad Arnold chain = ExpectedUpdateEventChain() 1485248108c1c925791d926105f42212b1033213a4dcGilad Arnold chain.add_event( 1486248108c1c925791d926105f42212b1033213a4dcGilad Arnold ExpectedUpdateEvent( 14877ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold version=source_release, 1488a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_error=self._error_initial_check), 1489a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_INITIAL_UPDATE_CHECK_SECONDS, 1490a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_timeout=self._timeout_err( 1491a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'an initial update check', 1492a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_INITIAL_UPDATE_CHECK_SECONDS)) 1493248108c1c925791d926105f42212b1033213a4dcGilad Arnold chain.add_event( 1494248108c1c925791d926105f42212b1033213a4dcGilad Arnold ExpectedUpdateEvent( 1495248108c1c925791d926105f42212b1033213a4dcGilad Arnold event_type=EVENT_TYPE_DOWNLOAD_STARTED, 1496248108c1c925791d926105f42212b1033213a4dcGilad Arnold event_result=EVENT_RESULT_SUCCESS, 14977ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold version=source_release, 1498a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_error=self._error_download_started), 1499a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_DOWNLOAD_STARTED_SECONDS, 1500a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_timeout=self._timeout_err( 1501a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'a download started notification', 1502a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_DOWNLOAD_STARTED_SECONDS, 1503a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold event_type=EVENT_TYPE_DOWNLOAD_STARTED)) 1504248108c1c925791d926105f42212b1033213a4dcGilad Arnold chain.add_event( 1505248108c1c925791d926105f42212b1033213a4dcGilad Arnold ExpectedUpdateEvent( 1506248108c1c925791d926105f42212b1033213a4dcGilad Arnold event_type=EVENT_TYPE_DOWNLOAD_FINISHED, 1507248108c1c925791d926105f42212b1033213a4dcGilad Arnold event_result=EVENT_RESULT_SUCCESS, 15087ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold version=source_release, 1509a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_error=self._error_download_finished), 1510a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_DOWNLOAD_COMPLETED_SECONDS, 1511a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_timeout=self._timeout_err( 1512a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'a download finished notification', 1513a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_DOWNLOAD_COMPLETED_SECONDS, 1514a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold event_type=EVENT_TYPE_DOWNLOAD_FINISHED)) 1515248108c1c925791d926105f42212b1033213a4dcGilad Arnold chain.add_event( 1516248108c1c925791d926105f42212b1033213a4dcGilad Arnold ExpectedUpdateEvent( 1517248108c1c925791d926105f42212b1033213a4dcGilad Arnold event_type=EVENT_TYPE_UPDATE_COMPLETE, 1518248108c1c925791d926105f42212b1033213a4dcGilad Arnold event_result=EVENT_RESULT_SUCCESS, 15197ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold version=source_release, 1520a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_error=self._error_update_complete), 1521a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_UPDATE_COMPLETED_SECONDS, 1522a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold on_timeout=self._timeout_err( 1523a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold 'an update complete notification', 1524a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._WAIT_FOR_UPDATE_COMPLETED_SECONDS, 1525a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold event_type=EVENT_TYPE_UPDATE_COMPLETE)) 152631887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 1527fac9a5d238337da91939328386c2fa4fdfb6d957Alex Deymo log_verifier.verify_expected_events_chain(chain) 152831887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen 152931887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Wait after an update completion (safety margin). 153031887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen _wait(self._WAIT_AFTER_UPDATE_SECONDS, 'after update completion') 153131887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen finally: 153231887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen # Terminate perf monitoring process and collect its output. 1533f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold perf_data = test_platform.stop_update_perf() 153431887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen if perf_data: 153531887b7a4ea7e1517d468c2ccf94df14ba6d0315David Zeuthen self._report_perf_data(perf_data) 1536f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 15374cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold # Only update the stateful partition (the test updated the rootfs). 1538f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.finalize_update() 15394cec366fff66e4603d9851c76680a6bb3f10fbe6Gilad Arnold 1540f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa # Reboot the DUT after the update. 1541bacc35bdb2fb44b0a760c05d364854cd92fb3f98Gilad Arnold test_platform.reboot_device() 15420ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1543f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa # Trigger a second update check (again, test vs MP). 1544f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.trigger_update(self._omaha_devserver) 15450ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 154634618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar if test_platform.oobe_triggers_update(): 154734618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # If DUT automatically checks for update during OOBE, 154834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # checking the post-update CrOS version and slot is sufficient. 154934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # This command checks the OS version. 155034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # The slot is checked a little later, after the else block. 155134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar test_platform.verify_version(target_release) 155234618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar else: 155334618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # Observe post-reboot update check, which should indicate that the 155434618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # image version has been updated. 155534618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar chain = ExpectedUpdateEventChain() 155634618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar expected_events = [ 155734618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar ExpectedUpdateEvent( 155834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar event_type=EVENT_TYPE_UPDATE_COMPLETE, 155934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar event_result=EVENT_RESULT_SUCCESS_REBOOT, 156034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar version=target_release, 156134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar previous_version=source_release, 156234618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar on_error=self._error_reboot_after_update), 156334618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # Newer versions send a "rebooted_after_update" message 156434618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # after reboot with the previous version instead of another 156534618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar # "update_complete". 156634618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar ExpectedUpdateEvent( 156734618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar event_type=EVENT_TYPE_REBOOTED_AFTER_UPDATE, 156834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar event_result=EVENT_RESULT_SUCCESS, 156934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar version=target_release, 157034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar previous_version=source_release, 157134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar on_error=self._error_reboot_after_update), 157234618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar ] 157334618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar chain.add_event( 157434618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar expected_events, 157534618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar self._WAIT_FOR_UPDATE_CHECK_AFTER_REBOOT_SECONDS, 157634618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar on_timeout=self._timeout_err( 157734618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 'a successful reboot notification', 157834618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar self._WAIT_FOR_UPDATE_CHECK_AFTER_REBOOT_SECONDS, 157934618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar event_type=EVENT_TYPE_UPDATE_COMPLETE)) 158034618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar 158134618870e735e83db984401a54ca1eecd8e4ac14Niranjan Kumar log_verifier.verify_expected_events_chain(chain) 1582f4fa49b09cfa64e039355f78df2c34a9acca0c0cChris Sosa 1583f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Make sure we're using a different slot after the update. 1584f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold target_active_slot = test_platform.get_active_slot() 1585f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold if target_active_slot == source_active_slot: 15861b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold err_msg = 'The active image slot did not change after the update.' 15877ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold if None in (source_release, target_release): 15887ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold err_msg += (' The DUT likely rebooted into the old image, which ' 15897ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold 'probably means that the payload we applied was ' 15907ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold 'corrupt. But since we did not check the source ' 15917ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold 'and/or target version we cannot say for sure.') 15927ac0a621527b1ecb4b82fea910c730682f9c1f76Gilad Arnold elif source_release == target_release: 15931b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold err_msg += (' Given that the source and target versions are ' 15941b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'identical, the DUT likely rebooted into the old ' 15951b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'image. This probably means that the payload we ' 15961b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'applied was corrupt.') 15971b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold else: 15981b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold err_msg += (' This is strange since the DUT reported the ' 15991b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'correct target version. This is probably a system ' 16001b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold 'bug; check the DUT system log.') 16011b4da911c86c446bef8d0d5cb63dcf605ce6d289Gilad Arnold raise error.TestFail(err_msg) 16020338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold 1603f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold logging.info('Target active slot changed as expected: %s', 1604f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold target_active_slot) 1605eb300ac8af429e51f751b95ce375636fbb649e37Gilad Arnold 16060338ff3b754d52b41ca2d0432164a757daa1112dGilad Arnold logging.info('Update successful, test completed') 16070ed760cbee4ce839988585003d445465bb0b95b9Gilad Arnold 1608ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 16098e00e8b4a421c53a15f751fc90bb06d738ccbd78David Haddock def run_once(self, host, test_conf): 1610ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa """Performs a complete auto update test. 1611ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1612ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @param host: a host object representing the DUT 1613ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @param test_conf: a dictionary containing test configuration values 1614ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1615ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa @raise error.TestError if anything went wrong with setting up the test; 1616ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa error.TestFail if any part of the test has failed. 1617ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1618ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa """ 1619f789cf3a52c720344062f0a6c782bb758f08b189Don Garrett 1620ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa self._host = host 1621ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1622009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi # Find a devserver to use. We first try to pick a devserver with the 1623009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi # least load. In case all devservers' load are higher than threshold, 1624009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi # fall back to the old behavior by picking a devserver based on the 1625009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi # payload URI, with which ImageServer.resolve will return a random 1626009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi # devserver based on the hash of the URI. 162701b28a630230336c3277dd7b9d375ee012931b77Dan Shi # The picked devserver needs to respect the location of the host if 162801b28a630230336c3277dd7b9d375ee012931b77Dan Shi # `prefer_local_devserver` is set to True or `restricted_subnets` is 162901b28a630230336c3277dd7b9d375ee012931b77Dan Shi # set. 163001b28a630230336c3277dd7b9d375ee012931b77Dan Shi hostname = self._host.hostname if self._host else None 163101b28a630230336c3277dd7b9d375ee012931b77Dan Shi least_loaded_devserver = dev_server.get_least_loaded_devserver( 163201b28a630230336c3277dd7b9d375ee012931b77Dan Shi hostname=hostname) 16334e2153dd994a088d92e2a4c1cbee9043e08f9fe7Dan Shi if least_loaded_devserver: 16344e2153dd994a088d92e2a4c1cbee9043e08f9fe7Dan Shi logging.debug('Choose the least loaded devserver: %s', 16354e2153dd994a088d92e2a4c1cbee9043e08f9fe7Dan Shi least_loaded_devserver) 16364e2153dd994a088d92e2a4c1cbee9043e08f9fe7Dan Shi autotest_devserver = dev_server.ImageServer(least_loaded_devserver) 16374e2153dd994a088d92e2a4c1cbee9043e08f9fe7Dan Shi else: 1638009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi logging.warning('No devserver meets the maximum load requirement. ' 1639009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi 'Pick a random devserver to use.') 1640009ab000fbeeec0e0be9a2d6e744107a2b82306bDan Shi autotest_devserver = dev_server.ImageServer.resolve( 164169b9b975fccab1ede1e8849ed17e7e655a9d6b48Dan Shi test_conf['target_payload_uri'], host.hostname) 1642ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa devserver_hostname = urlparse.urlparse( 1643ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa autotest_devserver.url()).hostname 1644ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1645f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Obtain a test platform implementation. 1646f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform = TestPlatform.create(host) 1647f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.initialize(autotest_devserver, self._devserver_dir) 1648f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold 1649ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa # Stage source images and update payloads onto a devserver. 1650f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold staged_urls = test_platform.prep_artifacts(test_conf) 1651a0ca5707ed10a6575ed290f341294331455f7769Gilad Arnold self._source_image_installed = bool(staged_urls.source_url) 1652ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1653f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold # Prepare the DUT (install source version etc). 1654f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.prep_device_for_update(test_conf['source_release']) 1655ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa 1656ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa self._omaha_devserver = OmahaDevserver( 1657f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold devserver_hostname, self._devserver_dir, 1658f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold staged_urls.target_url) 1659ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa self._omaha_devserver.start_devserver() 1660c6ed2f633a1c4609a4b212da66b3e506008ee3a7Nam T. Nguyen 1661ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa try: 1662f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self.run_update_test(test_platform, test_conf) 1663ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa except ExpectedUpdateEventChainFailed: 1664f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold self._dump_update_engine_log(test_platform) 1665ed8c1858bc32b066e4fbe4100a9ad49bdf1adc61Chris Sosa raise 1666f789cf3a52c720344062f0a6c782bb758f08b189Don Garrett 1667f1d1104c736f79145209c76a147c1334f8dbb6a9Gilad Arnold test_platform.check_device_after_update(test_conf['target_release']) 1668