dev_server_unittest.py revision bf59e756d583ad90a487ab53539c41040000ae40 
1#!/usr/bin/python
2#
3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Unit tests for client/common_lib/cros/dev_server.py."""
8
9import __builtin__
10
11import httplib
12import json
13import mox
14import os
15import StringIO
16import time
17import unittest
18import urllib2
19
20import common
21from autotest_lib.client.bin import utils as bin_utils
22from autotest_lib.client.common_lib import android_utils
23from autotest_lib.client.common_lib import error
24from autotest_lib.client.common_lib import global_config
25from autotest_lib.client.common_lib import utils
26from autotest_lib.client.common_lib.cros import dev_server
27from autotest_lib.client.common_lib.cros import retry
28
29
30def retry_mock(ExceptionToCheck, timeout_min, exception_to_raise=None,
31               label=None):
32    """A mock retry decorator to use in place of the actual one for testing.
33
34    @param ExceptionToCheck: the exception to check.
35    @param timeout_mins: Amount of time in mins to wait before timing out.
36    @param exception_to_raise: the exception to raise in retry.retry
37    @param label: used in debug messages
38
39    """
40    def inner_retry(func):
41        """The actual decorator.
42
43        @param func: Function to be called in decorator.
44
45        """
46        return func
47
48    return inner_retry
49
50
51class MockSshResponse(object):
52    """An ssh response mocked for testing."""
53
54    def __init__(self, output, exit_status=0):
55        self.stdout = output
56        self.exit_status = exit_status
57        self.stderr = 'SSH connection error occurred.'
58
59
60class MockSshError(error.CmdError):
61    """An ssh error response mocked for testing."""
62
63    def __init__(self):
64        self.result_obj = MockSshResponse('error', exit_status=255)
65
66
67E403 = urllib2.HTTPError(url='',
68                         code=httplib.FORBIDDEN,
69                         msg='Error 403',
70                         hdrs=None,
71                         fp=StringIO.StringIO('Expected.'))
72E500 = urllib2.HTTPError(url='',
73                         code=httplib.INTERNAL_SERVER_ERROR,
74                         msg='Error 500',
75                         hdrs=None,
76                         fp=StringIO.StringIO('Expected.'))
77CMD_ERROR = error.CmdError('error_cmd', MockSshError().result_obj)
78
79
80class RunCallTest(mox.MoxTestBase):
81    """Unit tests for ImageServerBase.run_call or DevServer.run_call."""
82
83    def setUp(self):
84        """Set up the test"""
85        self.test_call = 'http://nothing/test'
86        self.hostname = 'nothing'
87        self.contents = 'true'
88        self.contents_readline = ['file/one', 'file/two']
89        self.save_ssh_config = dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER
90        super(RunCallTest, self).setUp()
91        self.mox.StubOutWithMock(urllib2, 'urlopen')
92        self.mox.StubOutWithMock(utils, 'run')
93
94
95    def tearDown(self):
96        """Tear down the test"""
97        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = self.save_ssh_config
98        super(RunCallTest, self).tearDown()
99
100
101    def testRunCallWithSingleCallHTTP(self):
102        """Test dev_server.ImageServerBase.run_call using http with arg:
103        (call)."""
104        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = False
105
106        urllib2.urlopen(mox.StrContains(self.test_call)).AndReturn(
107                StringIO.StringIO(self.contents))
108        self.mox.ReplayAll()
109        response = dev_server.ImageServerBase.run_call(self.test_call)
110        self.assertEquals(self.contents, response)
111
112
113    def testRunCallWithCallAndReadlineHTTP(self):
114        """Test dev_server.ImageServerBase.run_call using http with arg:
115        (call, readline=True)."""
116        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = False
117
118        urllib2.urlopen(mox.StrContains(self.test_call)).AndReturn(
119                StringIO.StringIO('\n'.join(self.contents_readline)))
120        self.mox.ReplayAll()
121        response = dev_server.ImageServerBase.run_call(
122                self.test_call, readline=True)
123        self.assertEquals(self.contents_readline, response)
124
125
126    def testRunCallWithCallAndTimeoutHTTP(self):
127        """Test dev_server.ImageServerBase.run_call using http with args:
128        (call, timeout=xxx)."""
129        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = False
130
131        urllib2.urlopen(mox.StrContains(self.test_call), data=None).AndReturn(
132                StringIO.StringIO(self.contents))
133        self.mox.ReplayAll()
134        response = dev_server.ImageServerBase.run_call(
135                self.test_call, timeout=60)
136        self.assertEquals(self.contents, response)
137
138
139    def testRunCallWithSingleCallSSH(self):
140        """Test dev_server.ImageServerBase.run_call using ssh with arg:
141        (call)."""
142        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = True
143        self.mox.StubOutWithMock(utils, 'get_restricted_subnet')
144        utils.get_restricted_subnet(
145                self.hostname, utils.RESTRICTED_SUBNETS).AndReturn(
146                self.hostname)
147
148        to_return = MockSshResponse(self.contents)
149        utils.run(mox.StrContains(self.test_call),
150                  timeout=mox.IgnoreArg()).AndReturn(to_return)
151        self.mox.ReplayAll()
152        response = dev_server.ImageServerBase.run_call(self.test_call)
153        self.assertEquals(self.contents, response)
154
155
156    def testRunCallWithCallAndReadlineSSH(self):
157        """Test dev_server.ImageServerBase.run_call using ssh with args:
158        (call, readline=True)."""
159        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = True
160        self.mox.StubOutWithMock(utils, 'get_restricted_subnet')
161        utils.get_restricted_subnet(
162                self.hostname, utils.RESTRICTED_SUBNETS).AndReturn(
163                self.hostname)
164
165        to_return = MockSshResponse('\n'.join(self.contents_readline))
166        utils.run(mox.StrContains(self.test_call),
167                  timeout=mox.IgnoreArg()).AndReturn(to_return)
168        self.mox.ReplayAll()
169        response = dev_server.ImageServerBase.run_call(
170                self.test_call, readline=True)
171        self.assertEquals(self.contents_readline, response)
172
173
174    def testRunCallWithCallAndTimeoutSSH(self):
175        """Test dev_server.ImageServerBase.run_call using ssh with args:
176        (call, timeout=xxx)."""
177        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = True
178        self.mox.StubOutWithMock(utils, 'get_restricted_subnet')
179        utils.get_restricted_subnet(
180                self.hostname, utils.RESTRICTED_SUBNETS).AndReturn(
181                self.hostname)
182
183        to_return = MockSshResponse(self.contents)
184        utils.run(mox.StrContains(self.test_call),
185                  timeout=mox.IgnoreArg()).AndReturn(to_return)
186        self.mox.ReplayAll()
187        response = dev_server.ImageServerBase.run_call(
188                self.test_call, timeout=60)
189        self.assertEquals(self.contents, response)
190
191
192    def testRunCallWithExceptionHTTP(self):
193        """Test dev_server.ImageServerBase.run_call using http with raising
194        exception."""
195        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = False
196        urllib2.urlopen(mox.StrContains(self.test_call)).AndRaise(E500)
197        self.mox.ReplayAll()
198        self.assertRaises(urllib2.HTTPError,
199                          dev_server.ImageServerBase.run_call,
200                          self.test_call)
201
202
203    def testRunCallWithExceptionSSH(self):
204        """Test dev_server.ImageServerBase.run_call using ssh with raising
205        exception."""
206        dev_server.ENABLE_SSH_CONNECTION_FOR_DEVSERVER = True
207        self.mox.StubOutWithMock(utils, 'get_restricted_subnet')
208        utils.get_restricted_subnet(
209                self.hostname, utils.RESTRICTED_SUBNETS).AndReturn(
210                self.hostname)
211
212        utils.run(mox.StrContains(self.test_call),
213                  timeout=mox.IgnoreArg()).AndRaise(MockSshError())
214        self.mox.ReplayAll()
215        self.assertRaises(error.CmdError,
216                          dev_server.ImageServerBase.run_call,
217                          self.test_call)
218
219
220    def testRunCallByDevServerHTTP(self):
221        """Test dev_server.DevServer.run_call, which uses http, and can be
222        directly called by CrashServer."""
223        urllib2.urlopen(
224                mox.StrContains(self.test_call), data=None).AndReturn(
225                        StringIO.StringIO(self.contents))
226        self.mox.ReplayAll()
227        response = dev_server.DevServer.run_call(
228               self.test_call, timeout=60)
229        self.assertEquals(self.contents, response)
230
231
232class DevServerTest(mox.MoxTestBase):
233    """Unit tests for dev_server.DevServer.
234
235    @var _HOST: fake dev server host address.
236    """
237
238    _HOST = 'http://nothing'
239    _CRASH_HOST = 'http://nothing-crashed'
240    _CONFIG = global_config.global_config
241
242
243    def setUp(self):
244        """Set up the test"""
245        super(DevServerTest, self).setUp()
246        self.crash_server = dev_server.CrashServer(DevServerTest._CRASH_HOST)
247        self.dev_server = dev_server.ImageServer(DevServerTest._HOST)
248        self.android_dev_server = dev_server.AndroidBuildServer(
249                DevServerTest._HOST)
250        self.mox.StubOutWithMock(dev_server.ImageServerBase, 'run_call')
251        self.mox.StubOutWithMock(urllib2, 'urlopen')
252        self.mox.StubOutWithMock(utils, 'run')
253        self.mox.StubOutWithMock(os.path, 'exists')
254        # Hide local restricted_subnets setting.
255        dev_server.RESTRICTED_SUBNETS = []
256
257
258    def testSimpleResolve(self):
259        """One devserver, verify we resolve to it."""
260        self.mox.StubOutWithMock(dev_server, '_get_dev_server_list')
261        self.mox.StubOutWithMock(dev_server.ImageServer, 'devserver_healthy')
262        dev_server._get_dev_server_list().MultipleTimes().AndReturn(
263                [DevServerTest._HOST])
264        dev_server.ImageServer.devserver_healthy(DevServerTest._HOST).AndReturn(
265                                                                        True)
266        self.mox.ReplayAll()
267        devserver = dev_server.ImageServer.resolve('my_build')
268        self.assertEquals(devserver.url(), DevServerTest._HOST)
269
270
271    def testResolveWithFailure(self):
272        """Ensure we rehash on a failed ping on a bad_host."""
273        self.mox.StubOutWithMock(dev_server, '_get_dev_server_list')
274        bad_host, good_host = 'http://bad_host:99', 'http://good_host:8080'
275        dev_server._get_dev_server_list().MultipleTimes().AndReturn(
276                [bad_host, good_host])
277        argument1 = mox.StrContains(bad_host)
278        argument2 = mox.StrContains(good_host)
279
280        # Mock out bad ping failure to bad_host by raising devserver exception.
281        dev_server.ImageServerBase.run_call(
282                argument1, timeout=mox.IgnoreArg()).AndRaise(
283                        dev_server.DevServerException())
284        # Good host is good.
285        dev_server.ImageServerBase.run_call(
286                argument2, timeout=mox.IgnoreArg()).AndReturn(
287                        '{"free_disk": 1024}')
288
289        self.mox.ReplayAll()
290        host = dev_server.ImageServer.resolve(0) # Using 0 as it'll hash to 0.
291        self.assertEquals(host.url(), good_host)
292        self.mox.VerifyAll()
293
294
295    def testResolveWithFailureURLError(self):
296        """Ensure we rehash on a failed ping using http on a bad_host after
297        urlerror."""
298        # Set retry.retry to retry_mock for just returning the original
299        # method for this test. This is to save waiting time for real retry,
300        # which is defined by dev_server.DEVSERVER_SSH_TIMEOUT_MINS.
301        # Will reset retry.retry to real retry at the end of this test.
302        real_retry = retry.retry
303        retry.retry = retry_mock
304
305        self.mox.StubOutWithMock(dev_server, '_get_dev_server_list')
306        bad_host, good_host = 'http://bad_host:99', 'http://good_host:8080'
307        dev_server._get_dev_server_list().MultipleTimes().AndReturn(
308                [bad_host, good_host])
309        argument1 = mox.StrContains(bad_host)
310        argument2 = mox.StrContains(good_host)
311
312        # Mock out bad ping failure to bad_host by raising devserver exception.
313        dev_server.ImageServerBase.run_call(
314                argument1, timeout=mox.IgnoreArg()).MultipleTimes().AndRaise(
315                        urllib2.URLError('urlopen connection timeout'))
316
317        # Good host is good.
318        dev_server.ImageServerBase.run_call(
319                argument2, timeout=mox.IgnoreArg()).AndReturn(
320                        '{"free_disk": 1024}')
321
322        self.mox.ReplayAll()
323        host = dev_server.ImageServer.resolve(0) # Using 0 as it'll hash to 0.
324        self.assertEquals(host.url(), good_host)
325        self.mox.VerifyAll()
326
327        retry.retry = real_retry
328
329
330    def testResolveWithManyDevservers(self):
331        """Should be able to return different urls with multiple devservers."""
332        self.mox.StubOutWithMock(dev_server.ImageServer, 'servers')
333        self.mox.StubOutWithMock(dev_server.DevServer, 'devserver_healthy')
334
335        host0_expected = 'http://host0:8080'
336        host1_expected = 'http://host1:8082'
337
338        dev_server.ImageServer.servers().MultipleTimes().AndReturn(
339                [host0_expected, host1_expected])
340        dev_server.ImageServer.devserver_healthy(host0_expected).AndReturn(True)
341        dev_server.ImageServer.devserver_healthy(host1_expected).AndReturn(True)
342
343        self.mox.ReplayAll()
344        host0 = dev_server.ImageServer.resolve(0)
345        host1 = dev_server.ImageServer.resolve(1)
346        self.mox.VerifyAll()
347
348        self.assertEqual(host0.url(), host0_expected)
349        self.assertEqual(host1.url(), host1_expected)
350
351
352    def testCmdErrorRetryCollectAULog(self):
353        """Devserver should retry _collect_au_log() on CMDError,
354        but pass through real exception."""
355        dev_server.ImageServerBase.run_call(
356                mox.IgnoreArg()).AndRaise(CMD_ERROR)
357        dev_server.ImageServerBase.run_call(
358                mox.IgnoreArg()).AndRaise(E500)
359        self.mox.ReplayAll()
360        self.assertFalse(self.dev_server.collect_au_log(
361                '100.0.0.0', 100, 'path/'))
362
363
364    def testURLErrorRetryCollectAULog(self):
365        """Devserver should retry _collect_au_log() on URLError,
366        but pass through real exception."""
367        self.mox.StubOutWithMock(time, 'sleep')
368
369        refused = urllib2.URLError('[Errno 111] Connection refused')
370        dev_server.ImageServerBase.run_call(
371                mox.IgnoreArg()).AndRaise(refused)
372        time.sleep(mox.IgnoreArg())
373        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E403)
374        self.mox.ReplayAll()
375        self.assertFalse(self.dev_server.collect_au_log(
376                '100.0.0.0', 100, 'path/'))
377
378
379    def testCmdErrorRetryKillAUProcess(self):
380        """Devserver should retry _kill_au_process() on CMDError,
381        but pass through real exception."""
382        dev_server.ImageServerBase.run_call(
383                mox.IgnoreArg()).AndRaise(CMD_ERROR)
384        dev_server.ImageServerBase.run_call(
385                mox.IgnoreArg()).AndRaise(E500)
386        self.mox.ReplayAll()
387        self.assertFalse(self.dev_server.kill_au_process_for_host(
388                '100.0.0.0', 100))
389
390
391    def testURLErrorRetryKillAUProcess(self):
392        """Devserver should retry _kill_au_process() on URLError,
393        but pass through real exception."""
394        self.mox.StubOutWithMock(time, 'sleep')
395
396        refused = urllib2.URLError('[Errno 111] Connection refused')
397        dev_server.ImageServerBase.run_call(
398                mox.IgnoreArg()).AndRaise(refused)
399        time.sleep(mox.IgnoreArg())
400        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E403)
401        self.mox.ReplayAll()
402        self.assertFalse(self.dev_server.kill_au_process_for_host(
403                '100.0.0.0', 100))
404
405
406    def testCmdErrorRetryCleanTrackLog(self):
407        """Devserver should retry _clean_track_log() on CMDError,
408        but pass through real exception."""
409        dev_server.ImageServerBase.run_call(
410                mox.IgnoreArg()).AndRaise(CMD_ERROR)
411        dev_server.ImageServerBase.run_call(
412                mox.IgnoreArg()).AndRaise(E500)
413        self.mox.ReplayAll()
414        self.assertFalse(self.dev_server.clean_track_log('100.0.0.0', 100))
415
416
417    def testURLErrorRetryCleanTrackLog(self):
418        """Devserver should retry _clean_track_log() on URLError,
419        but pass through real exception."""
420        self.mox.StubOutWithMock(time, 'sleep')
421
422        refused = urllib2.URLError('[Errno 111] Connection refused')
423        dev_server.ImageServerBase.run_call(
424                mox.IgnoreArg()).AndRaise(refused)
425        time.sleep(mox.IgnoreArg())
426        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E403)
427        self.mox.ReplayAll()
428        self.assertFalse(self.dev_server.clean_track_log('100.0.0.0', 100))
429
430
431    def _mockWriteFile(self):
432        """Mock write content to a file."""
433        mock_file = self.mox.CreateMockAnything()
434        open(mox.IgnoreArg(), 'w').AndReturn(mock_file)
435        mock_file.__enter__().AndReturn(mock_file)
436        mock_file.write(mox.IgnoreArg())
437        mock_file.__exit__(None, None, None)
438
439
440    def _preSetupForAutoUpdate(self, **kwargs):
441        """Pre-setup for testing auto_update logics and error handling in
442        devserver."""
443        response1 = (True, 100)
444        response2 = (True, 'Completed')
445        argument1 = mox.And(mox.StrContains(self._HOST),
446                            mox.StrContains('cros_au'))
447        argument2 = mox.And(mox.StrContains(self._HOST),
448                            mox.StrContains('get_au_status'))
449        argument3 = mox.And(mox.StrContains(self._HOST),
450                            mox.StrContains('handler_cleanup'))
451        argument4 = mox.And(mox.StrContains(self._HOST),
452                            mox.StrContains('collect_cros_au_log'))
453        argument5 = mox.And(mox.StrContains(self._HOST),
454                            mox.StrContains('kill_au_proc'))
455
456        retry_error = None
457        if 'retry_error' in kwargs:
458            retry_error = kwargs['retry_error']
459
460        raised_error = E403
461        if 'raised_error' in kwargs:
462            raised_error = kwargs['raised_error']
463
464        if 'cros_au_error' in kwargs:
465            if retry_error:
466                dev_server.ImageServerBase.run_call(argument1).AndRaise(
467                        retry_error)
468                time.sleep(mox.IgnoreArg())
469
470            if kwargs['cros_au_error']:
471                dev_server.ImageServerBase.run_call(argument1).AndRaise(
472                        raised_error)
473            else:
474                dev_server.ImageServerBase.run_call(argument1).AndReturn(
475                        json.dumps(response1))
476
477        if 'get_au_status_error' in kwargs:
478            if retry_error:
479                dev_server.ImageServerBase.run_call(argument2).AndRaise(
480                        retry_error)
481                time.sleep(mox.IgnoreArg())
482
483            if kwargs['get_au_status_error']:
484                dev_server.ImageServerBase.run_call(argument2).AndRaise(
485                        raised_error)
486            else:
487                dev_server.ImageServerBase.run_call(argument2).AndReturn(
488                        json.dumps(response2))
489
490        if 'handler_cleanup_error' in kwargs:
491            if kwargs['handler_cleanup_error']:
492                dev_server.ImageServerBase.run_call(argument3).AndRaise(
493                        raised_error)
494            else:
495                dev_server.ImageServerBase.run_call(argument3).AndReturn('True')
496
497        if 'collect_au_log_error' in kwargs:
498            if kwargs['collect_au_log_error']:
499                dev_server.ImageServerBase.run_call(argument4).AndRaise(
500                        raised_error)
501            else:
502                dev_server.ImageServerBase.run_call(argument4).AndReturn('log')
503                os.path.exists(mox.IgnoreArg()).AndReturn(True)
504                self._mockWriteFile()
505
506        if 'kill_au_proc_error' in kwargs:
507            if kwargs['kill_au_proc_error']:
508                dev_server.ImageServerBase.run_call(argument5).AndRaise(
509                        raised_error)
510            else:
511                dev_server.ImageServerBase.run_call(argument5).AndReturn('True')
512
513
514    def testSuccessfulTriggerAutoUpdate(self):
515        """Verify the dev server's auto_update() succeeds."""
516        kwargs={'cros_au_error': False, 'get_au_status_error': False,
517                'handler_cleanup_error': False}
518        self._preSetupForAutoUpdate(**kwargs)
519
520        self.mox.ReplayAll()
521        self.dev_server.auto_update('100.0.0.0', '')
522        self.mox.VerifyAll()
523
524
525    def testSuccessfulTriggerAutoUpdateWithCollectingLog(self):
526        """Verify the dev server's auto_update() with collecting logs
527        succeeds."""
528        kwargs={'cros_au_error': False, 'get_au_status_error': False,
529                'handler_cleanup_error': False, 'collect_au_log_error': False}
530        self.mox.StubOutWithMock(__builtin__, 'open')
531        self._preSetupForAutoUpdate(**kwargs)
532
533        self.mox.ReplayAll()
534        self.dev_server.auto_update('100.0.0.0', '', log_dir='path/')
535        self.mox.VerifyAll()
536
537
538    def testCrOSAUURLErrorRetryTriggerAutoUpdateSucceed(self):
539        """Devserver should retry cros_au() on URLError."""
540        self.mox.StubOutWithMock(time, 'sleep')
541        refused = urllib2.URLError('[Errno 111] Connection refused')
542        kwargs={'retry_error': refused, 'cros_au_error': False,
543                'get_au_status_error': False, 'handler_cleanup_error': False,
544                'collect_au_log_error': False}
545        self.mox.StubOutWithMock(__builtin__, 'open')
546        self._preSetupForAutoUpdate(**kwargs)
547
548        self.mox.ReplayAll()
549        self.dev_server.auto_update('100.0.0.0', '', log_dir='path/')
550        self.mox.VerifyAll()
551
552
553    def testCrOSAUCmdErrorRetryTriggerAutoUpdateSucceed(self):
554        """Devserver should retry cros_au() on CMDError."""
555        self.mox.StubOutWithMock(time, 'sleep')
556        self.mox.StubOutWithMock(__builtin__, 'open')
557        kwargs={'retry_error': CMD_ERROR, 'cros_au_error': False,
558                'get_au_status_error': False, 'handler_cleanup_error': False,
559                'collect_au_log_error': False}
560        self._preSetupForAutoUpdate(**kwargs)
561
562        self.mox.ReplayAll()
563        self.dev_server.auto_update('100.0.0.0', '', log_dir='path/')
564        self.mox.VerifyAll()
565
566
567    def testCrOSAUURLErrorRetryTriggerAutoUpdateFail(self):
568        """Devserver should retry cros_au() on URLError, but pass through
569        real exception."""
570        self.mox.StubOutWithMock(time, 'sleep')
571        refused = urllib2.URLError('[Errno 111] Connection refused')
572        kwargs={'retry_error': refused, 'cros_au_error': True,
573                'raised_error': E500}
574
575        for i in range(dev_server.AU_RETRY_LIMIT):
576            self._preSetupForAutoUpdate(**kwargs)
577            if i < dev_server.AU_RETRY_LIMIT - 1:
578                time.sleep(mox.IgnoreArg())
579
580        self.mox.ReplayAll()
581        self.assertRaises(dev_server.DevServerException,
582                          self.dev_server.auto_update,
583                          '', '')
584
585
586    def testCrOSAUCmdErrorRetryTriggerAutoUpdateFail(self):
587        """Devserver should retry cros_au() on CMDError, but pass through
588        real exception."""
589        self.mox.StubOutWithMock(time, 'sleep')
590        kwargs={'retry_error': CMD_ERROR, 'cros_au_error': True}
591
592        for i in range(dev_server.AU_RETRY_LIMIT):
593            self._preSetupForAutoUpdate(**kwargs)
594            if i < dev_server.AU_RETRY_LIMIT - 1:
595                time.sleep(mox.IgnoreArg())
596
597        self.mox.ReplayAll()
598        self.assertRaises(dev_server.DevServerException,
599                          self.dev_server.auto_update,
600                          '', '')
601
602
603    def testGetAUStatusErrorInAutoUpdate(self):
604        """Verify devserver's auto_update() logics for handling get_au_status
605        errors.
606
607        Func auto_update() should call 'handler_cleanup' and 'collect_au_log'
608        even if '_trigger_auto_update()' failed.
609        """
610        self.mox.StubOutWithMock(time, 'sleep')
611        self.mox.StubOutWithMock(__builtin__, 'open')
612        kwargs={'cros_au_error': False, 'get_au_status_error': True,
613                'handler_cleanup_error': False, 'collect_au_log_error': False,
614                'kill_au_proc_error': False}
615
616        for i in range(dev_server.AU_RETRY_LIMIT):
617            self._preSetupForAutoUpdate(**kwargs)
618            if i < dev_server.AU_RETRY_LIMIT - 1:
619                time.sleep(mox.IgnoreArg())
620
621        self.mox.ReplayAll()
622        self.assertRaises(dev_server.DevServerException,
623                          self.dev_server.auto_update,
624                          '100.0.0.0', 'build', log_dir='path/')
625
626
627    def testCleanUpErrorInAutoUpdate(self):
628        """Verify devserver's auto_update() logics for handling handler_cleanup
629        errors.
630
631        Func auto_update() should call 'handler_cleanup' and 'collect_au_log'
632        no matter '_trigger_auto_update()' succeeds or fails.
633        """
634        self.mox.StubOutWithMock(time, 'sleep')
635        self.mox.StubOutWithMock(__builtin__, 'open')
636        kwargs={'cros_au_error': False, 'get_au_status_error': False,
637                'handler_cleanup_error': True, 'collect_au_log_error': False,
638                'kill_au_proc_error': False}
639
640
641        for i in range(dev_server.AU_RETRY_LIMIT):
642            self._preSetupForAutoUpdate(**kwargs)
643            if i < dev_server.AU_RETRY_LIMIT - 1:
644                time.sleep(mox.IgnoreArg())
645
646        self.mox.ReplayAll()
647        self.assertRaises(dev_server.DevServerException,
648                          self.dev_server.auto_update,
649                          '100.0.0.0', 'build', log_dir='path/')
650
651
652    def testCollectLogErrorInAutoUpdate(self):
653        """Verify devserver's auto_update() logics for handling collect_au_log
654        errors."""
655        self.mox.StubOutWithMock(time, 'sleep')
656        kwargs={'cros_au_error': False, 'get_au_status_error': False,
657                'handler_cleanup_error': False, 'collect_au_log_error': True,
658                'kill_au_proc_error': False}
659
660
661        for i in range(dev_server.AU_RETRY_LIMIT):
662            self._preSetupForAutoUpdate(**kwargs)
663            if i < dev_server.AU_RETRY_LIMIT - 1:
664                time.sleep(mox.IgnoreArg())
665
666        self.mox.ReplayAll()
667        self.assertRaises(dev_server.DevServerException,
668                          self.dev_server.auto_update,
669                          '100.0.0.0', 'build', log_dir='path/')
670
671
672    def testGetAUStatusErrorAndCleanUpErrorInAutoUpdate(self):
673        """Verify devserver's auto_update() logics for handling get_au_status
674        and handler_cleanup errors.
675
676        Func auto_update() should call 'handler_cleanup' and 'collect_au_log'
677        even if '_trigger_auto_update()' fails.
678        """
679        self.mox.StubOutWithMock(time, 'sleep')
680        self.mox.StubOutWithMock(__builtin__, 'open')
681        kwargs={'cros_au_error': False, 'get_au_status_error': True,
682                'handler_cleanup_error': True, 'collect_au_log_error': False,
683                'kill_au_proc_error': False}
684
685
686        for i in range(dev_server.AU_RETRY_LIMIT):
687            self._preSetupForAutoUpdate(**kwargs)
688            if i < dev_server.AU_RETRY_LIMIT - 1:
689                time.sleep(mox.IgnoreArg())
690
691        self.mox.ReplayAll()
692        self.assertRaises(dev_server.DevServerException,
693                          self.dev_server.auto_update,
694                          '100.0.0.0', 'build', log_dir='path/')
695
696
697    def testGetAUStatusErrorAndCleanUpErrorAndCollectLogErrorInAutoUpdate(self):
698        """Verify devserver's auto_update() logics for handling get_au_status,
699        handler_cleanup, and collect_au_log errors.
700
701        Func auto_update() should call 'handler_cleanup' and 'collect_au_log'
702        even if '_trigger_auto_update()' fails.
703        """
704        self.mox.StubOutWithMock(time, 'sleep')
705        kwargs={'cros_au_error': False, 'get_au_status_error': True,
706                'handler_cleanup_error': True, 'collect_au_log_error': True,
707                'kill_au_proc_error': False}
708
709        for i in range(dev_server.AU_RETRY_LIMIT):
710            self._preSetupForAutoUpdate(**kwargs)
711            if i < dev_server.AU_RETRY_LIMIT - 1:
712                time.sleep(mox.IgnoreArg())
713
714        self.mox.ReplayAll()
715        self.assertRaises(dev_server.DevServerException,
716                          self.dev_server.auto_update,
717                          '100.0.0.0', 'build', log_dir='path/')
718
719
720    def testGetAUStatusErrorAndCleanUpErrorAndCollectLogErrorAndKillErrorInAutoUpdate(self):
721        """Verify devserver's auto_update() logics for handling get_au_status,
722        handler_cleanup, collect_au_log, and kill_au_proc errors.
723
724        Func auto_update() should call 'handler_cleanup' and 'collect_au_log'
725        even if '_trigger_auto_update()' fails.
726        """
727        self.mox.StubOutWithMock(time, 'sleep')
728
729        kwargs={'cros_au_error': False, 'get_au_status_error': True,
730                'handler_cleanup_error': True, 'collect_au_log_error': True,
731                'kill_au_proc_error': True}
732
733        for i in range(dev_server.AU_RETRY_LIMIT):
734            self._preSetupForAutoUpdate(**kwargs)
735            if i < dev_server.AU_RETRY_LIMIT - 1:
736                time.sleep(mox.IgnoreArg())
737
738        self.mox.ReplayAll()
739        self.assertRaises(dev_server.DevServerException,
740                          self.dev_server.auto_update,
741                          '100.0.0.0', 'build', log_dir='path/')
742
743
744    def testSuccessfulTriggerDownloadSync(self):
745        """Call the dev server's download method with synchronous=True."""
746        name = 'fake/image'
747        self.mox.StubOutWithMock(dev_server.ImageServer, '_finish_download')
748        argument1 = mox.And(mox.StrContains(self._HOST), mox.StrContains(name),
749                            mox.StrContains('stage?'))
750        argument2 = mox.And(mox.StrContains(self._HOST), mox.StrContains(name),
751                            mox.StrContains('is_staged'))
752        dev_server.ImageServerBase.run_call(argument1).AndReturn('Success')
753        dev_server.ImageServerBase.run_call(argument2).AndReturn('True')
754        self.dev_server._finish_download(name, mox.IgnoreArg(), mox.IgnoreArg())
755
756        # Synchronous case requires a call to finish download.
757        self.mox.ReplayAll()
758        self.dev_server.trigger_download(name, synchronous=True)
759        self.mox.VerifyAll()
760
761
762    def testSuccessfulTriggerDownloadASync(self):
763        """Call the dev server's download method with synchronous=False."""
764        name = 'fake/image'
765        argument1 = mox.And(mox.StrContains(self._HOST), mox.StrContains(name),
766                            mox.StrContains('stage?'))
767        argument2 = mox.And(mox.StrContains(self._HOST), mox.StrContains(name),
768                            mox.StrContains('is_staged'))
769        dev_server.ImageServerBase.run_call(argument1).AndReturn('Success')
770        dev_server.ImageServerBase.run_call(argument2).AndReturn('True')
771
772        self.mox.ReplayAll()
773        self.dev_server.trigger_download(name, synchronous=False)
774        self.mox.VerifyAll()
775
776
777    def testURLErrorRetryTriggerDownload(self):
778        """Should retry on URLError, but pass through real exception."""
779        self.mox.StubOutWithMock(time, 'sleep')
780
781        refused = urllib2.URLError('[Errno 111] Connection refused')
782        dev_server.ImageServerBase.run_call(
783                mox.IgnoreArg()).AndRaise(refused)
784        time.sleep(mox.IgnoreArg())
785        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E403)
786        self.mox.ReplayAll()
787        self.assertRaises(dev_server.DevServerException,
788                          self.dev_server.trigger_download,
789                          '')
790
791
792    def testErrorTriggerDownload(self):
793        """Should call the dev server's download method using http, fail
794        gracefully."""
795        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E500)
796        self.mox.ReplayAll()
797        self.assertRaises(dev_server.DevServerException,
798                          self.dev_server.trigger_download,
799                          '')
800
801
802    def testForbiddenTriggerDownload(self):
803        """Should call the dev server's download method using http,
804        get exception."""
805        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E403)
806        self.mox.ReplayAll()
807        self.assertRaises(dev_server.DevServerException,
808                          self.dev_server.trigger_download,
809                          '')
810
811
812    def testCmdErrorTriggerDownload(self):
813        """Should call the dev server's download method using ssh, retry
814        trigger_download when getting error.CmdError, raise exception for
815        urllib2.HTTPError."""
816        dev_server.ImageServerBase.run_call(
817                mox.IgnoreArg()).AndRaise(CMD_ERROR)
818        dev_server.ImageServerBase.run_call(
819                mox.IgnoreArg()).AndRaise(E500)
820        self.mox.ReplayAll()
821        self.assertRaises(dev_server.DevServerException,
822                          self.dev_server.trigger_download,
823                          '')
824
825
826    def testSuccessfulFinishDownload(self):
827        """Should successfully call the dev server's finish download method."""
828        name = 'fake/image'
829        argument1 = mox.And(mox.StrContains(self._HOST),
830                            mox.StrContains(name),
831                            mox.StrContains('stage?'))
832        argument2 = mox.And(mox.StrContains(self._HOST),
833                            mox.StrContains(name),
834                            mox.StrContains('is_staged'))
835        dev_server.ImageServerBase.run_call(argument1).AndReturn('Success')
836        dev_server.ImageServerBase.run_call(argument2).AndReturn('True')
837
838        # Synchronous case requires a call to finish download.
839        self.mox.ReplayAll()
840        self.dev_server.finish_download(name)  # Raises on failure.
841        self.mox.VerifyAll()
842
843
844    def testErrorFinishDownload(self):
845        """Should call the dev server's finish download method using http, fail
846        gracefully."""
847        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E500)
848        self.mox.ReplayAll()
849        self.assertRaises(dev_server.DevServerException,
850                          self.dev_server.finish_download,
851                          '')
852
853
854    def testCmdErrorFinishDownload(self):
855        """Should call the dev server's finish download method using ssh,
856        retry finish_download when getting error.CmdError, raise exception
857        for urllib2.HTTPError."""
858        dev_server.ImageServerBase.run_call(
859                mox.IgnoreArg()).AndRaise(CMD_ERROR)
860        dev_server.ImageServerBase.run_call(
861                mox.IgnoreArg()).AndRaise(E500)
862        self.mox.ReplayAll()
863        self.assertRaises(dev_server.DevServerException,
864                          self.dev_server.finish_download,
865                          '')
866
867
868    def testListControlFiles(self):
869        """Should successfully list control files from the dev server."""
870        name = 'fake/build'
871        control_files = ['file/one', 'file/two']
872        argument = mox.And(mox.StrContains(self._HOST),
873                           mox.StrContains(name))
874        dev_server.ImageServerBase.run_call(
875                argument, readline=True).AndReturn(control_files)
876
877        self.mox.ReplayAll()
878        paths = self.dev_server.list_control_files(name)
879        self.assertEquals(len(paths), 2)
880        for f in control_files:
881            self.assertTrue(f in paths)
882
883
884    def testFailedListControlFiles(self):
885        """Should call the dev server's list-files method using http, get
886        exception."""
887        dev_server.ImageServerBase.run_call(
888                mox.IgnoreArg(), readline=True).AndRaise(E500)
889        self.mox.ReplayAll()
890        self.assertRaises(dev_server.DevServerException,
891                          self.dev_server.list_control_files,
892                          '')
893
894
895    def testExplodingListControlFiles(self):
896        """Should call the dev server's list-files method using http, get
897        exception."""
898        dev_server.ImageServerBase.run_call(
899                mox.IgnoreArg(), readline=True).AndRaise(E403)
900        self.mox.ReplayAll()
901        self.assertRaises(dev_server.DevServerException,
902                          self.dev_server.list_control_files,
903                          '')
904
905
906    def testCmdErrorListControlFiles(self):
907        """Should call the dev server's list-files method using ssh, retry
908        list_control_files when getting error.CmdError, raise exception for
909        urllib2.HTTPError."""
910        dev_server.ImageServerBase.run_call(
911                mox.IgnoreArg(), readline=True).AndRaise(CMD_ERROR)
912        dev_server.ImageServerBase.run_call(
913                mox.IgnoreArg(), readline=True).AndRaise(E500)
914        self.mox.ReplayAll()
915        self.assertRaises(dev_server.DevServerException,
916                          self.dev_server.list_control_files,
917                          '')
918
919    def testListSuiteControls(self):
920        """Should successfully list all contents of control files from the dev
921        server."""
922        name = 'fake/build'
923        control_contents = ['control file one', 'control file two']
924        argument = mox.And(mox.StrContains(self._HOST),
925                           mox.StrContains(name))
926        dev_server.ImageServerBase.run_call(
927                argument).AndReturn(json.dumps(control_contents))
928
929        self.mox.ReplayAll()
930        file_contents = self.dev_server.list_suite_controls(name)
931        self.assertEquals(len(file_contents), 2)
932        for f in control_contents:
933            self.assertTrue(f in file_contents)
934
935
936    def testFailedListSuiteControls(self):
937        """Should call the dev server's list_suite_controls method using http,
938        get exception."""
939        dev_server.ImageServerBase.run_call(
940                mox.IgnoreArg()).AndRaise(E500)
941        self.mox.ReplayAll()
942        self.assertRaises(dev_server.DevServerException,
943                          self.dev_server.list_suite_controls,
944                          '')
945
946
947    def testExplodingListSuiteControls(self):
948        """Should call the dev server's list_suite_controls method using http,
949        get exception."""
950        dev_server.ImageServerBase.run_call(
951                mox.IgnoreArg()).AndRaise(E403)
952        self.mox.ReplayAll()
953        self.assertRaises(dev_server.DevServerException,
954                          self.dev_server.list_suite_controls,
955                          '')
956
957
958    def testCmdErrorListSuiteControls(self):
959        """Should call the dev server's list_suite_controls method using ssh,
960        retry list_suite_controls when getting error.CmdError, raise exception
961        for urllib2.HTTPError."""
962        dev_server.ImageServerBase.run_call(
963                mox.IgnoreArg()).AndRaise(CMD_ERROR)
964        dev_server.ImageServerBase.run_call(
965                mox.IgnoreArg()).AndRaise(E500)
966        self.mox.ReplayAll()
967        self.assertRaises(dev_server.DevServerException,
968                          self.dev_server.list_suite_controls,
969                          '')
970
971
972    def testGetControlFile(self):
973        """Should successfully get a control file from the dev server."""
974        name = 'fake/build'
975        file = 'file/one'
976        contents = 'Multi-line\nControl File Contents\n'
977        argument = mox.And(mox.StrContains(self._HOST),
978                            mox.StrContains(name),
979                            mox.StrContains(file))
980        dev_server.ImageServerBase.run_call(argument).AndReturn(contents)
981
982        self.mox.ReplayAll()
983        self.assertEquals(self.dev_server.get_control_file(name, file),
984                          contents)
985
986
987    def testErrorGetControlFile(self):
988        """Should try to get the contents of a control file using http, get
989        exception."""
990        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E500)
991        self.mox.ReplayAll()
992        self.assertRaises(dev_server.DevServerException,
993                          self.dev_server.get_control_file,
994                          '', '')
995
996
997    def testForbiddenGetControlFile(self):
998        """Should try to get the contents of a control file using http, get
999        exception."""
1000        dev_server.ImageServerBase.run_call(mox.IgnoreArg()).AndRaise(E403)
1001        self.mox.ReplayAll()
1002        self.assertRaises(dev_server.DevServerException,
1003                          self.dev_server.get_control_file,
1004                          '', '')
1005
1006
1007    def testCmdErrorGetControlFile(self):
1008        """Should try to get the contents of a control file using ssh, retry
1009        get_control_file when getting error.CmdError, raise exception for
1010        urllib2.HTTPError."""
1011        dev_server.ImageServerBase.run_call(
1012                mox.IgnoreArg()).AndRaise(CMD_ERROR)
1013        dev_server.ImageServerBase.run_call(
1014                mox.IgnoreArg()).AndRaise(E500)
1015        self.mox.ReplayAll()
1016        self.assertRaises(dev_server.DevServerException,
1017                          self.dev_server.get_control_file,
1018                          '', '')
1019
1020
1021    def testGetLatestBuild(self):
1022        """Should successfully return a build for a given target."""
1023        self.mox.StubOutWithMock(dev_server.ImageServer, 'servers')
1024        self.mox.StubOutWithMock(dev_server.DevServer, 'devserver_healthy')
1025
1026        dev_server.ImageServer.servers().AndReturn([self._HOST])
1027        dev_server.ImageServer.devserver_healthy(self._HOST).AndReturn(True)
1028
1029        target = 'x86-generic-release'
1030        build_string = 'R18-1586.0.0-a1-b1514'
1031        argument = mox.And(mox.StrContains(self._HOST),
1032                           mox.StrContains(target))
1033        dev_server.ImageServerBase.run_call(argument).AndReturn(build_string)
1034
1035        self.mox.ReplayAll()
1036        build = dev_server.ImageServer.get_latest_build(target)
1037        self.assertEquals(build_string, build)
1038
1039
1040    def testGetLatestBuildWithManyDevservers(self):
1041        """Should successfully return newest build with multiple devservers."""
1042        self.mox.StubOutWithMock(dev_server.ImageServer, 'servers')
1043        self.mox.StubOutWithMock(dev_server.DevServer, 'devserver_healthy')
1044
1045        host0_expected = 'http://host0:8080'
1046        host1_expected = 'http://host1:8082'
1047
1048        dev_server.ImageServer.servers().MultipleTimes().AndReturn(
1049                [host0_expected, host1_expected])
1050
1051        dev_server.ImageServer.devserver_healthy(host0_expected).AndReturn(True)
1052        dev_server.ImageServer.devserver_healthy(host1_expected).AndReturn(True)
1053
1054        target = 'x86-generic-release'
1055        build_string1 = 'R9-1586.0.0-a1-b1514'
1056        build_string2 = 'R19-1586.0.0-a1-b3514'
1057        argument1 = mox.And(mox.StrContains(host0_expected),
1058                            mox.StrContains(target))
1059        argument2 = mox.And(mox.StrContains(host1_expected),
1060                            mox.StrContains(target))
1061        dev_server.ImageServerBase.run_call(argument1).AndReturn(build_string1)
1062        dev_server.ImageServerBase.run_call(argument2).AndReturn(build_string2)
1063
1064        self.mox.ReplayAll()
1065        build = dev_server.ImageServer.get_latest_build(target)
1066        self.assertEquals(build_string2, build)
1067
1068
1069    def testCrashesAreSetToTheCrashServer(self):
1070        """Should send symbolicate dump rpc calls to crash_server."""
1071        self.mox.ReplayAll()
1072        call = self.crash_server.build_call('symbolicate_dump')
1073        self.assertTrue(call.startswith(self._CRASH_HOST))
1074
1075
1076    def _stageTestHelper(self, artifacts=[], files=[], archive_url=None):
1077        """Helper to test combos of files/artifacts/urls with stage call."""
1078        expected_archive_url = archive_url
1079        if not archive_url:
1080            expected_archive_url = 'gs://my_default_url'
1081            self.mox.StubOutWithMock(dev_server, '_get_image_storage_server')
1082            dev_server._get_image_storage_server().AndReturn(
1083                'gs://my_default_url')
1084            name = 'fake/image'
1085        else:
1086            # This is embedded in the archive_url. Not needed.
1087            name = ''
1088
1089        argument1 = mox.And(mox.StrContains(expected_archive_url),
1090                            mox.StrContains(name),
1091                            mox.StrContains('artifacts=%s' %
1092                                            ','.join(artifacts)),
1093                            mox.StrContains('files=%s' % ','.join(files)),
1094                            mox.StrContains('stage?'))
1095        argument2 = mox.And(mox.StrContains(expected_archive_url),
1096                            mox.StrContains(name),
1097                            mox.StrContains('artifacts=%s' %
1098                                            ','.join(artifacts)),
1099                            mox.StrContains('files=%s' % ','.join(files)),
1100                            mox.StrContains('is_staged'))
1101        dev_server.ImageServerBase.run_call(argument1).AndReturn('Success')
1102        dev_server.ImageServerBase.run_call(argument2).AndReturn('True')
1103
1104        self.mox.ReplayAll()
1105        self.dev_server.stage_artifacts(name, artifacts, files, archive_url)
1106        self.mox.VerifyAll()
1107
1108
1109    def testStageArtifactsBasic(self):
1110        """Basic functionality to stage artifacts (similar to
1111        trigger_download)."""
1112        self._stageTestHelper(artifacts=['full_payload', 'stateful'])
1113
1114
1115    def testStageArtifactsBasicWithFiles(self):
1116        """Basic functionality to stage artifacts (similar to
1117        trigger_download)."""
1118        self._stageTestHelper(artifacts=['full_payload', 'stateful'],
1119                              files=['taco_bell.coupon'])
1120
1121
1122    def testStageArtifactsOnlyFiles(self):
1123        """Test staging of only file artifacts."""
1124        self._stageTestHelper(files=['tasty_taco_bell.coupon'])
1125
1126
1127    def testStageWithArchiveURL(self):
1128        """Basic functionality to stage artifacts (similar to
1129        trigger_download)."""
1130        self._stageTestHelper(files=['tasty_taco_bell.coupon'],
1131                              archive_url='gs://tacos_galore/my/dir')
1132
1133
1134    def testStagedFileUrl(self):
1135        """Sanity tests that the staged file url looks right."""
1136        devserver_label = 'x86-mario-release/R30-1234.0.0'
1137        url = self.dev_server.get_staged_file_url('stateful.tgz',
1138                                                  devserver_label)
1139        expected_url = '/'.join([self._HOST, 'static', devserver_label,
1140                                 'stateful.tgz'])
1141        self.assertEquals(url, expected_url)
1142
1143        devserver_label = 'something_crazy/that/you_MIGHT/hate'
1144        url = self.dev_server.get_staged_file_url('chromiumos_image.bin',
1145                                                  devserver_label)
1146        expected_url = '/'.join([self._HOST, 'static', devserver_label,
1147                                 'chromiumos_image.bin'])
1148        self.assertEquals(url, expected_url)
1149
1150
1151    def _StageTimeoutHelper(self):
1152        """Helper class for testing staging timeout."""
1153        self.mox.StubOutWithMock(dev_server.ImageServer, 'call_and_wait')
1154        dev_server.ImageServer.call_and_wait(
1155                call_name='stage',
1156                artifacts=mox.IgnoreArg(),
1157                files=mox.IgnoreArg(),
1158                archive_url=mox.IgnoreArg(),
1159                error_message=mox.IgnoreArg()).AndRaise(bin_utils.TimeoutError())
1160
1161
1162    def test_StageArtifactsTimeout(self):
1163        """Test DevServerException is raised when stage_artifacts timed out."""
1164        self._StageTimeoutHelper()
1165        self.mox.ReplayAll()
1166        self.assertRaises(dev_server.DevServerException,
1167                          self.dev_server.stage_artifacts,
1168                          image='fake/image', artifacts=['full_payload'])
1169        self.mox.VerifyAll()
1170
1171
1172    def test_TriggerDownloadTimeout(self):
1173        """Test DevServerException is raised when trigger_download timed out."""
1174        self._StageTimeoutHelper()
1175        self.mox.ReplayAll()
1176        self.assertRaises(dev_server.DevServerException,
1177                          self.dev_server.trigger_download,
1178                          image='fake/image')
1179        self.mox.VerifyAll()
1180
1181
1182    def test_FinishDownloadTimeout(self):
1183        """Test DevServerException is raised when finish_download timed out."""
1184        self._StageTimeoutHelper()
1185        self.mox.ReplayAll()
1186        self.assertRaises(dev_server.DevServerException,
1187                          self.dev_server.finish_download,
1188                          image='fake/image')
1189        self.mox.VerifyAll()
1190
1191
1192    def test_compare_load(self):
1193        """Test load comparison logic.
1194        """
1195        load_high_cpu = {'devserver': 'http://devserver_1:8082',
1196                         dev_server.DevServer.CPU_LOAD: 100.0,
1197                         dev_server.DevServer.NETWORK_IO: 1024*1024*1.0,
1198                         dev_server.DevServer.DISK_IO: 1024*1024.0}
1199        load_high_network = {'devserver': 'http://devserver_1:8082',
1200                             dev_server.DevServer.CPU_LOAD: 1.0,
1201                             dev_server.DevServer.NETWORK_IO: 1024*1024*100.0,
1202                             dev_server.DevServer.DISK_IO: 1024*1024*1.0}
1203        load_1 = {'devserver': 'http://devserver_1:8082',
1204                  dev_server.DevServer.CPU_LOAD: 1.0,
1205                  dev_server.DevServer.NETWORK_IO: 1024*1024*1.0,
1206                  dev_server.DevServer.DISK_IO: 1024*1024*2.0}
1207        load_2 = {'devserver': 'http://devserver_1:8082',
1208                  dev_server.DevServer.CPU_LOAD: 1.0,
1209                  dev_server.DevServer.NETWORK_IO: 1024*1024*1.0,
1210                  dev_server.DevServer.DISK_IO: 1024*1024*1.0}
1211        self.assertFalse(dev_server._is_load_healthy(load_high_cpu))
1212        self.assertFalse(dev_server._is_load_healthy(load_high_network))
1213        self.assertTrue(dev_server._compare_load(load_1, load_2) > 0)
1214
1215
1216    def _testSuccessfulTriggerDownloadAndroid(self, synchronous=True):
1217        """Call the dev server's download method with given synchronous
1218        setting.
1219
1220        @param synchronous: True to call the download method synchronously.
1221        """
1222        target = 'test_target'
1223        branch = 'test_branch'
1224        build_id = '123456'
1225        artifacts = android_utils.AndroidArtifacts.get_artifacts_for_reimage(
1226                None)
1227        self.mox.StubOutWithMock(dev_server.AndroidBuildServer,
1228                                 '_finish_download')
1229        argument1 = mox.And(mox.StrContains(self._HOST),
1230                            mox.StrContains(target),
1231                            mox.StrContains(branch),
1232                            mox.StrContains(build_id),
1233                            mox.StrContains('stage?'))
1234        argument2 = mox.And(mox.StrContains(self._HOST),
1235                            mox.StrContains(target),
1236                            mox.StrContains(branch),
1237                            mox.StrContains(build_id),
1238                            mox.StrContains('is_staged'))
1239        dev_server.ImageServerBase.run_call(argument1).AndReturn('Success')
1240        dev_server.ImageServerBase.run_call(argument2).AndReturn('True')
1241
1242        if synchronous:
1243            android_build_info = {'target': target,
1244                                  'build_id': build_id,
1245                                  'branch': branch}
1246            build = dev_server.ANDROID_BUILD_NAME_PATTERN % android_build_info
1247            self.android_dev_server._finish_download(
1248                    build, artifacts, '', target=target, build_id=build_id,
1249                    branch=branch)
1250
1251        # Synchronous case requires a call to finish download.
1252        self.mox.ReplayAll()
1253        self.android_dev_server.trigger_download(
1254                synchronous=synchronous, target=target, build_id=build_id,
1255                branch=branch)
1256        self.mox.VerifyAll()
1257
1258
1259    def testSuccessfulTriggerDownloadAndroidSync(self):
1260        """Call the dev server's download method with synchronous=True."""
1261        self._testSuccessfulTriggerDownloadAndroid(synchronous=True)
1262
1263
1264    def testSuccessfulTriggerDownloadAndroidAsync(self):
1265        """Call the dev server's download method with synchronous=False."""
1266        self._testSuccessfulTriggerDownloadAndroid(synchronous=False)
1267
1268
1269    def testGetUnrestrictedDevservers(self):
1270        """Test method get_unrestricted_devservers works as expected."""
1271        restricted_devserver = 'http://192.168.0.100:8080'
1272        unrestricted_devserver = 'http://172.1.1.3:8080'
1273        self.mox.StubOutWithMock(dev_server.ImageServer, 'servers')
1274        dev_server.ImageServer.servers().AndReturn([restricted_devserver,
1275                                                    unrestricted_devserver])
1276        self.mox.ReplayAll()
1277        self.assertEqual(dev_server.ImageServer.get_unrestricted_devservers(
1278                                [('192.168.0.0', 24)]),
1279                         [unrestricted_devserver])
1280
1281
1282    def testDevserverHealthy(self):
1283        """Test which types of connections that method devserver_healthy uses
1284        for different types of DevServer.
1285
1286        CrashServer always adopts DevServer.run_call.
1287        ImageServer and AndroidBuildServer use ImageServerBase.run_call.
1288        """
1289        argument = mox.StrContains(self._HOST)
1290
1291        # for testing CrashServer
1292        self.mox.StubOutWithMock(dev_server.DevServer, 'run_call')
1293        dev_server.DevServer.run_call(
1294                argument, timeout=mox.IgnoreArg()).AndReturn(
1295                        '{"free_disk": 1024}')
1296        # for testing ImageServer
1297        dev_server.ImageServerBase.run_call(
1298                argument, timeout=mox.IgnoreArg()).AndReturn(
1299                        '{"free_disk": 1024}')
1300        # for testing AndroidBuildServer
1301        dev_server.ImageServerBase.run_call(
1302                argument, timeout=mox.IgnoreArg()).AndReturn(
1303                        '{"free_disk": 1024}')
1304
1305        self.mox.ReplayAll()
1306        self.assertTrue(dev_server.CrashServer.devserver_healthy(self._HOST))
1307        self.assertTrue(dev_server.ImageServer.devserver_healthy(self._HOST))
1308        self.assertTrue(
1309                dev_server.AndroidBuildServer.devserver_healthy(self._HOST))
1310
1311
1312    def testLocateFile(self):
1313        """Test locating files for AndriodBuildServer."""
1314        file_name = 'fake_file'
1315        artifacts=['full_payload', 'stateful']
1316        build = 'fake_build'
1317        argument = mox.And(mox.StrContains(file_name),
1318                            mox.StrContains(build),
1319                            mox.StrContains('locate_file'))
1320        dev_server.ImageServerBase.run_call(argument).AndReturn('file_path')
1321
1322        self.mox.ReplayAll()
1323        file_location = 'http://nothing/static/fake_build/file_path'
1324        self.assertEqual(self.android_dev_server.locate_file(
1325                file_name, artifacts, build, None), file_location)
1326
1327    def testCmdErrorLocateFile(self):
1328        """Test locating files for AndriodBuildServer for retry
1329        error.CmdError, and raise urllib2.URLError."""
1330        dev_server.ImageServerBase.run_call(
1331                mox.IgnoreArg()).AndRaise(CMD_ERROR)
1332        dev_server.ImageServerBase.run_call(
1333                mox.IgnoreArg()).AndRaise(E500)
1334        self.mox.ReplayAll()
1335        self.assertRaises(dev_server.DevServerException,
1336                          self.dev_server.trigger_download,
1337                          '')
1338
1339
1340    def testGetAvailableDevserversForCrashServer(self):
1341        """Test method get_available_devservers for CrashServer."""
1342        crash_servers = ['http://crash_servers1:8080']
1343        host = '127.0.0.1'
1344        self.mox.StubOutWithMock(dev_server.CrashServer, 'servers')
1345        dev_server.CrashServer.servers().AndReturn(crash_servers)
1346        self.mox.ReplayAll()
1347        self.assertEqual(dev_server.CrashServer.get_available_devservers(host),
1348                        (crash_servers, False))
1349
1350
1351    def testGetAvailableDevserversForImageServer(self):
1352        """Test method get_available_devservers for ImageServer."""
1353        unrestricted_host = '100.0.0.99'
1354        unrestricted_servers = ['http://100.0.0.10:8080',
1355                                'http://128.0.0.10:8080']
1356        same_subnet_unrestricted_servers = ['http://100.0.0.10:8080']
1357        restricted_host = '127.0.0.99'
1358        restricted_servers = ['http://127.0.0.10:8080']
1359        all_servers = unrestricted_servers + restricted_servers
1360        # Set restricted subnets
1361        restricted_subnets = [('127.0.0.0', 24)]
1362        self.mox.StubOutWithMock(dev_server.ImageServerBase, 'servers')
1363        dev_server.ImageServerBase.servers().MultipleTimes().AndReturn(
1364                all_servers)
1365        self.mox.ReplayAll()
1366        # dut in unrestricted subnet shall be offered devserver in the same
1367        # subnet first, and allow retry.
1368        self.assertEqual(
1369                dev_server.ImageServer.get_available_devservers(
1370                        unrestricted_host, True, restricted_subnets),
1371                (same_subnet_unrestricted_servers, True))
1372
1373        # If prefer_local_devserver is set to False, allow any devserver in
1374        # unrestricted subet to be available, and retry is not allowed.
1375        self.assertEqual(
1376                dev_server.ImageServer.get_available_devservers(
1377                        unrestricted_host, False, restricted_subnets),
1378                (unrestricted_servers, False))
1379
1380        # When no hostname is specified, all devservers in unrestricted subnets
1381        # should be considered, and retry is not allowed.
1382        self.assertEqual(
1383                dev_server.ImageServer.get_available_devservers(
1384                        None, True, restricted_subnets),
1385                (unrestricted_servers, False))
1386
1387        # dut in restricted subnet should only be offered devserver in the
1388        # same restricted subnet, and retry is not allowed.
1389        self.assertEqual(
1390                dev_server.ImageServer.get_available_devservers(
1391                        restricted_host, True, restricted_subnets),
1392                (restricted_servers, False))
1393
1394
1395if __name__ == "__main__":
1396    unittest.main()
1397