complete_failures_unittest.py revision 1a3c8dd59f71086eb2f5d08b0f13df95428e60fa
11a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood#!/usr/bin/python 21a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood# 31a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 41a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood# Use of this source code is governed by a BSD-style license that can be 51a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood# found in the LICENSE file. 61a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 71a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hoodimport datetime, unittest 81a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 91a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hoodimport mox 101a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 111a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hoodimport common, complete_failures 121a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hoodfrom autotest_lib.client.common_lib import mail 131a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 141a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 151a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hoodclass EmailAboutTestFailureTests(mox.MoxTestBase): 161a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """ 171a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood Test the core logic of the comlete_failures.py script. 181a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 191a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood The core logic is to send emails only if we have not yet done so for a 201a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood given test before and to take actions if the test has been failing for 211a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood long enough. 221a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 231a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """ 241a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def setUp(self): 251a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood super(EmailAboutTestFailureTests, self).setUp() 261a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 271a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # We need to mock out the send function in all tests or else the 281a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # emails will be sent out during tests. 291a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.StubOutWithMock(mail, 'send') 301a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 311a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self._orignal_too_late = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG 321a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 331a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 341a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def tearDown(self): 351a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orignal_too_late 361a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 371a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 381a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def test_deal_with_new_failing_test(self): 391a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """ 401a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood Test adding a failing test to the storage. 411a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 421a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood We expect the email sending code to be called if it is added. 431a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 441a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """ 451a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # We will want to keep all the datetime logic intact and so we need to 461a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # keep a reference to the unmocked datetime. 471a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.datetime = datetime.datetime 481a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.StubOutWithMock(datetime, 'datetime') 491a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood datetime.datetime.today().AndReturn(self.datetime(2012, 1, 1)) 501a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60 511a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 521a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood mail.send( 531a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 'chromeos-test-health@google.com', 541a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood ['chromeos-lab-infrastructure@google.com'], 551a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood [], 561a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 'Long Failing Tests', 571a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 'The following tests have been failing for at ' 581a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 'least %i days:\n\ntest' 591a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood % complete_failures._DAYS_TO_BE_FAILING_TOO_LONG) 601a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 611a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood storage = {} 621a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 631a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # The ReplayAll is required or else a mox object sneaks its way into 641a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # the storage object somehow. 651a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.ReplayAll() 661a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures.email_about_test_failure( 671a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood {'test': datetime.datetime.min}, storage) 681a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 691a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.assertEqual(storage['test'], self.datetime(2012, 1, 1)) 701a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.VerifyAll() 711a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 721a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 731a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def test_remove_test_if_it_has_succeeded_recently_enough(self): 741a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """Test that we remove a passing test from the storage object.""" 751a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood storage = {'test': datetime.datetime(2012, 1, 1)} 761a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60 771a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood today = datetime.datetime(2012, 4, 10) 781a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood safe_date = datetime.datetime(2012, 4, 9) 791a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 801a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.StubOutWithMock(datetime, 'datetime') 811a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood datetime.datetime.today().AndReturn(today) 821a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 831a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.ReplayAll() 841a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures.email_about_test_failure({'test': safe_date}, storage) 851a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 861a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.assertTrue('test' not in storage) 871a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.VerifyAll() 881a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 891a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 901a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def test_no_crashing_on_test_that_has_never_failed_for_too_long(self): 911a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """Test that we do not crash for tests that have always passed.""" 921a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood storage = {} 931a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60 941a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood today = datetime.datetime(2012,4,10) 951a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood safe_date = datetime.datetime(2012,4,9) 961a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 971a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.StubOutWithMock(datetime, 'datetime') 981a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood datetime.datetime.today().AndReturn(today) 991a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1001a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.ReplayAll() 1011a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures.email_about_test_failure({'test': safe_date}, storage) 1021a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1031a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.assertTrue('test' not in storage) 1041a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.VerifyAll() 1051a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1061a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1071a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def test_do_not_send_email_if_test_already_in_storage(self): 1081a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """Test only send emails on newly problematic tests.""" 1091a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood storage = {'test': datetime.datetime(2012, 1, 1)} 1101a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.datetime = datetime.datetime 1111a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.StubOutWithMock(datetime, 'datetime') 1121a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood datetime.datetime.today().AndReturn(self.datetime(2012, 1, 1)) 1131a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1141a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.ReplayAll() 1151a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures.email_about_test_failure( 1161a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood {'test': datetime.datetime.min}, storage) 1171a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1181a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.VerifyAll() 1191a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1201a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1211a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood def test_do_not_delete_if_still_failing(self): 1221a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood """Test that an old failing test is not removed from storage.""" 1231a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # We will want to keep all the datetime logic intact and so we need to 1241a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # keep a reference to the unmocked datetime. 1251a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.datetime = datetime.datetime 1261a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood today = datetime.datetime(2012, 1, 1) 1271a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.StubOutWithMock(datetime, 'datetime') 1281a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood datetime.datetime.today().AndReturn(today) 1291a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1301a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood storage = {'test': datetime.datetime.min} 1311a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1321a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # The ReplayAll is required or else a mox object sneaks its way into 1331a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood # the storage object somehow. 1341a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.ReplayAll() 1351a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood complete_failures.email_about_test_failure( 1361a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood {'test': datetime.datetime.min}, storage) 1371a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1381a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.assertTrue('test' in storage) 1391a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood self.mox.VerifyAll() 1401a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1411a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood 1421a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hoodif __name__ == '__main__': 1431a3c8dd59f71086eb2f5d08b0f13df95428e60faKeyar Hood unittest.main() 144