1# Copyright (C) 2009 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import unittest 30 31from webkitpy.common.net.layouttestresults import LayoutTestResults 32from webkitpy.common.net.buildbot import BuildBot, Builder, Build 33from webkitpy.layout_tests.layout_package import test_results 34from webkitpy.layout_tests.layout_package import test_failures 35from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup 36 37 38class BuilderTest(unittest.TestCase): 39 def _mock_test_result(self, testname): 40 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 41 42 def _install_fetch_build(self, failure): 43 def _mock_fetch_build(build_number): 44 build = Build( 45 builder=self.builder, 46 build_number=build_number, 47 revision=build_number + 1000, 48 is_green=build_number < 4 49 ) 50 results = [self._mock_test_result(testname) for testname in failure(build_number)] 51 build._layout_test_results = LayoutTestResults(results) 52 return build 53 self.builder._fetch_build = _mock_fetch_build 54 55 def setUp(self): 56 self.buildbot = BuildBot() 57 self.builder = Builder(u"Test Builder \u2661", self.buildbot) 58 self._install_fetch_build(lambda build_number: ["test1", "test2"]) 59 60 def test_find_regression_window(self): 61 regression_window = self.builder.find_regression_window(self.builder.build(10)) 62 self.assertEqual(regression_window.build_before_failure().revision(), 1003) 63 self.assertEqual(regression_window.failing_build().revision(), 1004) 64 65 regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2) 66 self.assertEqual(regression_window.build_before_failure(), None) 67 self.assertEqual(regression_window.failing_build().revision(), 1008) 68 69 def test_none_build(self): 70 self.builder._fetch_build = lambda build_number: None 71 regression_window = self.builder.find_regression_window(self.builder.build(10)) 72 self.assertEqual(regression_window.build_before_failure(), None) 73 self.assertEqual(regression_window.failing_build(), None) 74 75 def test_flaky_tests(self): 76 self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) 77 regression_window = self.builder.find_regression_window(self.builder.build(10)) 78 self.assertEqual(regression_window.build_before_failure().revision(), 1009) 79 self.assertEqual(regression_window.failing_build().revision(), 1010) 80 81 def test_failure_and_flaky(self): 82 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) 83 regression_window = self.builder.find_regression_window(self.builder.build(10)) 84 self.assertEqual(regression_window.build_before_failure().revision(), 1003) 85 self.assertEqual(regression_window.failing_build().revision(), 1004) 86 87 def test_no_results(self): 88 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) 89 regression_window = self.builder.find_regression_window(self.builder.build(10)) 90 self.assertEqual(regression_window.build_before_failure().revision(), 1003) 91 self.assertEqual(regression_window.failing_build().revision(), 1004) 92 93 def test_failure_after_flaky(self): 94 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) 95 regression_window = self.builder.find_regression_window(self.builder.build(10)) 96 self.assertEqual(regression_window.build_before_failure().revision(), 1006) 97 self.assertEqual(regression_window.failing_build().revision(), 1007) 98 99 def test_find_blameworthy_regression_window(self): 100 self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004]) 101 self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None) 102 # Flakey test avoidance requires at least 2 red builds: 103 self.assertEqual(self.builder.find_blameworthy_regression_window(4), None) 104 self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004]) 105 # Green builder: 106 self.assertEqual(self.builder.find_blameworthy_regression_window(3), None) 107 108 def test_build_caching(self): 109 self.assertEqual(self.builder.build(10), self.builder.build(10)) 110 111 def test_build_and_revision_for_filename(self): 112 expectations = { 113 "r47483 (1)/" : (47483, 1), 114 "r47483 (1).zip" : (47483, 1), 115 } 116 for filename, revision_and_build in expectations.items(): 117 self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) 118 119 120class BuildTest(unittest.TestCase): 121 def test_layout_test_results(self): 122 build = Build(None, None, None, None) 123 build._fetch_results_html = lambda: None 124 # Test that layout_test_results() returns None if the fetch fails. 125 self.assertEqual(build.layout_test_results(), None) 126 127 128class BuildBotTest(unittest.TestCase): 129 130 _example_one_box_status = ''' 131 <table> 132 <tr> 133 <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> 134 <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> 135 <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> 136 <tr> 137 <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> 138 <td class="LastBuild box" >no build</td> 139 <td align="center" class="Activity building">building<br />< 1 min</td> 140 <tr> 141 <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> 142 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> 143 <td align="center" class="Activity idle">idle<br />3 pending</td> 144 <tr> 145 <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td> 146 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td> 147 <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td> 148 </table> 149''' 150 _expected_example_one_box_parsings = [ 151 { 152 'is_green': True, 153 'build_number' : 3693, 154 'name': u'Windows Debug (Tests)', 155 'built_revision': 47380, 156 'activity': 'building', 157 'pending_builds': 0, 158 }, 159 { 160 'is_green': False, 161 'build_number' : None, 162 'name': u'SnowLeopard Intel Release', 163 'built_revision': None, 164 'activity': 'building', 165 'pending_builds': 0, 166 }, 167 { 168 'is_green': False, 169 'build_number' : 654, 170 'name': u'Qt Linux Release', 171 'built_revision': 47383, 172 'activity': 'idle', 173 'pending_builds': 3, 174 }, 175 { 176 'is_green': True, 177 'build_number' : 2090, 178 'name': u'Qt Windows 32-bit Debug', 179 'built_revision': 60563, 180 'activity': 'building', 181 'pending_builds': 0, 182 }, 183 ] 184 185 def test_status_parsing(self): 186 buildbot = BuildBot() 187 188 soup = BeautifulSoup(self._example_one_box_status) 189 status_table = soup.find("table") 190 input_rows = status_table.findAll('tr') 191 192 for x in range(len(input_rows)): 193 status_row = input_rows[x] 194 expected_parsing = self._expected_example_one_box_parsings[x] 195 196 builder = buildbot._parse_builder_status_from_row(status_row) 197 198 # Make sure we aren't parsing more or less than we expect 199 self.assertEquals(builder.keys(), expected_parsing.keys()) 200 201 for key, expected_value in expected_parsing.items(): 202 self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) 203 204 def test_core_builder_methods(self): 205 buildbot = BuildBot() 206 207 # Override builder_statuses function to not touch the network. 208 def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. 209 return BuildBotTest._expected_example_one_box_parsings 210 buildbot.builder_statuses = example_builder_statuses 211 212 buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] 213 self.assertEquals(buildbot.red_core_builders_names(), []) 214 self.assertTrue(buildbot.core_builders_are_green()) 215 216 buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] 217 self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) 218 self.assertFalse(buildbot.core_builders_are_green()) 219 220 def test_builder_name_regexps(self): 221 buildbot = BuildBot() 222 223 # For complete testing, this list should match the list of builders at build.webkit.org: 224 example_builders = [ 225 {'name': u'Leopard Intel Release (Build)', }, 226 {'name': u'Leopard Intel Release (Tests)', }, 227 {'name': u'Leopard Intel Debug (Build)', }, 228 {'name': u'Leopard Intel Debug (Tests)', }, 229 {'name': u'SnowLeopard Intel Release (Build)', }, 230 {'name': u'SnowLeopard Intel Release (Tests)', }, 231 {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', }, 232 {'name': u'SnowLeopard Intel Leaks', }, 233 {'name': u'Windows Release (Build)', }, 234 {'name': u'Windows 7 Release (Tests)', }, 235 {'name': u'Windows Debug (Build)', }, 236 {'name': u'Windows XP Debug (Tests)', }, 237 {'name': u'Windows 7 Release (WebKit2 Tests)', }, 238 {'name': u'GTK Linux 32-bit Release', }, 239 {'name': u'GTK Linux 32-bit Debug', }, 240 {'name': u'GTK Linux 64-bit Debug', }, 241 {'name': u'Qt Linux Release', }, 242 {'name': u'Qt Linux Release minimal', }, 243 {'name': u'Qt Linux ARMv7 Release', }, 244 {'name': u'Qt Windows 32-bit Release', }, 245 {'name': u'Qt Windows 32-bit Debug', }, 246 {'name': u'Chromium Win Release', }, 247 {'name': u'Chromium Mac Release', }, 248 {'name': u'Chromium Linux Release', }, 249 {'name': u'Chromium Win Release (Tests)', }, 250 {'name': u'Chromium Mac Release (Tests)', }, 251 {'name': u'Chromium Linux Release (Tests)', }, 252 {'name': u'New run-webkit-tests', }, 253 {'name': u'WinCairo Debug (Build)', }, 254 {'name': u'WinCE Release (Build)', }, 255 {'name': u'EFL Linux Release (Build)', }, 256 ] 257 name_regexps = [ 258 "SnowLeopard.*Build", 259 "SnowLeopard.*\(Test", 260 "SnowLeopard.*\(WebKit2 Test", 261 "Leopard.*", 262 "Windows.*Build", 263 "Windows.*\(Test", 264 "WinCairo", 265 "WinCE", 266 "EFL", 267 "GTK.*32", 268 "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. 269 "Qt", 270 "Chromium.*Release$", 271 ] 272 expected_builders = [ 273 {'name': u'Leopard Intel Release (Build)', }, 274 {'name': u'Leopard Intel Release (Tests)', }, 275 {'name': u'Leopard Intel Debug (Build)', }, 276 {'name': u'Leopard Intel Debug (Tests)', }, 277 {'name': u'SnowLeopard Intel Release (Build)', }, 278 {'name': u'SnowLeopard Intel Release (Tests)', }, 279 {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', }, 280 {'name': u'Windows Release (Build)', }, 281 {'name': u'Windows 7 Release (Tests)', }, 282 {'name': u'Windows Debug (Build)', }, 283 {'name': u'Windows XP Debug (Tests)', }, 284 {'name': u'GTK Linux 32-bit Release', }, 285 {'name': u'GTK Linux 32-bit Debug', }, 286 {'name': u'GTK Linux 64-bit Debug', }, 287 {'name': u'Qt Linux Release', }, 288 {'name': u'Qt Linux Release minimal', }, 289 {'name': u'Qt Linux ARMv7 Release', }, 290 {'name': u'Qt Windows 32-bit Release', }, 291 {'name': u'Qt Windows 32-bit Debug', }, 292 {'name': u'Chromium Win Release', }, 293 {'name': u'Chromium Mac Release', }, 294 {'name': u'Chromium Linux Release', }, 295 {'name': u'WinCairo Debug (Build)', }, 296 {'name': u'WinCE Release (Build)', }, 297 {'name': u'EFL Linux Release (Build)', }, 298 ] 299 300 # This test should probably be updated if the default regexp list changes 301 self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) 302 303 builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) 304 self.assertEquals(builders, expected_builders) 305 306 def test_builder_with_name(self): 307 buildbot = BuildBot() 308 309 builder = buildbot.builder_with_name("Test Builder") 310 self.assertEqual(builder.name(), "Test Builder") 311 self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder") 312 self.assertEqual(builder.url_encoded_name(), "Test%20Builder") 313 self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder") 314 315 # Override _fetch_build_dictionary function to not touch the network. 316 def mock_fetch_build_dictionary(self, build_number): 317 build_dictionary = { 318 "sourceStamp": { 319 "revision" : 2 * build_number, 320 }, 321 "number" : int(build_number), 322 "results" : build_number % 2, # 0 means pass 323 } 324 return build_dictionary 325 buildbot._fetch_build_dictionary = mock_fetch_build_dictionary 326 327 build = builder.build(10) 328 self.assertEqual(build.builder(), builder) 329 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10") 330 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29") 331 self.assertEqual(build.revision(), 20) 332 self.assertEqual(build.is_green(), True) 333 334 build = build.previous_build() 335 self.assertEqual(build.builder(), builder) 336 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9") 337 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29") 338 self.assertEqual(build.revision(), 18) 339 self.assertEqual(build.is_green(), False) 340 341 self.assertEqual(builder.build(None), None) 342 343 _example_directory_listing = ''' 344<h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1> 345 346<table> 347 <tr class="alt"> 348 <th>Filename</th> 349 <th>Size</th> 350 <th>Content type</th> 351 <th>Content encoding</th> 352 </tr> 353<tr class="directory "> 354 <td><a href="r47483%20%281%29/"><b>r47483 (1)/</b></a></td> 355 <td><b></b></td> 356 <td><b>[Directory]</b></td> 357 <td><b></b></td> 358</tr> 359<tr class="file alt"> 360 <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td> 361 <td>89K</td> 362 <td>[application/zip]</td> 363 <td></td> 364</tr> 365''' 366 _expected_files = [ 367 { 368 "filename" : "r47483 (1)/", 369 "size" : "", 370 "type" : "[Directory]", 371 "encoding" : "", 372 }, 373 { 374 "filename" : "r47484 (2).zip", 375 "size" : "89K", 376 "type" : "[application/zip]", 377 "encoding" : "", 378 }, 379 ] 380 381 def test_parse_build_to_revision_map(self): 382 buildbot = BuildBot() 383 files = buildbot._parse_twisted_directory_listing(self._example_directory_listing) 384 self.assertEqual(self._expected_files, files) 385 386 # Revision, is_green 387 # Ordered from newest (highest number) to oldest. 388 fake_builder1 = [ 389 [2, False], 390 [1, True], 391 ] 392 fake_builder2 = [ 393 [2, False], 394 [1, True], 395 ] 396 fake_builders = [ 397 fake_builder1, 398 fake_builder2, 399 ] 400 def _build_from_fake(self, fake_builder, index): 401 if index >= len(fake_builder): 402 return None 403 fake_build = fake_builder[index] 404 build = Build( 405 builder=fake_builder, 406 build_number=index, 407 revision=fake_build[0], 408 is_green=fake_build[1], 409 ) 410 def mock_previous_build(): 411 return self._build_from_fake(fake_builder, index + 1) 412 build.previous_build = mock_previous_build 413 return build 414 415 def _fake_builds_at_index(self, index): 416 return [self._build_from_fake(builder, index) for builder in self.fake_builders] 417 418 def test_last_green_revision(self): 419 buildbot = BuildBot() 420 def mock_builds_from_builders(only_core_builders): 421 return self._fake_builds_at_index(0) 422 buildbot._latest_builds_from_builders = mock_builds_from_builders 423 self.assertEqual(buildbot.last_green_revision(), 1) 424 425 def _fetch_build(self, build_number): 426 if build_number == 5: 427 return "correct build" 428 return "wrong build" 429 430 def _fetch_revision_to_build_map(self): 431 return {'r5': 5, 'r2': 2, 'r3': 3} 432 433 def test_latest_cached_build(self): 434 b = Builder('builder', BuildBot()) 435 b._fetch_build = self._fetch_build 436 b._fetch_revision_to_build_map = self._fetch_revision_to_build_map 437 self.assertEquals("correct build", b.latest_cached_build()) 438 439 def results_url(self): 440 return "some-url" 441 442 def test_results_zip_url(self): 443 b = Build(None, 123, 123, False) 444 b.results_url = self.results_url 445 self.assertEquals("some-url.zip", b.results_zip_url()) 446 447 def test_results(self): 448 builder = Builder('builder', BuildBot()) 449 b = Build(builder, 123, 123, True) 450 self.assertTrue(b.results()) 451 452 453if __name__ == '__main__': 454 unittest.main() 455