1# Copyright (C) 2009 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#    * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#    * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#    * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import unittest
30
31from webkitpy.common.net.layouttestresults import LayoutTestResults
32from webkitpy.common.net.buildbot import BuildBot, Builder, Build
33from webkitpy.layout_tests.layout_package import test_results
34from webkitpy.layout_tests.layout_package import test_failures
35from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
36
37
38class BuilderTest(unittest.TestCase):
39    def _mock_test_result(self, testname):
40        return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
41
42    def _install_fetch_build(self, failure):
43        def _mock_fetch_build(build_number):
44            build = Build(
45                builder=self.builder,
46                build_number=build_number,
47                revision=build_number + 1000,
48                is_green=build_number < 4
49            )
50            results = [self._mock_test_result(testname) for testname in failure(build_number)]
51            build._layout_test_results = LayoutTestResults(results)
52            return build
53        self.builder._fetch_build = _mock_fetch_build
54
55    def setUp(self):
56        self.buildbot = BuildBot()
57        self.builder = Builder(u"Test Builder \u2661", self.buildbot)
58        self._install_fetch_build(lambda build_number: ["test1", "test2"])
59
60    def test_find_regression_window(self):
61        regression_window = self.builder.find_regression_window(self.builder.build(10))
62        self.assertEqual(regression_window.build_before_failure().revision(), 1003)
63        self.assertEqual(regression_window.failing_build().revision(), 1004)
64
65        regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
66        self.assertEqual(regression_window.build_before_failure(), None)
67        self.assertEqual(regression_window.failing_build().revision(), 1008)
68
69    def test_none_build(self):
70        self.builder._fetch_build = lambda build_number: None
71        regression_window = self.builder.find_regression_window(self.builder.build(10))
72        self.assertEqual(regression_window.build_before_failure(), None)
73        self.assertEqual(regression_window.failing_build(), None)
74
75    def test_flaky_tests(self):
76        self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
77        regression_window = self.builder.find_regression_window(self.builder.build(10))
78        self.assertEqual(regression_window.build_before_failure().revision(), 1009)
79        self.assertEqual(regression_window.failing_build().revision(), 1010)
80
81    def test_failure_and_flaky(self):
82        self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
83        regression_window = self.builder.find_regression_window(self.builder.build(10))
84        self.assertEqual(regression_window.build_before_failure().revision(), 1003)
85        self.assertEqual(regression_window.failing_build().revision(), 1004)
86
87    def test_no_results(self):
88        self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
89        regression_window = self.builder.find_regression_window(self.builder.build(10))
90        self.assertEqual(regression_window.build_before_failure().revision(), 1003)
91        self.assertEqual(regression_window.failing_build().revision(), 1004)
92
93    def test_failure_after_flaky(self):
94        self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
95        regression_window = self.builder.find_regression_window(self.builder.build(10))
96        self.assertEqual(regression_window.build_before_failure().revision(), 1006)
97        self.assertEqual(regression_window.failing_build().revision(), 1007)
98
99    def test_find_blameworthy_regression_window(self):
100        self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
101        self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
102        # Flakey test avoidance requires at least 2 red builds:
103        self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
104        self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
105        # Green builder:
106        self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
107
108    def test_build_caching(self):
109        self.assertEqual(self.builder.build(10), self.builder.build(10))
110
111    def test_build_and_revision_for_filename(self):
112        expectations = {
113            "r47483 (1)/" : (47483, 1),
114            "r47483 (1).zip" : (47483, 1),
115        }
116        for filename, revision_and_build in expectations.items():
117            self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
118
119
120class BuildTest(unittest.TestCase):
121    def test_layout_test_results(self):
122        build = Build(None, None, None, None)
123        build._fetch_results_html = lambda: None
124        # Test that layout_test_results() returns None if the fetch fails.
125        self.assertEqual(build.layout_test_results(), None)
126
127
128class BuildBotTest(unittest.TestCase):
129
130    _example_one_box_status = '''
131    <table>
132    <tr>
133    <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td>
134      <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td>
135      <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td>
136    <tr>
137    <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td>
138      <td class="LastBuild box" >no build</td>
139      <td align="center" class="Activity building">building<br />< 1 min</td>
140    <tr>
141    <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td>
142      <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td>
143      <td align="center" class="Activity idle">idle<br />3 pending</td>
144    <tr>
145    <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td>
146      <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td>
147      <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td>
148    </table>
149'''
150    _expected_example_one_box_parsings = [
151        {
152            'is_green': True,
153            'build_number' : 3693,
154            'name': u'Windows Debug (Tests)',
155            'built_revision': 47380,
156            'activity': 'building',
157            'pending_builds': 0,
158        },
159        {
160            'is_green': False,
161            'build_number' : None,
162            'name': u'SnowLeopard Intel Release',
163            'built_revision': None,
164            'activity': 'building',
165            'pending_builds': 0,
166        },
167        {
168            'is_green': False,
169            'build_number' : 654,
170            'name': u'Qt Linux Release',
171            'built_revision': 47383,
172            'activity': 'idle',
173            'pending_builds': 3,
174        },
175        {
176            'is_green': True,
177            'build_number' : 2090,
178            'name': u'Qt Windows 32-bit Debug',
179            'built_revision': 60563,
180            'activity': 'building',
181            'pending_builds': 0,
182        },
183    ]
184
185    def test_status_parsing(self):
186        buildbot = BuildBot()
187
188        soup = BeautifulSoup(self._example_one_box_status)
189        status_table = soup.find("table")
190        input_rows = status_table.findAll('tr')
191
192        for x in range(len(input_rows)):
193            status_row = input_rows[x]
194            expected_parsing = self._expected_example_one_box_parsings[x]
195
196            builder = buildbot._parse_builder_status_from_row(status_row)
197
198            # Make sure we aren't parsing more or less than we expect
199            self.assertEquals(builder.keys(), expected_parsing.keys())
200
201            for key, expected_value in expected_parsing.items():
202                self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value)))
203
204    def test_core_builder_methods(self):
205        buildbot = BuildBot()
206
207        # Override builder_statuses function to not touch the network.
208        def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to.
209            return BuildBotTest._expected_example_one_box_parsings
210        buildbot.builder_statuses = example_builder_statuses
211
212        buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ]
213        self.assertEquals(buildbot.red_core_builders_names(), [])
214        self.assertTrue(buildbot.core_builders_are_green())
215
216        buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ]
217        self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ])
218        self.assertFalse(buildbot.core_builders_are_green())
219
220    def test_builder_name_regexps(self):
221        buildbot = BuildBot()
222
223        # For complete testing, this list should match the list of builders at build.webkit.org:
224        example_builders = [
225            {'name': u'Leopard Intel Release (Build)', },
226            {'name': u'Leopard Intel Release (Tests)', },
227            {'name': u'Leopard Intel Debug (Build)', },
228            {'name': u'Leopard Intel Debug (Tests)', },
229            {'name': u'SnowLeopard Intel Release (Build)', },
230            {'name': u'SnowLeopard Intel Release (Tests)', },
231            {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', },
232            {'name': u'SnowLeopard Intel Leaks', },
233            {'name': u'Windows Release (Build)', },
234            {'name': u'Windows 7 Release (Tests)', },
235            {'name': u'Windows Debug (Build)', },
236            {'name': u'Windows XP Debug (Tests)', },
237            {'name': u'Windows 7 Release (WebKit2 Tests)', },
238            {'name': u'GTK Linux 32-bit Release', },
239            {'name': u'GTK Linux 32-bit Debug', },
240            {'name': u'GTK Linux 64-bit Debug', },
241            {'name': u'Qt Linux Release', },
242            {'name': u'Qt Linux Release minimal', },
243            {'name': u'Qt Linux ARMv7 Release', },
244            {'name': u'Qt Windows 32-bit Release', },
245            {'name': u'Qt Windows 32-bit Debug', },
246            {'name': u'Chromium Win Release', },
247            {'name': u'Chromium Mac Release', },
248            {'name': u'Chromium Linux Release', },
249            {'name': u'Chromium Win Release (Tests)', },
250            {'name': u'Chromium Mac Release (Tests)', },
251            {'name': u'Chromium Linux Release (Tests)', },
252            {'name': u'New run-webkit-tests', },
253            {'name': u'WinCairo Debug (Build)', },
254            {'name': u'WinCE Release (Build)', },
255            {'name': u'EFL Linux Release (Build)', },
256        ]
257        name_regexps = [
258            "SnowLeopard.*Build",
259            "SnowLeopard.*\(Test",
260            "SnowLeopard.*\(WebKit2 Test",
261            "Leopard.*",
262            "Windows.*Build",
263            "Windows.*\(Test",
264            "WinCairo",
265            "WinCE",
266            "EFL",
267            "GTK.*32",
268            "GTK.*64.*Debug",  # Disallow the 64-bit Release bot which is broken.
269            "Qt",
270            "Chromium.*Release$",
271        ]
272        expected_builders = [
273            {'name': u'Leopard Intel Release (Build)', },
274            {'name': u'Leopard Intel Release (Tests)', },
275            {'name': u'Leopard Intel Debug (Build)', },
276            {'name': u'Leopard Intel Debug (Tests)', },
277            {'name': u'SnowLeopard Intel Release (Build)', },
278            {'name': u'SnowLeopard Intel Release (Tests)', },
279            {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', },
280            {'name': u'Windows Release (Build)', },
281            {'name': u'Windows 7 Release (Tests)', },
282            {'name': u'Windows Debug (Build)', },
283            {'name': u'Windows XP Debug (Tests)', },
284            {'name': u'GTK Linux 32-bit Release', },
285            {'name': u'GTK Linux 32-bit Debug', },
286            {'name': u'GTK Linux 64-bit Debug', },
287            {'name': u'Qt Linux Release', },
288            {'name': u'Qt Linux Release minimal', },
289            {'name': u'Qt Linux ARMv7 Release', },
290            {'name': u'Qt Windows 32-bit Release', },
291            {'name': u'Qt Windows 32-bit Debug', },
292            {'name': u'Chromium Win Release', },
293            {'name': u'Chromium Mac Release', },
294            {'name': u'Chromium Linux Release', },
295            {'name': u'WinCairo Debug (Build)', },
296            {'name': u'WinCE Release (Build)', },
297            {'name': u'EFL Linux Release (Build)', },
298        ]
299
300        # This test should probably be updated if the default regexp list changes
301        self.assertEquals(buildbot.core_builder_names_regexps, name_regexps)
302
303        builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps)
304        self.assertEquals(builders, expected_builders)
305
306    def test_builder_with_name(self):
307        buildbot = BuildBot()
308
309        builder = buildbot.builder_with_name("Test Builder")
310        self.assertEqual(builder.name(), "Test Builder")
311        self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder")
312        self.assertEqual(builder.url_encoded_name(), "Test%20Builder")
313        self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder")
314
315        # Override _fetch_build_dictionary function to not touch the network.
316        def mock_fetch_build_dictionary(self, build_number):
317            build_dictionary = {
318                "sourceStamp": {
319                    "revision" : 2 * build_number,
320                    },
321                "number" : int(build_number),
322                "results" : build_number % 2, # 0 means pass
323            }
324            return build_dictionary
325        buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
326
327        build = builder.build(10)
328        self.assertEqual(build.builder(), builder)
329        self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10")
330        self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29")
331        self.assertEqual(build.revision(), 20)
332        self.assertEqual(build.is_green(), True)
333
334        build = build.previous_build()
335        self.assertEqual(build.builder(), builder)
336        self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9")
337        self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29")
338        self.assertEqual(build.revision(), 18)
339        self.assertEqual(build.is_green(), False)
340
341        self.assertEqual(builder.build(None), None)
342
343    _example_directory_listing = '''
344<h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1>
345
346<table>
347        <tr class="alt">
348            <th>Filename</th>
349            <th>Size</th>
350            <th>Content type</th>
351            <th>Content encoding</th>
352        </tr>
353<tr class="directory ">
354    <td><a href="r47483%20%281%29/"><b>r47483 (1)/</b></a></td>
355    <td><b></b></td>
356    <td><b>[Directory]</b></td>
357    <td><b></b></td>
358</tr>
359<tr class="file alt">
360    <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td>
361    <td>89K</td>
362    <td>[application/zip]</td>
363    <td></td>
364</tr>
365'''
366    _expected_files = [
367        {
368            "filename" : "r47483 (1)/",
369            "size" : "",
370            "type" : "[Directory]",
371            "encoding" : "",
372        },
373        {
374            "filename" : "r47484 (2).zip",
375            "size" : "89K",
376            "type" : "[application/zip]",
377            "encoding" : "",
378        },
379    ]
380
381    def test_parse_build_to_revision_map(self):
382        buildbot = BuildBot()
383        files = buildbot._parse_twisted_directory_listing(self._example_directory_listing)
384        self.assertEqual(self._expected_files, files)
385
386    # Revision, is_green
387    # Ordered from newest (highest number) to oldest.
388    fake_builder1 = [
389        [2, False],
390        [1, True],
391    ]
392    fake_builder2 = [
393        [2, False],
394        [1, True],
395    ]
396    fake_builders = [
397        fake_builder1,
398        fake_builder2,
399    ]
400    def _build_from_fake(self, fake_builder, index):
401        if index >= len(fake_builder):
402            return None
403        fake_build = fake_builder[index]
404        build = Build(
405            builder=fake_builder,
406            build_number=index,
407            revision=fake_build[0],
408            is_green=fake_build[1],
409        )
410        def mock_previous_build():
411            return self._build_from_fake(fake_builder, index + 1)
412        build.previous_build = mock_previous_build
413        return build
414
415    def _fake_builds_at_index(self, index):
416        return [self._build_from_fake(builder, index) for builder in self.fake_builders]
417
418    def test_last_green_revision(self):
419        buildbot = BuildBot()
420        def mock_builds_from_builders(only_core_builders):
421            return self._fake_builds_at_index(0)
422        buildbot._latest_builds_from_builders = mock_builds_from_builders
423        self.assertEqual(buildbot.last_green_revision(), 1)
424
425    def _fetch_build(self, build_number):
426        if build_number == 5:
427            return "correct build"
428        return "wrong build"
429
430    def _fetch_revision_to_build_map(self):
431        return {'r5': 5, 'r2': 2, 'r3': 3}
432
433    def test_latest_cached_build(self):
434        b = Builder('builder', BuildBot())
435        b._fetch_build = self._fetch_build
436        b._fetch_revision_to_build_map = self._fetch_revision_to_build_map
437        self.assertEquals("correct build", b.latest_cached_build())
438
439    def results_url(self):
440        return "some-url"
441
442    def test_results_zip_url(self):
443        b = Build(None, 123, 123, False)
444        b.results_url = self.results_url
445        self.assertEquals("some-url.zip", b.results_zip_url())
446
447    def test_results(self):
448        builder = Builder('builder', BuildBot())
449        b = Build(builder, 123, 123, True)
450        self.assertTrue(b.results())
451
452
453if __name__ == '__main__':
454    unittest.main()
455