1// Copyright (C) 2011 Google Inc. All rights reserved. 2// 3// Redistribution and use in source and binary forms, with or without 4// modification, are permitted provided that the following conditions are 5// met: 6// 7// * Redistributions of source code must retain the above copyright 8// notice, this list of conditions and the following disclaimer. 9// * Redistributions in binary form must reproduce the above 10// copyright notice, this list of conditions and the following disclaimer 11// in the documentation and/or other materials provided with the 12// distribution. 13// * Neither the name of Google Inc. nor the names of its 14// contributors may be used to endorse or promote products derived from 15// this software without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29module('flakiness_dashboard'); 30 31// FIXME(jparent): Rename this once it isn't globals. 32function resetGlobals() 33{ 34 allExpectations = null; 35 g_resultsByBuilder = {}; 36 g_allTestsTrie = null; 37 var historyInstance = new history.History(flakinessConfig); 38 // FIXME(jparent): Remove this once global isn't used. 39 g_history = historyInstance; 40 g_testToResultsMap = {}; 41 42 for (var key in history.DEFAULT_CROSS_DASHBOARD_STATE_VALUES) 43 historyInstance.crossDashboardState[key] = history.DEFAULT_CROSS_DASHBOARD_STATE_VALUES[key]; 44 45 LOAD_BUILDBOT_DATA({ 46 'masters': [{ 47 name: 'ChromiumWebkit', 48 url: 'dummyurl', 49 tests: {'layout-tests': {'builders': ['WebKit Linux', 'WebKit Linux (dbg)', 'WebKit Linux (deps)', 'WebKit Mac10.7', 'WebKit Win', 'WebKit Win (dbg)']}}, 50 groups: ['@ToT Chromium', '@ToT Blink'], 51 },{ 52 name :'ChromiumWin', 53 url: 'dummyurl2', 54 tests: {'ash_unittests': {'builders': ['XP Tests (1)', 'Win7 Tests (1)']}}, 55 groups: ['@ToT Chromium'], 56 }], 57 }); 58 59 return historyInstance; 60} 61 62var FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "MISSING", 63 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT"} 64 65test('splitTestList', 1, function() { 66 var historyInstance = new history.History(flakinessConfig); 67 // FIXME(jparent): Remove this once global isn't used. 68 g_history = historyInstance; 69 historyInstance.dashboardSpecificState.tests = 'test.foo test.foo1\ntest.foo2\ntest.foo3,foo\\bar\\baz.html'; 70 equal(splitTestList().toString(), 'test.foo,test.foo1,test.foo2,test.foo3,foo/bar/baz.html'); 71}); 72 73test('headerForTestTableHtml', 1, function() { 74 var container = document.createElement('div'); 75 container.innerHTML = headerForTestTableHtml(); 76 equal(container.querySelectorAll('input').length, 4); 77}); 78 79test('htmlForTestTypeSwitcherGroup', 6, function() { 80 resetGlobals(); 81 var historyInstance = new history.History(flakinessConfig); 82 // FIXME(jparent): Remove this once global isn't used. 83 g_history = historyInstance; 84 var container = document.createElement('div'); 85 historyInstance.crossDashboardState.testType = 'ash_unittests'; 86 container.innerHTML = ui.html.testTypeSwitcher(true); 87 var selects = container.querySelectorAll('select'); 88 equal(selects.length, 2); 89 var group = selects[1]; 90 equal(group.parentNode.textContent.indexOf('Group:'), 0); 91 equal(group.children.length, 1); 92 93 historyInstance.crossDashboardState.testType = 'layout-tests'; 94 container.innerHTML = ui.html.testTypeSwitcher(true); 95 var selects = container.querySelectorAll('select'); 96 equal(selects.length, 2); 97 var group = selects[1]; 98 equal(group.parentNode.textContent.indexOf('Group:'), 0); 99 equal(group.children.length, 2); 100}); 101 102test('htmlForIndividualTestOnAllBuilders', 1, function() { 103 resetGlobals(); 104 builders.loadBuildersList('@ToT Blink', 'layout-tests'); 105 equal(htmlForIndividualTestOnAllBuilders('foo/nonexistant.html'), '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all recorded runs.</div>'); 106}); 107 108test('htmlForIndividualTestOnAllBuildersWithResultsLinksNonexistant', 1, function() { 109 resetGlobals(); 110 builders.loadBuildersList('@ToT Blink', 'layout-tests'); 111 equal(htmlForIndividualTestOnAllBuildersWithResultsLinks('foo/nonexistant.html'), 112 '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all recorded runs.</div>' + 113 '<div class=expectations test=foo/nonexistant.html>' + 114 '<div>' + 115 '<span class=link onclick="g_history.setQueryParameter(\'showExpectations\', true)">Show results</span> | ' + 116 '<span class=link onclick="g_history.setQueryParameter(\'showLargeExpectations\', true)">Show large thumbnails</span> | ' + 117 '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b>' + 118 '</div>' + 119 '</div>'); 120}); 121 122test('htmlForIndividualTestOnAllBuildersWithResultsLinks', 1, function() { 123 resetGlobals(); 124 builders.loadBuildersList('@ToT Blink', 'layout-tests'); 125 126 var builderName = 'WebKit Linux'; 127 g_resultsByBuilder[builderName] = {buildNumbers: [2, 1], blinkRevision: [1234, 1233], failure_map: FAILURE_MAP}; 128 129 var test = 'dummytest.html'; 130 var resultsObject = createResultsObjectForTest(test, builderName); 131 resultsObject.rawResults = [[1, 'F']]; 132 resultsObject.rawTimes = [[1, 0]]; 133 resultsObject.bugs = ["crbug.com/1234", "webkit.org/5678"]; 134 g_testToResultsMap[test] = [resultsObject]; 135 136 equal(htmlForIndividualTestOnAllBuildersWithResultsLinks(test), 137 '<table class=test-table><thead><tr>' + 138 '<th sortValue=test><div class=table-header-content><span></span><span class=header-text>test</span></div></th>' + 139 '<th sortValue=bugs><div class=table-header-content><span></span><span class=header-text>bugs</span></div></th>' + 140 '<th sortValue=expectations><div class=table-header-content><span></span><span class=header-text>expectations</span></div></th>' + 141 '<th sortValue=slowest><div class=table-header-content><span></span><span class=header-text>slowest run</span></div></th>' + 142 '<th sortValue=flakiness colspan=10000><div class=table-header-content><span></span><span class=header-text>flakiness (numbers are runtimes in seconds)</span></div></th>' + 143 '</tr></thead>' + 144 '<tbody><tr>' + 145 '<td class="test-link builder-name">WebKit Linux' + 146 '<td class=options-container>' + 147 '<div><a href="http://crbug.com/1234">crbug.com/1234</a></div>' + 148 '<div><a href="http://webkit.org/5678">webkit.org/5678</a></div>' + 149 '<td class=options-container><td><td title="TEXT. Click for more info." class="results TEXT" onclick=\'showPopupForBuild(event, "WebKit Linux",0,"dummytest.html")\'> ' + 150 '<td title="NO DATA. Click for more info." class="results NODATA" onclick=\'showPopupForBuild(event, "WebKit Linux",1,"dummytest.html")\'> ' + 151 '</tbody>' + 152 '</table>' + 153 '<div>The following builders either don\'t run this test (e.g. it\'s skipped) or all recorded runs passed:</div>' + 154 '<div class=skipped-builder-list>' + 155 '<div class=skipped-builder>WebKit Linux (dbg)</div><div class=skipped-builder>WebKit Mac10.7</div><div class=skipped-builder>WebKit Win</div><div class=skipped-builder>WebKit Win (dbg)</div>' + 156 '</div>' + 157 '<div class=expectations test=dummytest.html>' + 158 '<div><span class=link onclick="g_history.setQueryParameter(\'showExpectations\', true)">Show results</span> | ' + 159 '<span class=link onclick="g_history.setQueryParameter(\'showLargeExpectations\', true)">Show large thumbnails</span> | ' + 160 '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b></div>' + 161 '</div>'); 162}); 163 164test('individualTestsForSubstringList', 2, function() { 165 var builderName = 'WebKit Linux'; 166 g_resultsByBuilder[builderName] = { 167 buildNumbers: [2, 1], 168 blinkRevision: [1234, 1233], 169 failure_map: FAILURE_MAP, 170 tests: { 171 'foo/one.html': { results: [1, 'F'], times: [1, 0] }, 172 'virtual/foo/one.html': { results: [1, 'F'], times: [1, 0] }, 173 } 174 }; 175 176 g_history.dashboardSpecificState.showChrome = true; 177 var testToMatch = 'foo/one.html'; 178 g_history.dashboardSpecificState.tests = testToMatch; 179 deepEqual(individualTestsForSubstringList(), [testToMatch, 'virtual/foo/one.html']); 180 181 g_history.dashboardSpecificState.showChrome = false; 182 deepEqual(individualTestsForSubstringList(), [testToMatch]); 183}); 184 185test('htmlForIndividualTest', 2, function() { 186 var historyInstance = resetGlobals(); 187 builders.loadBuildersList('@ToT Blink', 'layout-tests'); 188 var test = 'foo/nonexistant.html'; 189 190 historyInstance.dashboardSpecificState.showChrome = false; 191 192 equal(htmlForIndividualTest(test), htmlForIndividualTestOnAllBuilders(test) + 193 '<div class=expectations test=foo/nonexistant.html>' + 194 '<div><span class=link onclick=\"g_history.setQueryParameter(\'showExpectations\', true)\">Show results</span> | ' + 195 '<span class=link onclick=\"g_history.setQueryParameter(\'showLargeExpectations\', true)\">Show large thumbnails</span> | ' + 196 '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b></div>' + 197 '</div>'); 198 199 historyInstance.dashboardSpecificState.showChrome = true; 200 201 equal(htmlForIndividualTest(test), 202 '<h2><a href="' + TEST_URL_BASE_PATH_FOR_BROWSING + 'foo/nonexistant.html" target="_blank">foo/nonexistant.html</a></h2>' + 203 htmlForIndividualTestOnAllBuildersWithResultsLinks(test)); 204}); 205 206test('linkifyBugs', 4, function() { 207 equal(linkifyBugs(["crbug.com/1234", "webkit.org/5678"]), 208 '<div><a href="http://crbug.com/1234">crbug.com/1234</a></div><div><a href="http://webkit.org/5678">webkit.org/5678</a></div>'); 209 equal(linkifyBugs(["crbug.com/1234"]), '<div><a href="http://crbug.com/1234">crbug.com/1234</a></div>'); 210 equal(linkifyBugs(["Bug(nick)"]), '<div>Bug(nick)</div>'); 211 equal(linkifyBugs([]), ''); 212}); 213 214test('htmlForSingleTestRow', 1, function() { 215 var historyInstance = resetGlobals(); 216 var builder = 'dummyBuilder'; 217 var test = createResultsObjectForTest('foo/exists.html', builder); 218 historyInstance.dashboardSpecificState.showNonFlaky = true; 219 g_resultsByBuilder[builder] = {buildNumbers: [2, 1], blinkRevision: [1234, 1233], failure_map: FAILURE_MAP}; 220 test.rawResults = [[1, 'F'], [2, 'I']]; 221 test.rawTimes = [[1, 0], [2, 5]]; 222 var expected = '<tr>' + 223 '<td class="test-link"><span class="link" onclick="g_history.setQueryParameter(\'tests\',\'foo/exists.html\');">foo/exists.html</span>' + 224 '<td class=options-container><a href="https://code.google.com/p/chromium/issues/entry?template=Layout%20Test%20Failure&summary=Layout%20Test%20foo%2Fexists.html%20is%20failing&comment=The%20following%20layout%20test%20is%20failing%20on%20%5Binsert%20platform%5D%0A%0Afoo%2Fexists.html%0A%0AProbable%20cause%3A%0A%0A%5Binsert%20probable%20cause%5D">File new bug</a>' + 225 '<td class=options-container>' + 226 '<td>' + 227 '<td title="TEXT. Click for more info." class="results TEXT" onclick=\'showPopupForBuild(event, "dummyBuilder",0,"foo/exists.html")\'> ' + 228 '<td title="IMAGE. Click for more info." class="results IMAGE" onclick=\'showPopupForBuild(event, "dummyBuilder",1,"foo/exists.html")\'>5'; 229 equal(htmlForSingleTestRow(test), expected); 230}); 231 232test('lookupVirtualTestSuite', 2, function() { 233 equal(lookupVirtualTestSuite('fast/canvas/foo.html'), ''); 234 equal(lookupVirtualTestSuite('virtual/gpu/fast/canvas/foo.html'), 'virtual/gpu/fast/canvas'); 235}); 236 237test('baseTest', 2, function() { 238 equal(baseTest('fast/canvas/foo.html', ''), 'fast/canvas/foo.html'); 239 equal(baseTest('virtual/gpu/fast/canvas/foo.html', 'virtual/gpu/fast/canvas'), 'fast/canvas/foo.html'); 240}); 241 242test('sortTests', 4, function() { 243 var test1 = createResultsObjectForTest('foo/test1.html', 'dummyBuilder'); 244 var test2 = createResultsObjectForTest('foo/test2.html', 'dummyBuilder'); 245 var test3 = createResultsObjectForTest('foo/test3.html', 'dummyBuilder'); 246 test1.expectations = 'b'; 247 test2.expectations = 'a'; 248 test3.expectations = ''; 249 250 var tests = [test1, test2, test3]; 251 sortTests(tests, 'expectations', FORWARD); 252 deepEqual(tests, [test2, test1, test3]); 253 sortTests(tests, 'expectations', BACKWARD); 254 deepEqual(tests, [test3, test1, test2]); 255 256 test1.bugs = 'b'; 257 test2.bugs = 'a'; 258 test3.bugs = ''; 259 260 var tests = [test1, test2, test3]; 261 sortTests(tests, 'bugs', FORWARD); 262 deepEqual(tests, [test2, test1, test3]); 263 sortTests(tests, 'bugs', BACKWARD); 264 deepEqual(tests, [test3, test1, test2]); 265}); 266 267test('popup', 2, function() { 268 ui.popup.show(document.body, 'dummy content'); 269 ok(document.querySelector('#popup')); 270 ui.popup.hide(); 271 ok(!document.querySelector('#popup')); 272}); 273 274test('gpuResultsPath', 3, function() { 275 equal(gpuResultsPath('777777', 'Win7 Release (ATI)'), '777777_Win7_Release_ATI_'); 276 equal(gpuResultsPath('123', 'GPU Linux (dbg)(NVIDIA)'), '123_GPU_Linux_dbg_NVIDIA_'); 277 equal(gpuResultsPath('12345', 'GPU Mac'), '12345_GPU_Mac'); 278}); 279 280test('TestTrie', 3, function() { 281 var builders = { 282 "Dummy Chromium Windows Builder": true, 283 "Dummy GTK Linux Builder": true, 284 "Dummy Apple Mac Lion Builder": true 285 }; 286 287 var resultsByBuilder = { 288 "Dummy Chromium Windows Builder": { 289 tests: { 290 "foo": true, 291 "foo/bar/1.html": true, 292 "foo/bar/baz": true 293 } 294 }, 295 "Dummy GTK Linux Builder": { 296 tests: { 297 "bar": true, 298 "foo/1.html": true, 299 "foo/bar/2.html": true, 300 "foo/bar/baz/1.html": true, 301 } 302 }, 303 "Dummy Apple Mac Lion Builder": { 304 tests: { 305 "foo/bar/3.html": true, 306 "foo/bar/baz/foo": true, 307 } 308 } 309 }; 310 var expectedTrie = { 311 "foo": { 312 "bar": { 313 "1.html": true, 314 "2.html": true, 315 "3.html": true, 316 "baz": { 317 "1.html": true, 318 "foo": true 319 } 320 }, 321 "1.html": true 322 }, 323 "bar": true 324 } 325 326 var trie = new TestTrie(builders, resultsByBuilder); 327 deepEqual(trie._trie, expectedTrie); 328 329 var leafsOfCompleteTrieTraversal = []; 330 var expectedLeafs = ["foo/bar/1.html", "foo/bar/baz/1.html", "foo/bar/baz/foo", "foo/bar/2.html", "foo/bar/3.html", "foo/1.html", "bar"]; 331 trie.forEach(function(triePath) { 332 leafsOfCompleteTrieTraversal.push(triePath); 333 }); 334 deepEqual(leafsOfCompleteTrieTraversal, expectedLeafs); 335 336 var leafsOfPartialTrieTraversal = []; 337 expectedLeafs = ["foo/bar/1.html", "foo/bar/baz/1.html", "foo/bar/baz/foo", "foo/bar/2.html", "foo/bar/3.html"]; 338 trie.forEach(function(triePath) { 339 leafsOfPartialTrieTraversal.push(triePath); 340 }, "foo/bar"); 341 deepEqual(leafsOfPartialTrieTraversal, expectedLeafs); 342}); 343 344test('changeTestTypeInvalidatesGroup', 1, function() { 345 var historyInstance = resetGlobals(); 346 var originalGroup = '@ToT Blink'; 347 var originalTestType = 'layout-tests'; 348 builders.loadBuildersList(originalGroup, originalTestType); 349 historyInstance.crossDashboardState.group = originalGroup; 350 historyInstance.crossDashboardState.testType = originalTestType; 351 352 historyInstance.invalidateQueryParameters({'testType': 'ui_tests'}); 353 notEqual(historyInstance.crossDashboardState.group, originalGroup, "group should have been invalidated"); 354}); 355 356test('shouldShowTest', 9, function() { 357 var historyInstance = new history.History(flakinessConfig); 358 historyInstance.parseParameters(); 359 // FIXME(jparent): Change to use the flakiness_dashboard's history object 360 // once it exists, rather than tracking global. 361 g_history = historyInstance; 362 var test = createResultsObjectForTest('foo/test.html', 'dummyBuilder'); 363 364 equal(shouldShowTest(test), false, 'default layout test, hide it.'); 365 historyInstance.dashboardSpecificState.showNonFlaky = true; 366 equal(shouldShowTest(test), true, 'show correct expectations.'); 367 historyInstance.dashboardSpecificState.showNonFlaky = false; 368 369 test = createResultsObjectForTest('foo/test.html', 'dummyBuilder'); 370 test.expectations = "WONTFIX"; 371 equal(shouldShowTest(test), false, 'by default hide wontfix'); 372 historyInstance.dashboardSpecificState.showWontFix = true; 373 equal(shouldShowTest(test), true, 'show wontfix'); 374 historyInstance.dashboardSpecificState.showWontFix = false; 375 376 test = createResultsObjectForTest('foo/test.html', 'dummyBuilder'); 377 test.expectations = "SKIP"; 378 equal(shouldShowTest(test), false, 'we hide skip tests by default'); 379 historyInstance.dashboardSpecificState.showSkip = true; 380 equal(shouldShowTest(test), true, 'show skip test'); 381 historyInstance.dashboardSpecificState.showSkip = false; 382 383 test = createResultsObjectForTest('foo/test.html', 'dummyBuilder'); 384 test.isFlaky = true; 385 equal(shouldShowTest(test), false, 'hide flaky tests by default'); 386 historyInstance.dashboardSpecificState.showFlaky = true; 387 equal(shouldShowTest(test), true, 'show flaky test'); 388 historyInstance.dashboardSpecificState.showFlaky = false; 389 390 test = createResultsObjectForTest('foo/test.html', 'dummyBuilder'); 391 historyInstance.crossDashboardState.testType = 'not layout tests'; 392 equal(shouldShowTest(test), true, 'show all non layout tests'); 393}); 394