15267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// Copyright (C) 2013 Google Inc. All rights reserved. 25267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// 35267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// Redistribution and use in source and binary forms, with or without 45267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// modification, are permitted provided that the following conditions are 55267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// met: 65267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// 75267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// * Redistributions of source code must retain the above copyright 85267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// notice, this list of conditions and the following disclaimer. 95267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// * Redistributions in binary form must reproduce the above 105267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// copyright notice, this list of conditions and the following disclaimer 115267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// in the documentation and/or other materials provided with the 125267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// distribution. 135267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// * Neither the name of Google Inc. nor the names of its 145267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// contributors may be used to endorse or promote products derived from 155267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// this software without specific prior written permission. 165267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// 175267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 295267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)var overview = overview || {}; 305267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 315267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)(function() { 325267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 335267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview._resultsByTestType = {}; 345267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview._testTypeIndex = 0; 355267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 365267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// FIXME: This is a gross hack to make it so that changing the test type in loadNextTestType doesn't reload the page. 375267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)history.reloadRequiringParameters = history.reloadRequiringParameters.filter(function(item) { return item != 'testType'; }); 385267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 395267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview.loadNextTestType = function(historyInstance) 405267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles){ 415267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (overview._testTypeIndex == builders.testTypes.length) { 425267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) overview._generatePage(); 435267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return; 445267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) } 455267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 465267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) historyInstance.crossDashboardState.testType = builders.testTypes[overview._testTypeIndex++]; 475267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 485267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) $('content').innerHTML = (overview._testTypeIndex - 1) + '/' + builders.testTypes.length + ' loaded. Loading ' + historyInstance.crossDashboardState.testType + '...'; 495267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 505267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // FIXME: Gross hack to allow loading all the builders for different test types. 515267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // Change loader.js to allow you to pass in the state that it fills instead of setting globals. 525267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) g_resultsByBuilder = {}; 535267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) overview._resultsByTestType[historyInstance.crossDashboardState.testType] = g_resultsByBuilder; 545267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) new loader.Loader().load(); 555267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)} 565267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 575267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview._getFlakyData = function(allTestTypes, resultsByTestType, flipCountThreshold) 585267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles){ 595267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var flakyData = {}; 605267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) allTestTypes.forEach(function(testType) { 615267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flakyData[testType] = { 625267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flakyBelowThreshold: {}, 635267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flaky: {}, 645267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) testCount: 0 655267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) } 665267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 675267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var resultsByBuilder = resultsByTestType[testType]; 685267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) for (var builder in resultsByBuilder) { 695267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var totalTestCount = results.testCounts(resultsByBuilder[builder][results.NUM_FAILURES_BY_TYPE]).totalTests[0]; 705267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flakyData[testType].testCount = Math.max(totalTestCount, flakyData[testType].testCount); 715267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 725267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var allTestsForThisBuilder = resultsByBuilder[builder].tests; 735267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) for (var test in allTestsForThisBuilder) { 745267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var resultsForTest = {}; 755267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var testData = resultsByBuilder[builder].tests[test].results; 765267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var failureMap = resultsByBuilder[builder][results.FAILURE_MAP]; 775267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) results.determineFlakiness(failureMap, testData, resultsForTest); 785267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 795267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (resultsForTest.isFlaky) 805267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flakyData[testType].flaky[test] = true; 815267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 825267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (!resultsForTest.isFlaky || resultsForTest.flipCount <= flipCountThreshold) 835267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) continue; 845267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flakyData[testType].flakyBelowThreshold[test] = true; 855267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) } 865267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) } 875267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) }); 885267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return flakyData; 895267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)} 905267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 915267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview._generatePage = function() 925267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles){ 935267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var flipCountThreshold = Number(g_history.dashboardSpecificState.flipCount); 945267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var flakyData = overview._getFlakyData(builders.testTypes, overview._resultsByTestType, flipCountThreshold); 955267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) $('content').innerHTML = overview._htmlForFlakyTests(flakyData, g_history.crossDashboardState.group) + 965267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '<div>*Tests that fail due to a bad patch being committed are counted as flaky.</div>'; 975267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)} 985267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 995267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview._htmlForFlakyTests = function(flakyData, group) 1005267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles){ 1015267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var html = '<table><tr><th>Test type</th><th>flaky count / total count</th><th>percent</th><th></th></tr>'; 1025267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1035267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) Object.keys(flakyData).forEach(function(testType) { 1045267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var testCount = flakyData[testType].testCount; 1055267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (!testCount) 1065267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return; 1075267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1085267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // We want the list of tests to stay stable as you drag the flakiness slider, so only 1095267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // exclude tests that never flake, even at the lowest flakiness threshold. 1105267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var flakeCountIgnoringThreshold = Object.keys(flakyData[testType].flaky).length; 1115267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (!g_history.dashboardSpecificState.showNoFlakes && !flakeCountIgnoringThreshold) 1125267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return; 1135267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1145267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var tests = Object.keys(flakyData[testType].flakyBelowThreshold); 1155267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var flakyCount = tests.length; 1165267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) var percentage = Math.round(100 * flakyCount / testCount); 1175267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) html += '<tr>' + 1185267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '<td><a href="flakiness_dashboard.html#group=' + group + '&testType=' + testType + '&tests=' + tests.join(',') + '" target=_blank>' + 1195267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) testType + 1205267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '</a></td>' + 1215267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '<td>' + flakyCount + ' / ' + testCount + '</td>' + 1225267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '<td>' + percentage + '%</td>' + 1235267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '<td><div class="flaky-bar" style="width:' + percentage * 5 + 'px"></div>' 1245267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '</tr>'; 1255267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) }); 1265267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1275267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return html + '</table>'; 1285267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)} 1295267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1305267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview.handleValidHashParameter = function(historyInstance, key, value) { 1315267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) switch(key) { 1325267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) case 'flipCount': 1335267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return history.validateParameter(historyInstance.dashboardSpecificState, key, value, 1345267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) function() { 1355267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return !isNaN(Number(value)); 1365267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) }); 1375267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1385267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) case 'showNoFlakes': 1395267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) historyInstance.dashboardSpecificState[key] = value == 'true'; 1405267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return true; 1415267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1425267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) default: 1435267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return false; 1445267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) } 1455267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)} 1465267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1475267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)overview._htmlForNavBar = function(flipCount, showNoFlakes) 1485267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles){ 1495267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return ui.html.navbar(ui.html.select('Group', 'group', builders.getAllGroupNames())) + 1505267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '<div id=flip-slider-container>' + 1515267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) ui.html.range('flipCount', 'Flakiness threshold (low-->high):', 1, 50, flipCount) + 1525267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) ui.html.checkbox('showNoFlakes', 'Show test suites with no flakes', showNoFlakes) + 1535267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) '</div>'; 1545267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)} 1555267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1565267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)// FIXME: Once dashboard_base, loader and ui stop using the g_history global, we can stop setting it here. 1575267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)g_history = new history.History({ 1585267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) defaultStateValues: { 1595267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) flipCount: 1, 1605267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) showNoFlakes: false 1615267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) }, 1625267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) generatePage: overview.loadNextTestType, 1635267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) handleValidHashParameter: overview.handleValidHashParameter, 1645267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)}); 1655267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)g_history.parseCrossDashboardParameters(); 1665267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1675267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)window.addEventListener('load', function() { 1685267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // FIXME: Come up with a better way to do this. This early return is just to avoid 1695267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // executing this code when it's loaded in the unittests. 1705267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (!$('navbar')) 1715267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return; 1725267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1735267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // Need to parseParameters so that flipCount has the correct value. 1745267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) g_history.parseParameters(); 1755267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) $('navbar').innerHTML = overview._htmlForNavBar(g_history.dashboardSpecificState.flipCount); 1765267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) overview.loadNextTestType(g_history); 1775267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)}, false); 1785267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1795267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)})(); 180