1// Copyright (C) 2011 Google Inc. All rights reserved.
2//
3// Redistribution and use in source and binary forms, with or without
4// modification, are permitted provided that the following conditions are
5// met:
6//
7//     * Redistributions of source code must retain the above copyright
8// notice, this list of conditions and the following disclaimer.
9//     * Redistributions in binary form must reproduce the above
10// copyright notice, this list of conditions and the following disclaimer
11// in the documentation and/or other materials provided with the
12// distribution.
13//     * Neither the name of Google Inc. nor the names of its
14// contributors may be used to endorse or promote products derived from
15// this software without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29module('flakiness_dashboard');
30
31// FIXME(jparent): Rename this once it isn't globals.
32function resetGlobals()
33{
34    allExpectations = null;
35    g_resultsByBuilder = {};
36    g_allTestsTrie = null;
37    var historyInstance = new history.History(flakinessConfig);
38    // FIXME(jparent): Remove this once global isn't used.
39    g_history = historyInstance;
40    g_testToResultsMap = {};
41
42    for (var key in history.DEFAULT_CROSS_DASHBOARD_STATE_VALUES)
43        historyInstance.crossDashboardState[key] = history.DEFAULT_CROSS_DASHBOARD_STATE_VALUES[key];
44
45    LOAD_BUILDBOT_DATA({
46        'masters': [{
47            name: 'ChromiumWebkit',
48            url: 'dummyurl',
49            tests: {'layout-tests': {'builders': ['WebKit Linux', 'WebKit Linux (dbg)', 'WebKit Linux (deps)', 'WebKit Mac10.7', 'WebKit Win', 'WebKit Win (dbg)']}},
50            groups: ['@ToT Chromium', '@ToT Blink'],
51        },{
52            name :'ChromiumWin',
53            url: 'dummyurl2',
54            tests: {'ash_unittests': {'builders': ['XP Tests (1)', 'Win7 Tests (1)']}},
55            groups: ['@ToT Chromium'],
56        }],
57    });
58
59   return historyInstance;
60}
61
62var FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "MISSING",
63    "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT"}
64
65test('splitTestList', 1, function() {
66    var historyInstance = new history.History(flakinessConfig);
67    // FIXME(jparent): Remove this once global isn't used.
68    g_history = historyInstance;
69    historyInstance.dashboardSpecificState.tests = 'test.foo test.foo1\ntest.foo2\ntest.foo3,foo\\bar\\baz.html';
70    equal(splitTestList().toString(), 'test.foo,test.foo1,test.foo2,test.foo3,foo/bar/baz.html');
71});
72
73test('headerForTestTableHtml', 1, function() {
74    var container = document.createElement('div');
75    container.innerHTML = headerForTestTableHtml();
76    equal(container.querySelectorAll('input').length, 4);
77});
78
79test('htmlForTestTypeSwitcherGroup', 6, function() {
80    resetGlobals();
81    var historyInstance = new history.History(flakinessConfig);
82    // FIXME(jparent): Remove this once global isn't used.
83    g_history = historyInstance;
84    var container = document.createElement('div');
85    historyInstance.crossDashboardState.testType = 'ash_unittests';
86    container.innerHTML = ui.html.testTypeSwitcher(true);
87    var selects = container.querySelectorAll('select');
88    equal(selects.length, 2);
89    var group = selects[1];
90    equal(group.parentNode.textContent.indexOf('Group:'), 0);
91    equal(group.children.length, 1);
92
93    historyInstance.crossDashboardState.testType = 'layout-tests';
94    container.innerHTML = ui.html.testTypeSwitcher(true);
95    var selects = container.querySelectorAll('select');
96    equal(selects.length, 2);
97    var group = selects[1];
98    equal(group.parentNode.textContent.indexOf('Group:'), 0);
99    equal(group.children.length, 2);
100});
101
102test('htmlForIndividualTestOnAllBuilders', 1, function() {
103    resetGlobals();
104    builders.loadBuildersList('@ToT Blink', 'layout-tests');
105    equal(htmlForIndividualTestOnAllBuilders('foo/nonexistant.html'), '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all recorded runs.</div>');
106});
107
108test('htmlForIndividualTestOnAllBuildersWithResultsLinksNonexistant', 1, function() {
109    resetGlobals();
110    builders.loadBuildersList('@ToT Blink', 'layout-tests');
111    equal(htmlForIndividualTestOnAllBuildersWithResultsLinks('foo/nonexistant.html'),
112        '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all recorded runs.</div>' +
113        '<div class=expectations test=foo/nonexistant.html>' +
114            '<div>' +
115                '<span class=link onclick="g_history.setQueryParameter(\'showExpectations\', true)">Show results</span> | ' +
116                '<span class=link onclick="g_history.setQueryParameter(\'showLargeExpectations\', true)">Show large thumbnails</span> | ' +
117                '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b>' +
118            '</div>' +
119        '</div>');
120});
121
122test('htmlForIndividualTestOnAllBuildersWithResultsLinks', 1, function() {
123    resetGlobals();
124    builders.loadBuildersList('@ToT Blink', 'layout-tests');
125
126    var builderName = 'WebKit Linux';
127    g_resultsByBuilder[builderName] = {buildNumbers: [2, 1], blinkRevision: [1234, 1233], failure_map: FAILURE_MAP};
128
129    var test = 'dummytest.html';
130    var resultsObject = createResultsObjectForTest(test, builderName);
131    resultsObject.rawResults = [[1, 'F']];
132    resultsObject.rawTimes = [[1, 0]];
133    resultsObject.bugs = ["crbug.com/1234", "webkit.org/5678"];
134    g_testToResultsMap[test] = [resultsObject];
135
136    equal(htmlForIndividualTestOnAllBuildersWithResultsLinks(test),
137        '<table class=test-table><thead><tr>' +
138                '<th sortValue=test><div class=table-header-content><span></span><span class=header-text>test</span></div></th>' +
139                '<th sortValue=bugs><div class=table-header-content><span></span><span class=header-text>bugs</span></div></th>' +
140                '<th sortValue=expectations><div class=table-header-content><span></span><span class=header-text>expectations</span></div></th>' +
141                '<th sortValue=slowest><div class=table-header-content><span></span><span class=header-text>slowest run</span></div></th>' +
142                '<th sortValue=flakiness colspan=10000><div class=table-header-content><span></span><span class=header-text>flakiness (numbers are runtimes in seconds)</span></div></th>' +
143            '</tr></thead>' +
144            '<tbody><tr>' +
145                '<td class="test-link builder-name">WebKit Linux' +
146                '<td class=options-container>' +
147                    '<div><a href="http://crbug.com/1234">crbug.com/1234</a></div>' +
148                    '<div><a href="http://webkit.org/5678">webkit.org/5678</a></div>' +
149                '<td class=options-container><td><td title="TEXT. Click for more info." class="results TEXT" onclick=\'showPopupForBuild(event, "WebKit Linux",0,"dummytest.html")\'>&nbsp;' +
150                '<td title="NO DATA. Click for more info." class="results NODATA" onclick=\'showPopupForBuild(event, "WebKit Linux",1,"dummytest.html")\'>&nbsp;' +
151            '</tbody>' +
152        '</table>' +
153        '<div>The following builders either don\'t run this test (e.g. it\'s skipped) or all recorded runs passed:</div>' +
154        '<div class=skipped-builder-list>' +
155            '<div class=skipped-builder>WebKit Linux (dbg)</div><div class=skipped-builder>WebKit Mac10.7</div><div class=skipped-builder>WebKit Win</div><div class=skipped-builder>WebKit Win (dbg)</div>' +
156        '</div>' +
157        '<div class=expectations test=dummytest.html>' +
158            '<div><span class=link onclick="g_history.setQueryParameter(\'showExpectations\', true)">Show results</span> | ' +
159            '<span class=link onclick="g_history.setQueryParameter(\'showLargeExpectations\', true)">Show large thumbnails</span> | ' +
160            '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b></div>' +
161        '</div>');
162});
163
164test('individualTestsForSubstringList', 2, function() {
165    var builderName = 'WebKit Linux';
166    g_resultsByBuilder[builderName] = {
167        buildNumbers: [2, 1],
168        blinkRevision: [1234, 1233],
169        failure_map: FAILURE_MAP,
170        tests: {
171            'foo/one.html': { results: [1, 'F'], times: [1, 0] },
172            'virtual/foo/one.html': { results: [1, 'F'], times: [1, 0] },
173        }
174    };
175
176    g_history.dashboardSpecificState.showChrome = true;
177    var testToMatch = 'foo/one.html';
178    g_history.dashboardSpecificState.tests = testToMatch;
179    deepEqual(individualTestsForSubstringList(), [testToMatch, 'virtual/foo/one.html']);
180
181    g_history.dashboardSpecificState.showChrome = false;
182    deepEqual(individualTestsForSubstringList(), [testToMatch]);
183});
184
185test('htmlForIndividualTest', 2, function() {
186    var historyInstance = resetGlobals();
187    builders.loadBuildersList('@ToT Blink', 'layout-tests');
188    var test = 'foo/nonexistant.html';
189
190    historyInstance.dashboardSpecificState.showChrome = false;
191
192    equal(htmlForIndividualTest(test), htmlForIndividualTestOnAllBuilders(test) +
193        '<div class=expectations test=foo/nonexistant.html>' +
194            '<div><span class=link onclick=\"g_history.setQueryParameter(\'showExpectations\', true)\">Show results</span> | ' +
195            '<span class=link onclick=\"g_history.setQueryParameter(\'showLargeExpectations\', true)\">Show large thumbnails</span> | ' +
196            '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b></div>' +
197        '</div>');
198
199    historyInstance.dashboardSpecificState.showChrome = true;
200
201    equal(htmlForIndividualTest(test),
202        '<h2><a href="' + TEST_URL_BASE_PATH_FOR_BROWSING + 'foo/nonexistant.html" target="_blank">foo/nonexistant.html</a></h2>' +
203        htmlForIndividualTestOnAllBuildersWithResultsLinks(test));
204});
205
206test('linkifyBugs', 4, function() {
207    equal(linkifyBugs(["crbug.com/1234", "webkit.org/5678"]),
208        '<div><a href="http://crbug.com/1234">crbug.com/1234</a></div><div><a href="http://webkit.org/5678">webkit.org/5678</a></div>');
209    equal(linkifyBugs(["crbug.com/1234"]), '<div><a href="http://crbug.com/1234">crbug.com/1234</a></div>');
210    equal(linkifyBugs(["Bug(nick)"]), '<div>Bug(nick)</div>');
211    equal(linkifyBugs([]), '');
212});
213
214test('htmlForSingleTestRow', 1, function() {
215    var historyInstance = resetGlobals();
216    var builder = 'dummyBuilder';
217    var test = createResultsObjectForTest('foo/exists.html', builder);
218    historyInstance.dashboardSpecificState.showNonFlaky = true;
219    g_resultsByBuilder[builder] = {buildNumbers: [2, 1], blinkRevision: [1234, 1233], failure_map: FAILURE_MAP};
220    test.rawResults = [[1, 'F'], [2, 'I']];
221    test.rawTimes = [[1, 0], [2, 5]];
222    var expected = '<tr>' +
223        '<td class="test-link"><span class="link" onclick="g_history.setQueryParameter(\'tests\',\'foo/exists.html\');">foo/exists.html</span>' +
224        '<td class=options-container><a href="https://code.google.com/p/chromium/issues/entry?template=Layout%20Test%20Failure&summary=Layout%20Test%20foo%2Fexists.html%20is%20failing&comment=The%20following%20layout%20test%20is%20failing%20on%20%5Binsert%20platform%5D%0A%0Afoo%2Fexists.html%0A%0AProbable%20cause%3A%0A%0A%5Binsert%20probable%20cause%5D">File new bug</a>' +
225        '<td class=options-container>' +
226        '<td>' +
227        '<td title="TEXT. Click for more info." class="results TEXT" onclick=\'showPopupForBuild(event, "dummyBuilder",0,"foo/exists.html")\'>&nbsp;' +
228        '<td title="IMAGE. Click for more info." class="results IMAGE" onclick=\'showPopupForBuild(event, "dummyBuilder",1,"foo/exists.html")\'>5';
229    equal(htmlForSingleTestRow(test), expected);
230});
231
232test('lookupVirtualTestSuite', 2, function() {
233    equal(lookupVirtualTestSuite('fast/canvas/foo.html'), '');
234    equal(lookupVirtualTestSuite('virtual/gpu/fast/canvas/foo.html'), 'virtual/gpu/fast/canvas');
235});
236
237test('baseTest', 2, function() {
238    equal(baseTest('fast/canvas/foo.html', ''), 'fast/canvas/foo.html');
239    equal(baseTest('virtual/gpu/fast/canvas/foo.html', 'virtual/gpu/fast/canvas'), 'fast/canvas/foo.html');
240});
241
242test('sortTests', 4, function() {
243    var test1 = createResultsObjectForTest('foo/test1.html', 'dummyBuilder');
244    var test2 = createResultsObjectForTest('foo/test2.html', 'dummyBuilder');
245    var test3 = createResultsObjectForTest('foo/test3.html', 'dummyBuilder');
246    test1.expectations = 'b';
247    test2.expectations = 'a';
248    test3.expectations = '';
249
250    var tests = [test1, test2, test3];
251    sortTests(tests, 'expectations', FORWARD);
252    deepEqual(tests, [test2, test1, test3]);
253    sortTests(tests, 'expectations', BACKWARD);
254    deepEqual(tests, [test3, test1, test2]);
255
256    test1.bugs = 'b';
257    test2.bugs = 'a';
258    test3.bugs = '';
259
260    var tests = [test1, test2, test3];
261    sortTests(tests, 'bugs', FORWARD);
262    deepEqual(tests, [test2, test1, test3]);
263    sortTests(tests, 'bugs', BACKWARD);
264    deepEqual(tests, [test3, test1, test2]);
265});
266
267test('popup', 2, function() {
268    ui.popup.show(document.body, 'dummy content');
269    ok(document.querySelector('#popup'));
270    ui.popup.hide();
271    ok(!document.querySelector('#popup'));
272});
273
274test('gpuResultsPath', 3, function() {
275  equal(gpuResultsPath('777777', 'Win7 Release (ATI)'), '777777_Win7_Release_ATI_');
276  equal(gpuResultsPath('123', 'GPU Linux (dbg)(NVIDIA)'), '123_GPU_Linux_dbg_NVIDIA_');
277  equal(gpuResultsPath('12345', 'GPU Mac'), '12345_GPU_Mac');
278});
279
280test('TestTrie', 3, function() {
281    var builders = {
282        "Dummy Chromium Windows Builder": true,
283        "Dummy GTK Linux Builder": true,
284        "Dummy Apple Mac Lion Builder": true
285    };
286
287    var resultsByBuilder = {
288        "Dummy Chromium Windows Builder": {
289            tests: {
290                "foo": true,
291                "foo/bar/1.html": true,
292                "foo/bar/baz": true
293            }
294        },
295        "Dummy GTK Linux Builder": {
296            tests: {
297                "bar": true,
298                "foo/1.html": true,
299                "foo/bar/2.html": true,
300                "foo/bar/baz/1.html": true,
301            }
302        },
303        "Dummy Apple Mac Lion Builder": {
304            tests: {
305                "foo/bar/3.html": true,
306                "foo/bar/baz/foo": true,
307            }
308        }
309    };
310    var expectedTrie = {
311        "foo": {
312            "bar": {
313                "1.html": true,
314                "2.html": true,
315                "3.html": true,
316                "baz": {
317                    "1.html": true,
318                    "foo": true
319                }
320            },
321            "1.html": true
322        },
323        "bar": true
324    }
325
326    var trie = new TestTrie(builders, resultsByBuilder);
327    deepEqual(trie._trie, expectedTrie);
328
329    var leafsOfCompleteTrieTraversal = [];
330    var expectedLeafs = ["foo/bar/1.html", "foo/bar/baz/1.html", "foo/bar/baz/foo", "foo/bar/2.html", "foo/bar/3.html", "foo/1.html", "bar"];
331    trie.forEach(function(triePath) {
332        leafsOfCompleteTrieTraversal.push(triePath);
333    });
334    deepEqual(leafsOfCompleteTrieTraversal, expectedLeafs);
335
336    var leafsOfPartialTrieTraversal = [];
337    expectedLeafs = ["foo/bar/1.html", "foo/bar/baz/1.html", "foo/bar/baz/foo", "foo/bar/2.html", "foo/bar/3.html"];
338    trie.forEach(function(triePath) {
339        leafsOfPartialTrieTraversal.push(triePath);
340    }, "foo/bar");
341    deepEqual(leafsOfPartialTrieTraversal, expectedLeafs);
342});
343
344test('changeTestTypeInvalidatesGroup', 1, function() {
345    var historyInstance = resetGlobals();
346    var originalGroup = '@ToT Blink';
347    var originalTestType = 'layout-tests';
348    builders.loadBuildersList(originalGroup, originalTestType);
349    historyInstance.crossDashboardState.group = originalGroup;
350    historyInstance.crossDashboardState.testType = originalTestType;
351
352    historyInstance.invalidateQueryParameters({'testType': 'ui_tests'});
353    notEqual(historyInstance.crossDashboardState.group, originalGroup, "group should have been invalidated");
354});
355
356test('shouldShowTest', 9, function() {
357    var historyInstance = new history.History(flakinessConfig);
358    historyInstance.parseParameters();
359    // FIXME(jparent): Change to use the flakiness_dashboard's history object
360    // once it exists, rather than tracking global.
361    g_history = historyInstance;
362    var test = createResultsObjectForTest('foo/test.html', 'dummyBuilder');
363
364    equal(shouldShowTest(test), false, 'default layout test, hide it.');
365    historyInstance.dashboardSpecificState.showNonFlaky = true;
366    equal(shouldShowTest(test), true, 'show correct expectations.');
367    historyInstance.dashboardSpecificState.showNonFlaky = false;
368
369    test = createResultsObjectForTest('foo/test.html', 'dummyBuilder');
370    test.expectations = "WONTFIX";
371    equal(shouldShowTest(test), false, 'by default hide wontfix');
372    historyInstance.dashboardSpecificState.showWontFix = true;
373    equal(shouldShowTest(test), true, 'show wontfix');
374    historyInstance.dashboardSpecificState.showWontFix = false;
375
376    test = createResultsObjectForTest('foo/test.html', 'dummyBuilder');
377    test.expectations = "SKIP";
378    equal(shouldShowTest(test), false, 'we hide skip tests by default');
379    historyInstance.dashboardSpecificState.showSkip = true;
380    equal(shouldShowTest(test), true, 'show skip test');
381    historyInstance.dashboardSpecificState.showSkip = false;
382
383    test = createResultsObjectForTest('foo/test.html', 'dummyBuilder');
384    test.isFlaky = true;
385    equal(shouldShowTest(test), false, 'hide flaky tests by default');
386    historyInstance.dashboardSpecificState.showFlaky = true;
387    equal(shouldShowTest(test), true, 'show flaky test');
388    historyInstance.dashboardSpecificState.showFlaky = false;
389
390    test = createResultsObjectForTest('foo/test.html', 'dummyBuilder');
391    historyInstance.crossDashboardState.testType = 'not layout tests';
392    equal(shouldShowTest(test), true, 'show all non layout tests');
393});
394