Summarizer.java revision f460dd42190ada4a2c147db5127a9d7870fe0101 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.dumprendertree2;
18
19import android.content.Context;
20import android.content.res.AssetManager;
21import android.content.res.Configuration;
22import android.content.res.Resources;
23import android.database.Cursor;
24import android.os.Build;
25import android.os.Message;
26import android.util.DisplayMetrics;
27import android.util.Log;
28
29import com.android.dumprendertree2.forwarder.ForwarderManager;
30
31import java.io.File;
32import java.net.MalformedURLException;
33import java.net.URI;
34import java.net.URL;
35import java.text.SimpleDateFormat;
36import java.util.ArrayList;
37import java.util.Date;
38import java.util.List;
39import java.util.regex.Matcher;
40import java.util.regex.Pattern;
41
42/**
43 * A class that collects information about tests that ran and can create HTML
44 * files with summaries and easy navigation.
45 */
46public class Summarizer {
47
48    private static final String LOG_TAG = "Summarizer";
49
50    private static final String CSS =
51            "<style type=\"text/css\">" +
52            "* {" +
53            "       font-family: Verdana;" +
54            "       border: 0;" +
55            "       margin: 0;" +
56            "       padding: 0;}" +
57            "body {" +
58            "       margin: 10px;}" +
59            "h1 {" +
60            "       font-size: 24px;" +
61            "       margin: 4px 0 4px 0;}" +
62            "h2 {" +
63            "       font-size:18px;" +
64            "       text-transform: uppercase;" +
65            "       margin: 20px 0 3px 0;}" +
66            "h3, h3 a {" +
67            "       font-size: 14px;" +
68            "       color: black;" +
69            "       text-decoration: none;" +
70            "       margin-top: 4px;" +
71            "       margin-bottom: 2px;}" +
72            "h3 a span.path {" +
73            "       text-decoration: underline;}" +
74            "h3 span.tri {" +
75            "       text-decoration: none;" +
76            "       float: left;" +
77            "       width: 20px;}" +
78            "h3 span.sqr {" +
79            "       text-decoration: none;" +
80            "       float: left;" +
81            "       width: 20px;}" +
82            "h3 span.sqr_pass {" +
83            "       color: #8ee100;}" +
84            "h3 span.sqr_fail {" +
85            "       color: #c30000;}" +
86            "span.source {" +
87            "       display: block;" +
88            "       font-size: 10px;" +
89            "       color: #888;" +
90            "       margin-left: 20px;" +
91            "       margin-bottom: 1px;}" +
92            "span.source a {" +
93            "       font-size: 10px;" +
94            "       color: #888;}" +
95            "h3 img {" +
96            "       width: 8px;" +
97            "       margin-right: 4px;}" +
98            "div.diff {" +
99            "       margin-bottom: 25px;}" +
100            "div.diff a {" +
101            "       font-size: 12px;" +
102            "       color: #888;}" +
103            "table.visual_diff {" +
104            "       border-bottom: 0px solid;" +
105            "       border-collapse: collapse;" +
106            "       width: 100%;" +
107            "       margin-bottom: 2px;}" +
108            "table.visual_diff tr.headers td {" +
109            "       border-bottom: 1px solid;" +
110            "       border-top: 0;" +
111            "       padding-bottom: 3px;}" +
112            "table.visual_diff tr.results td {" +
113            "       border-top: 1px dashed;" +
114            "       border-right: 1px solid;" +
115            "       font-size: 15px;" +
116            "       vertical-align: top;}" +
117            "table.visual_diff tr.results td.line_count {" +
118            "       background-color:#aaa;" +
119            "       min-width:20px;" +
120            "       text-align: right;" +
121            "       border-right: 1px solid;" +
122            "       border-left: 1px solid;" +
123            "       padding: 2px 1px 2px 0px;}" +
124            "table.visual_diff tr.results td.line {" +
125            "       padding: 2px 0px 2px 4px;" +
126            "       border-right: 1px solid;" +
127            "       width: 49.8%;}" +
128            "table.visual_diff tr.footers td {" +
129            "       border-top: 1px solid;" +
130            "       border-bottom: 0;}" +
131            "table.visual_diff tr td.space {" +
132            "       border: 0;" +
133            "       width: 0.4%}" +
134            "div.space {" +
135            "       margin-top:4px;}" +
136            "span.eql {" +
137            "       background-color: #f3f3f3;}" +
138            "span.del {" +
139            "       background-color: #ff8888; }" +
140            "span.ins {" +
141            "       background-color: #88ff88; }" +
142            "table.summary {" +
143            "       border: 1px solid black;" +
144            "       margin-top: 20px;}" +
145            "table.summary td {" +
146            "       padding: 3px;}" +
147            "span.listItem {" +
148            "       font-size: 11px;" +
149            "       font-weight: normal;" +
150            "       text-transform: uppercase;" +
151            "       padding: 3px;" +
152            "       -webkit-border-radius: 4px;}" +
153            "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" +
154            "       background-color: #ccc;" +
155            "       color: black;}" +
156            "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" +
157            "       background-color: #a700e4;" +
158            "       color: #fff;}" +
159            "span.timed_out {" +
160            "       background-color: #f3cb00;" +
161            "       color: black;}" +
162            "span.crashed {" +
163            "       background-color: #c30000;" +
164            "       color: #fff;}" +
165            "span.noLtc {" +
166            "       background-color: #944000;" +
167            "       color: #fff;}" +
168            "span.noEventSender {" +
169            "       background-color: #815600;" +
170            "       color: #fff;}" +
171            "</style>";
172
173    private static final String SCRIPT =
174            "<script type=\"text/javascript\">" +
175            "    function toggleDisplay(id) {" +
176            "        element = document.getElementById(id);" +
177            "        triangle = document.getElementById('tri.' + id);" +
178            "        if (element.style.display == 'none') {" +
179            "            element.style.display = 'inline';" +
180            "            triangle.innerHTML = '&#x25bc; ';" +
181            "        } else {" +
182            "            element.style.display = 'none';" +
183            "            triangle.innerHTML = '&#x25b6; ';" +
184            "        }" +
185            "    }" +
186            "</script>";
187
188    /** TODO: Make it a setting */
189    private static final String HTML_DETAILS_RELATIVE_PATH = "details.html";
190    private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt";
191
192    private static final int RESULTS_PER_DUMP = 500;
193    private static final int RESULTS_PER_DB_ACCESS = 50;
194
195    private int mCrashedTestsCount = 0;
196    private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>();
197    private List<AbstractResult> mExpectedFailures = new ArrayList<AbstractResult>();
198    private List<AbstractResult> mExpectedPasses = new ArrayList<AbstractResult>();
199    private List<AbstractResult> mUnexpectedPasses = new ArrayList<AbstractResult>();
200
201    private Cursor mUnexpectedFailuresCursor;
202    private Cursor mExpectedFailuresCursor;
203    private Cursor mUnexpectedPassesCursor;
204    private Cursor mExpectedPassesCursor;
205
206    private FileFilter mFileFilter;
207    private String mResultsRootDirPath;
208    private String mTestsRelativePath;
209    private Date mDate;
210
211    private int mResultsSinceLastHtmlDump = 0;
212    private int mResultsSinceLastDbAccess = 0;
213
214    private SummarizerDBHelper mDbHelper;
215
216    public Summarizer(FileFilter fileFilter, String resultsRootDirPath, Context context) {
217        mFileFilter = fileFilter;
218        mResultsRootDirPath = resultsRootDirPath;
219
220        /**
221         * We don't run the database I/O in a separate thread to avoid consumer/producer problem
222         * and to simplify code.
223         */
224        mDbHelper = new SummarizerDBHelper(context);
225        mDbHelper.open();
226    }
227
228    public static URI getDetailsUri() {
229        return new File(ManagerService.RESULTS_ROOT_DIR_PATH + File.separator +
230                HTML_DETAILS_RELATIVE_PATH).toURI();
231    }
232
233    public void appendTest(AbstractResult result) {
234        String relativePath = result.getRelativePath();
235
236        if (result.didCrash()) {
237            mCrashedTestsCount++;
238        }
239
240        if (result.didPass()) {
241            result.clearResults();
242            if (mFileFilter.isFail(relativePath)) {
243                mUnexpectedPasses.add(result);
244            } else {
245                mExpectedPasses.add(result);
246            }
247        } else {
248            if (mFileFilter.isFail(relativePath)) {
249                mExpectedFailures.add(result);
250            } else {
251                mUnexpectedFailures.add(result);
252            }
253        }
254
255        if (++mResultsSinceLastDbAccess == RESULTS_PER_DB_ACCESS) {
256            persistLists();
257            clearLists();
258        }
259    }
260
261    private void clearLists() {
262        mUnexpectedFailures.clear();
263        mExpectedFailures.clear();
264        mUnexpectedPasses.clear();
265        mExpectedPasses.clear();
266    }
267
268    private void persistLists() {
269        persistListToTable(mUnexpectedFailures, SummarizerDBHelper.UNEXPECTED_FAILURES_TABLE);
270        persistListToTable(mExpectedFailures, SummarizerDBHelper.EXPECTED_FAILURES_TABLE);
271        persistListToTable(mUnexpectedPasses, SummarizerDBHelper.UNEXPECTED_PASSES_TABLE);
272        persistListToTable(mExpectedPasses, SummarizerDBHelper.EXPECTED_PASSES_TABLE);
273        mResultsSinceLastDbAccess = 0;
274    }
275
276    private void persistListToTable(List<AbstractResult> results, String table) {
277        for (AbstractResult abstractResult : results) {
278            mDbHelper.insertAbstractResult(abstractResult, table);
279        }
280    }
281
282    public void setTestsRelativePath(String testsRelativePath) {
283        mTestsRelativePath = testsRelativePath;
284    }
285
286    public void summarize(Message onFinishMessage) {
287        persistLists();
288        clearLists();
289
290        mUnexpectedFailuresCursor =
291            mDbHelper.getAbstractResults(SummarizerDBHelper.UNEXPECTED_FAILURES_TABLE);
292        mUnexpectedPassesCursor =
293            mDbHelper.getAbstractResults(SummarizerDBHelper.UNEXPECTED_PASSES_TABLE);
294        mExpectedFailuresCursor =
295            mDbHelper.getAbstractResults(SummarizerDBHelper.EXPECTED_FAILURES_TABLE);
296        mExpectedPassesCursor =
297            mDbHelper.getAbstractResults(SummarizerDBHelper.EXPECTED_PASSES_TABLE);
298
299        String webKitRevision = getWebKitRevision();
300        createHtmlDetails(webKitRevision);
301        createTxtSummary(webKitRevision);
302
303        clearLists();
304        mUnexpectedFailuresCursor.close();
305        mUnexpectedPassesCursor.close();
306        mExpectedFailuresCursor.close();
307        mExpectedPassesCursor.close();
308
309        onFinishMessage.sendToTarget();
310    }
311
312    public void reset() {
313        mCrashedTestsCount = 0;
314        clearLists();
315        mDbHelper.reset();
316        mDate = new Date();
317    }
318
319    private void dumpHtmlToFile(StringBuilder html, boolean append) {
320        FsUtils.writeDataToStorage(new File(mResultsRootDirPath, HTML_DETAILS_RELATIVE_PATH),
321                html.toString().getBytes(), append);
322        html.setLength(0);
323        mResultsSinceLastHtmlDump = 0;
324    }
325
326    private void createTxtSummary(String webKitRevision) {
327        StringBuilder txt = new StringBuilder();
328
329        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
330        txt.append("Path: " + mTestsRelativePath + "\n");
331        txt.append("Date: " + dateFormat.format(mDate) + "\n");
332        txt.append("Build fingerprint: " + Build.FINGERPRINT + "\n");
333        txt.append("WebKit version: " + getWebKitVersionFromUserAgentString() + "\n");
334        txt.append("WebKit revision: " + webKitRevision + "\n");
335
336        txt.append("TOTAL:                     " + getTotalTestCount() + "\n");
337        txt.append("CRASHED (among all tests): " + mCrashedTestsCount + "\n");
338        txt.append("UNEXPECTED FAILURES:       " + mUnexpectedFailuresCursor.getCount() + "\n");
339        txt.append("UNEXPECTED PASSES:         " + mUnexpectedPassesCursor.getCount() + "\n");
340        txt.append("EXPECTED FAILURES:         " + mExpectedFailuresCursor.getCount() + "\n");
341        txt.append("EXPECTED PASSES:           " + mExpectedPassesCursor.getCount() + "\n");
342
343        FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH),
344                txt.toString().getBytes(), false);
345    }
346
347    private void createHtmlDetails(String webKitRevision) {
348        StringBuilder html = new StringBuilder();
349
350        html.append("<html><head>");
351        html.append(CSS);
352        html.append(SCRIPT);
353        html.append("</head><body>");
354
355        createTopSummaryTable(webKitRevision, html);
356        dumpHtmlToFile(html, false);
357
358        createResultsList(html, "Unexpected failures", mUnexpectedFailuresCursor);
359        createResultsList(html, "Unexpected passes", mUnexpectedPassesCursor);
360        createResultsList(html, "Expected failures", mExpectedFailuresCursor);
361        createResultsList(html, "Expected passes", mExpectedPassesCursor);
362
363        html.append("</body></html>");
364        dumpHtmlToFile(html, true);
365    }
366
367    private int getTotalTestCount() {
368        return mUnexpectedFailuresCursor.getCount() +
369                mUnexpectedPassesCursor.getCount() +
370                mExpectedPassesCursor.getCount() +
371                mExpectedFailuresCursor.getCount();
372    }
373
374    private String getWebKitVersionFromUserAgentString() {
375        Resources resources = new Resources(new AssetManager(), new DisplayMetrics(),
376                new Configuration());
377        String userAgent =
378                resources.getString(com.android.internal.R.string.web_user_agent);
379
380        Matcher matcher = Pattern.compile("AppleWebKit/([0-9]+?\\.[0-9])").matcher(userAgent);
381        if (matcher.find()) {
382            return matcher.group(1);
383        }
384        return "unknown";
385    }
386
387    private String getWebKitRevision() {
388        URL url = null;
389        try {
390            url = new URL(ForwarderManager.getHostSchemePort(false) + "ThirdPartyProject.prop");
391        } catch (MalformedURLException e) {
392            assert false;
393        }
394
395        String thirdPartyProjectContents = new String(FsUtils.readDataFromUrl(url));
396        Matcher matcher = Pattern.compile("^version=([0-9]+)", Pattern.MULTILINE).matcher(
397                thirdPartyProjectContents);
398        if (matcher.find()) {
399            return matcher.group(1);
400        }
401        return "unknown";
402    }
403
404    private void createTopSummaryTable(String webKitRevision, StringBuilder html) {
405        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
406        html.append("<h1>" + "Layout tests' results for: " +
407                (mTestsRelativePath.equals("") ? "all tests" : mTestsRelativePath) + "</h1>");
408        html.append("<h3>" + "Date: " + dateFormat.format(new Date()) + "</h3>");
409        html.append("<h3>" + "Build fingerprint: " + Build.FINGERPRINT + "</h3>");
410        html.append("<h3>" + "WebKit version: " + getWebKitVersionFromUserAgentString() + "</h3>");
411
412        html.append("<h3>" + "WebKit revision: ");
413        html.append("<a href=\"http://trac.webkit.org/browser/trunk?rev=" + webKitRevision +
414                "\" target=\"_blank\"><span class=\"path\">" + webKitRevision + "</span></a>");
415        html.append("</h3>");
416
417        html.append("<table class=\"summary\">");
418        createSummaryTableRow(html, "TOTAL", getTotalTestCount());
419        createSummaryTableRow(html, "CRASHED (among all tests)", mCrashedTestsCount);
420        createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailuresCursor.getCount());
421        createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPassesCursor.getCount());
422        createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailuresCursor.getCount());
423        createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPassesCursor.getCount());
424        html.append("</table>");
425    }
426
427    private void createSummaryTableRow(StringBuilder html, String caption, int size) {
428        html.append("<tr>");
429        html.append("    <td>" + caption + "</td>");
430        html.append("    <td>" + size + "</td>");
431        html.append("</tr>");
432    }
433
434    private void createResultsList(
435            StringBuilder html, String title, Cursor cursor) {
436        String relativePath;
437        String id = "";
438        AbstractResult.ResultCode resultCode;
439
440        html.append("<h2>" + title + " [" + cursor.getCount() + "]</h2>");
441
442        if (!cursor.moveToFirst()) {
443            return;
444        }
445
446        AbstractResult result;
447        do {
448            result = SummarizerDBHelper.getAbstractResult(cursor);
449
450            relativePath = result.getRelativePath();
451            resultCode = result.getResultCode();
452
453            html.append("<h3>");
454
455            /**
456             * Technically, two different paths could end up being the same, because
457             * ':' is a valid  character in a path. However, it is probably not going
458             * to cause any problems in this case
459             */
460            id = relativePath.replace(File.separator, ":");
461
462            /** Write the test name */
463            if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
464                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
465                html.append("return false;\">");
466                html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
467                html.append("<span class=\"path\">" + relativePath + "</span>");
468                html.append("</a>");
469            } else {
470                html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\"");
471                html.append(" target=\"_blank\">");
472                html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail"));
473                html.append("\">&#x25a0; </span>");
474                html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
475                html.append("</a>");
476            }
477
478            if (!result.didPass()) {
479                appendTags(html, result);
480            }
481
482            html.append("</h3>");
483            appendExpectedResultsSources(result, html);
484
485            if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
486                html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
487                html.append(result.getDiffAsHtml());
488                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
489                html.append("return false;\">Hide</a>");
490                html.append(" | ");
491                html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
492                html.append(" target=\"_blank\">Show source</a>");
493                html.append("</div>");
494            }
495
496            html.append("<div class=\"space\"></div>");
497
498            if (++mResultsSinceLastHtmlDump == RESULTS_PER_DUMP) {
499                dumpHtmlToFile(html, true);
500            }
501
502            cursor.moveToNext();
503        } while (!cursor.isAfterLast());
504    }
505
506    private void appendTags(StringBuilder html, AbstractResult result) {
507        /** Tag tests which crash, time out or where results don't match */
508        if (result.didCrash()) {
509            html.append(" <span class=\"listItem crashed\">Crashed</span>");
510        } else {
511            if (result.didTimeOut()) {
512                html.append(" <span class=\"listItem timed_out\">Timed out</span>");
513            }
514            AbstractResult.ResultCode resultCode = result.getResultCode();
515            if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) {
516                html.append(" <span class=\"listItem " + resultCode.name() + "\">");
517                html.append(resultCode.toString());
518                html.append("</span>");
519            }
520        }
521
522        /** Detect missing LTC function */
523        String additionalTextOutputString = result.getAdditionalTextOutputString();
524        if (additionalTextOutputString != null &&
525                additionalTextOutputString.contains("com.android.dumprendertree") &&
526                additionalTextOutputString.contains("has no method")) {
527            if (additionalTextOutputString.contains("LayoutTestController")) {
528                html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
529            }
530            if (additionalTextOutputString.contains("EventSender")) {
531                html.append(" <span class=\"listItem noEventSender\">");
532                html.append("ES function missing</span>");
533            }
534        }
535    }
536
537    private static final void appendExpectedResultsSources(AbstractResult result,
538            StringBuilder html) {
539        String textSource = result.getExpectedTextResultPath();
540        String imageSource = result.getExpectedImageResultPath();
541
542        if (textSource == null) {
543            // Show if a text result is missing. We may want to revisit this decision when we add
544            // support for image results.
545            html.append("<span class=\"source\">Expected textual result missing</span>");
546        } else {
547            html.append("<span class=\"source\">Expected textual result from: ");
548            html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" +
549                    textSource + "\"");
550            html.append(" target=\"_blank\">");
551            html.append(textSource + "</a></span>");
552        }
553        if (imageSource != null) {
554            html.append("<span class=\"source\">Expected image result from: ");
555            html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" +
556                    imageSource + "\"");
557            html.append(" target=\"_blank\">");
558            html.append(imageSource + "</a></span>");
559        }
560    }
561
562    private static final URL getViewSourceUrl(String relativePath) {
563        URL url = null;
564        try {
565            url = new URL("http", "localhost", ForwarderManager.HTTP_PORT,
566                    "/WebKitTools/DumpRenderTree/android/view_source.php?src=" +
567                    relativePath);
568        } catch (MalformedURLException e) {
569            assert false : "relativePath=" + relativePath;
570        }
571        return url;
572    }
573}
574