Summarizer.java revision 75aa0d382844dbee2d7931f21245112c27dd3d64 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.dumprendertree2;
18
19import android.content.res.AssetManager;
20import android.content.res.Configuration;
21import android.content.res.Resources;
22import android.os.Build;
23import android.util.DisplayMetrics;
24import android.util.Log;
25
26import com.android.dumprendertree2.forwarder.ForwarderManager;
27
28import java.io.File;
29import java.net.MalformedURLException;
30import java.net.URI;
31import java.net.URL;
32import java.text.SimpleDateFormat;
33import java.util.ArrayList;
34import java.util.Collections;
35import java.util.Date;
36import java.util.List;
37import java.util.regex.Matcher;
38import java.util.regex.Pattern;
39
40/**
41 * A class that collects information about tests that ran and can create HTML
42 * files with summaries and easy navigation.
43 */
44public class Summarizer {
45
46    private static final String LOG_TAG = "Summarizer";
47
48    private static final String CSS =
49            "<style type=\"text/css\">" +
50            "* {" +
51            "       font-family: Verdana;" +
52            "       border: 0;" +
53            "       margin: 0;" +
54            "       padding: 0;}" +
55            "body {" +
56            "       margin: 10px;}" +
57            "h1 {" +
58            "       font-size: 24px;" +
59            "       margin: 4px 0 4px 0;}" +
60            "h2 {" +
61            "       font-size:18px;" +
62            "       text-transform: uppercase;" +
63            "       margin: 20px 0 3px 0;}" +
64            "h3, h3 a {" +
65            "       font-size: 14px;" +
66            "       color: black;" +
67            "       text-decoration: none;" +
68            "       margin-top: 4px;" +
69            "       margin-bottom: 2px;}" +
70            "h3 a span.path {" +
71            "       text-decoration: underline;}" +
72            "h3 span.tri {" +
73            "       text-decoration: none;" +
74            "       float: left;" +
75            "       width: 20px;}" +
76            "h3 span.sqr {" +
77            "       text-decoration: none;" +
78            "       color: #8ee100;" +
79            "       float: left;" +
80            "       width: 20px;}" +
81            "span.source {" +
82            "       display: block;" +
83            "       font-size: 10px;" +
84            "       color: #888;" +
85            "       margin-left: 20px;" +
86            "       margin-bottom: 1px;}" +
87            "span.source a {" +
88            "       font-size: 10px;" +
89            "       color: #888;}" +
90            "h3 img {" +
91            "       width: 8px;" +
92            "       margin-right: 4px;}" +
93            "div.diff {" +
94            "       margin-bottom: 25px;}" +
95            "div.diff a {" +
96            "       font-size: 12px;" +
97            "       color: #888;}" +
98            "table.visual_diff {" +
99            "       border-bottom: 0px solid;" +
100            "       border-collapse: collapse;" +
101            "       width: 100%;" +
102            "       margin-bottom: 2px;}" +
103            "table.visual_diff tr.headers td {" +
104            "       border-bottom: 1px solid;" +
105            "       border-top: 0;" +
106            "       padding-bottom: 3px;}" +
107            "table.visual_diff tr.results td {" +
108            "       border-top: 1px dashed;" +
109            "       border-right: 1px solid;" +
110            "       font-size: 15px;" +
111            "       vertical-align: top;}" +
112            "table.visual_diff tr.results td.line_count {" +
113            "       background-color:#aaa;" +
114            "       min-width:20px;" +
115            "       text-align: right;" +
116            "       border-right: 1px solid;" +
117            "       border-left: 1px solid;" +
118            "       padding: 2px 1px 2px 0px;}" +
119            "table.visual_diff tr.results td.line {" +
120            "       padding: 2px 0px 2px 4px;" +
121            "       border-right: 1px solid;" +
122            "       width: 49.8%;}" +
123            "table.visual_diff tr.footers td {" +
124            "       border-top: 1px solid;" +
125            "       border-bottom: 0;}" +
126            "table.visual_diff tr td.space {" +
127            "       border: 0;" +
128            "       width: 0.4%}" +
129            "div.space {" +
130            "       margin-top:4px;}" +
131            "span.eql {" +
132            "       background-color: #f3f3f3;}" +
133            "span.del {" +
134            "       background-color: #ff8888; }" +
135            "span.ins {" +
136            "       background-color: #88ff88; }" +
137            "span.fail {" +
138            "       color: red;}" +
139            "span.pass {" +
140            "       color: green;}" +
141            "span.time_out {" +
142            "       color: orange;}" +
143            "table.summary {" +
144            "       border: 1px solid black;" +
145            "       margin-top: 20px;}" +
146            "table.summary td {" +
147            "       padding: 3px;}" +
148            "span.listItem {" +
149            "       font-size: 11px;" +
150            "       font-weight: normal;" +
151            "       text-transform: uppercase;" +
152            "       padding: 3px;" +
153            "       -webkit-border-radius: 4px;}" +
154            "span." + AbstractResult.ResultCode.PASS.name() + "{" +
155            "       background-color: #8ee100;" +
156            "       color: black;}" +
157            "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
158            "       background-color: #ccc;" +
159            "       color: black;}" +
160            "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
161            "       background-color: #a700e4;" +
162            "       color: #fff;}" +
163            "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
164            "       background-color: #f3cb00;" +
165            "       color: black;}" +
166            "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
167            "       background-color: #c30000;" +
168            "       color: #fff;}" +
169            "span.noLtc {" +
170            "       background-color: #944000;" +
171            "       color: #fff;}" +
172            "span.noEventSender {" +
173            "       background-color: #815600;" +
174            "       color: #fff;}" +
175            "</style>";
176
177    private static final String SCRIPT =
178            "<script type=\"text/javascript\">" +
179            "    function toggleDisplay(id) {" +
180            "        element = document.getElementById(id);" +
181            "        triangle = document.getElementById('tri.' + id);" +
182            "        if (element.style.display == 'none') {" +
183            "            element.style.display = 'inline';" +
184            "            triangle.innerHTML = '&#x25bc; ';" +
185            "        } else {" +
186            "            element.style.display = 'none';" +
187            "            triangle.innerHTML = '&#x25b6; ';" +
188            "        }" +
189            "    }" +
190            "</script>";
191
192    /** TODO: Make it a setting */
193    private static final String HTML_DETAILS_RELATIVE_PATH = "details.html";
194    private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt";
195
196    private int mCrashedTestsCount = 0;
197    private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>();
198    private List<AbstractResult> mExpectedFailures = new ArrayList<AbstractResult>();
199    private List<AbstractResult> mExpectedPasses = new ArrayList<AbstractResult>();
200    private List<AbstractResult> mUnexpectedPasses = new ArrayList<AbstractResult>();
201
202    private FileFilter mFileFilter;
203    private String mResultsRootDirPath;
204    private String mTestsRelativePath;
205    private Date mDate;
206
207    public Summarizer(FileFilter fileFilter, String resultsRootDirPath) {
208        mFileFilter = fileFilter;
209        mResultsRootDirPath = resultsRootDirPath;
210    }
211
212    public static URI getDetailsUri() {
213        return new File(ManagerService.RESULTS_ROOT_DIR_PATH + File.separator +
214                HTML_DETAILS_RELATIVE_PATH).toURI();
215    }
216
217    public void appendTest(AbstractResult result) {
218        String relativePath = result.getRelativePath();
219
220        if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
221            mCrashedTestsCount++;
222        }
223
224        if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
225            if (mFileFilter.isFail(relativePath)) {
226                mUnexpectedPasses.add(result);
227            } else {
228                mExpectedPasses.add(result);
229            }
230        } else {
231            if (mFileFilter.isFail(relativePath)) {
232                mExpectedFailures.add(result);
233            } else {
234                mUnexpectedFailures.add(result);
235            }
236        }
237    }
238
239    public void setTestsRelativePath(String testsRelativePath) {
240        mTestsRelativePath = testsRelativePath;
241    }
242
243    public void summarize() {
244        String webKitRevision = getWebKitRevision();
245        createHtmlDetails(webKitRevision);
246        createTxtSummary(webKitRevision);
247    }
248
249    public void reset() {
250        mCrashedTestsCount = 0;
251        mUnexpectedFailures.clear();
252        mExpectedFailures.clear();
253        mExpectedPasses.clear();
254        mDate = new Date();
255    }
256
257    private void createTxtSummary(String webKitRevision) {
258        StringBuilder txt = new StringBuilder();
259
260        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
261        txt.append("Path: " + mTestsRelativePath + "\n");
262        txt.append("Date: " + dateFormat.format(mDate) + "\n");
263        txt.append("Build fingerprint: " + Build.FINGERPRINT + "\n");
264        txt.append("WebKit version: " + getWebKitVersionFromUserAgentString() + "\n");
265        txt.append("WebKit revision: " + webKitRevision + "\n");
266
267        txt.append("TOTAL:                     " + getTotalTestCount() + "\n");
268        txt.append("CRASHED (among all tests): " + mCrashedTestsCount + "\n");
269        txt.append("UNEXPECTED FAILURES:       " + mUnexpectedFailures.size() + "\n");
270        txt.append("UNEXPECTED PASSES:         " + mUnexpectedPasses.size() + "\n");
271        txt.append("EXPECTED FAILURES:         " + mExpectedFailures.size() + "\n");
272        txt.append("EXPECTED PASSES:           " + mExpectedPasses.size() + "\n");
273
274        FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH),
275                txt.toString().getBytes(), false);
276    }
277
278    private void createHtmlDetails(String webKitRevision) {
279        StringBuilder html = new StringBuilder();
280
281        html.append("<html><head>");
282        html.append(CSS);
283        html.append(SCRIPT);
284        html.append("</head><body>");
285
286        createTopSummaryTable(webKitRevision, html);
287
288        createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
289
290        createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
291
292        createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
293
294        createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
295
296        html.append("</body></html>");
297
298        FsUtils.writeDataToStorage(new File(mResultsRootDirPath, HTML_DETAILS_RELATIVE_PATH),
299                html.toString().getBytes(), false);
300    }
301
302    private int getTotalTestCount() {
303        return mUnexpectedFailures.size() +
304                mUnexpectedPasses.size() +
305                mExpectedPasses.size() +
306                mExpectedFailures.size();
307    }
308
309    private String getWebKitVersionFromUserAgentString() {
310        Resources resources = new Resources(new AssetManager(), new DisplayMetrics(),
311                new Configuration());
312        String userAgent =
313                resources.getString(com.android.internal.R.string.web_user_agent);
314
315        Matcher matcher = Pattern.compile("AppleWebKit/([0-9]+?\\.[0-9])").matcher(userAgent);
316        if (matcher.find()) {
317            return matcher.group(1);
318        }
319        return "unknown";
320    }
321
322    private String getWebKitRevision() {
323        URL url = null;
324        try {
325            url = new URL(ForwarderManager.getHostSchemePort(false) + "ThirdPartyProject.prop");
326        } catch (MalformedURLException e) {
327            assert false;
328        }
329
330        String thirdPartyProjectContents = new String(FsUtils.readDataFromUrl(url));
331        Matcher matcher = Pattern.compile("^version=([0-9]+)", Pattern.MULTILINE).matcher(
332                thirdPartyProjectContents);
333        if (matcher.find()) {
334            return matcher.group(1);
335        }
336        return "unknown";
337    }
338
339    private void createTopSummaryTable(String webKitRevision, StringBuilder html) {
340        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
341        html.append("<h1>" + "Layout tests' results for: " +
342                (mTestsRelativePath.equals("") ? "all tests" : mTestsRelativePath) + "</h1>");
343        html.append("<h3>" + "Date: " + dateFormat.format(new Date()) + "</h3>");
344        html.append("<h3>" + "Build fingerprint: " + Build.FINGERPRINT + "</h3>");
345        html.append("<h3>" + "WebKit version: " + getWebKitVersionFromUserAgentString() + "</h3>");
346
347        html.append("<h3>" + "WebKit revision: ");
348        html.append("<a href=\"http://trac.webkit.org/browser/trunk?rev=" + webKitRevision +
349                "\" target=\"_blank\"><span class=\"path\">" + webKitRevision + "</span></a>");
350        html.append("</h3>");
351
352        html.append("<table class=\"summary\">");
353        createSummaryTableRow(html, "TOTAL", getTotalTestCount());
354        createSummaryTableRow(html, "CRASHED (among all tests)", mCrashedTestsCount);
355        createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailures.size());
356        createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPasses.size());
357        createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailures.size());
358        createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPasses.size());
359        html.append("</table>");
360    }
361
362    private void createSummaryTableRow(StringBuilder html, String caption, int size) {
363        html.append("<tr>");
364        html.append("    <td>" + caption + "</td>");
365        html.append("    <td>" + size + "</td>");
366        html.append("</tr>");
367    }
368
369    private void createResultsListWithDiff(StringBuilder html, String title,
370            List<AbstractResult> resultsList) {
371        String relativePath;
372        String id = "";
373        AbstractResult.ResultCode resultCode;
374
375        Collections.sort(resultsList);
376        html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
377        for (AbstractResult result : resultsList) {
378            relativePath = result.getRelativePath();
379            resultCode = result.getResultCode();
380            assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
381
382            html.append("<h3>");
383
384            /**
385             * Technically, two different paths could end up being the same, because
386             * ':' is a valid  character in a path. However, it is probably not going
387             * to cause any problems in this case
388             */
389            id = relativePath.replace(File.separator, ":");
390            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
391            html.append("return false;\">");
392            html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
393            html.append("<span class=\"path\">" + relativePath + "</span>");
394            html.append("</a>");
395
396            html.append(" <span class=\"listItem " + resultCode.name() + "\">");
397            html.append(resultCode.toString());
398            html.append("</span>");
399
400            /** Detect missing LTC function */
401            String additionalTextOutputString = result.getAdditionalTextOutputString();
402            if (additionalTextOutputString != null &&
403                    additionalTextOutputString.contains("com.android.dumprendertree") &&
404                    additionalTextOutputString.contains("has no method")) {
405                if (additionalTextOutputString.contains("LayoutTestController")) {
406                    html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
407                }
408                if (additionalTextOutputString.contains("EventSender")) {
409                    html.append(" <span class=\"listItem noEventSender\">");
410                    html.append("ES function missing</span>");
411                }
412            }
413
414            html.append("</h3>");
415            appendExpectedResultsSources(result, html);
416
417            html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
418            html.append(result.getDiffAsHtml());
419            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
420            html.append("return false;\">Hide</a>");
421            html.append(" | ");
422            html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
423            html.append(" target=\"_blank\">Show source</a>");
424            html.append("</div>");
425
426            html.append("<div class=\"space\"></div>");
427        }
428    }
429
430    private void createResultsListNoDiff(StringBuilder html, String title,
431            List<AbstractResult> resultsList) {
432        Collections.sort(resultsList);
433        html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
434        for (AbstractResult result : resultsList) {
435            html.append("<h3>");
436            html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() +
437                    "\"");
438            html.append(" target=\"_blank\">");
439            html.append("<span class=\"sqr\">&#x25a0; </span>");
440            html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
441            html.append("</a>");
442            html.append("</h3>");
443            appendExpectedResultsSources(result, html);
444            html.append("<div class=\"space\"></div>");
445        }
446    }
447
448    private static final void appendExpectedResultsSources(AbstractResult result,
449            StringBuilder html) {
450        String textSource = result.getExpectedTextResultPath();
451        String imageSource = result.getExpectedImageResultPath();
452
453        if (textSource != null) {
454            html.append("<span class=\"source\">Expected textual result from: ");
455            html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" +
456                    textSource + "\"");
457            html.append(" target=\"_blank\">");
458            html.append(textSource + "</a></span>");
459        }
460        if (imageSource != null) {
461            html.append("<span class=\"source\">Expected image result from: ");
462            html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" +
463                    imageSource + "\"");
464            html.append(" target=\"_blank\">");
465            html.append(imageSource + "</a></span>");
466        }
467    }
468
469    private static final URL getViewSourceUrl(String relativePath) {
470        URL url = null;
471        try {
472            url = new URL("http", "localhost", ForwarderManager.HTTP_PORT,
473                    "/WebKitTools/DumpRenderTree/android/view_source.php?src=" +
474                    relativePath);
475        } catch (MalformedURLException e) {
476            assert false : "relativePath=" + relativePath;
477        }
478        return url;
479    }
480}
481