Summarizer.java revision 1b034781f4c45608e4d57e46cd46dfab9fc64746 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.dumprendertree2;
18
19import android.content.res.AssetManager;
20import android.content.res.Configuration;
21import android.content.res.Resources;
22import android.os.Build;
23import android.util.DisplayMetrics;
24import android.util.Log;
25
26import com.android.dumprendertree2.forwarder.ForwarderManager;
27
28import java.io.File;
29import java.net.MalformedURLException;
30import java.net.URI;
31import java.net.URL;
32import java.text.SimpleDateFormat;
33import java.util.ArrayList;
34import java.util.Collections;
35import java.util.Date;
36import java.util.List;
37import java.util.regex.Matcher;
38import java.util.regex.Pattern;
39
40/**
41 * A class that collects information about tests that ran and can create HTML
42 * files with summaries and easy navigation.
43 */
44public class Summarizer {
45
46    private static final String LOG_TAG = "Summarizer";
47
48    private static final String CSS =
49            "<style type=\"text/css\">" +
50            "* {" +
51            "       font-family: Verdana;" +
52            "       border: 0;" +
53            "       margin: 0;" +
54            "       padding: 0;}" +
55            "body {" +
56            "       margin: 10px;}" +
57            "h1 {" +
58            "       font-size: 24px;" +
59            "       margin: 4px 0 4px 0;}" +
60            "h2 {" +
61            "       font-size:18px;" +
62            "       text-transform: uppercase;" +
63            "       margin: 20px 0 3px 0;}" +
64            "h3, h3 a {" +
65            "       font-size: 14px;" +
66            "       color: black;" +
67            "       text-decoration: none;" +
68            "       margin-bottom: 4px;}" +
69            "h3 a span.path {" +
70            "       text-decoration: underline;}" +
71            "h3 span.tri {" +
72            "       text-decoration: none;" +
73            "       float: left;" +
74            "       width: 20px;}" +
75            "h3 span.sqr {" +
76            "       text-decoration: none;" +
77            "       color: #8ee100;" +
78            "       float: left;" +
79            "       width: 20px;}" +
80            "h3 img {" +
81            "       width: 8px;" +
82            "       margin-right: 4px;}" +
83            "div.diff {" +
84            "       margin-bottom: 25px;}" +
85            "div.diff a {" +
86            "       font-size: 12px;" +
87            "       color: #888;}" +
88            "table.visual_diff {" +
89            "       border-bottom: 0px solid;" +
90            "       border-collapse: collapse;" +
91            "       width: 100%;" +
92            "       margin-bottom: 2px;}" +
93            "table.visual_diff tr.headers td {" +
94            "       border-bottom: 1px solid;" +
95            "       border-top: 0;" +
96            "       padding-bottom: 3px;}" +
97            "table.visual_diff tr.results td {" +
98            "       border-top: 1px dashed;" +
99            "       border-right: 1px solid;" +
100            "       font-size: 15px;" +
101            "       vertical-align: top;}" +
102            "table.visual_diff tr.results td.line_count {" +
103            "       background-color:#aaa;" +
104            "       min-width:20px;" +
105            "       text-align: right;" +
106            "       border-right: 1px solid;" +
107            "       border-left: 1px solid;" +
108            "       padding: 2px 1px 2px 0px;}" +
109            "table.visual_diff tr.results td.line {" +
110            "       padding: 2px 0px 2px 4px;" +
111            "       border-right: 1px solid;" +
112            "       width: 49.8%;}" +
113            "table.visual_diff tr.footers td {" +
114            "       border-top: 1px solid;" +
115            "       border-bottom: 0;}" +
116            "table.visual_diff tr td.space {" +
117            "       border: 0;" +
118            "       width: 0.4%}" +
119            "div.space {" +
120            "       margin-top:4px;}" +
121            "span.eql {" +
122            "       background-color: #f3f3f3;}" +
123            "span.del {" +
124            "       background-color: #ff8888; }" +
125            "span.ins {" +
126            "       background-color: #88ff88; }" +
127            "span.fail {" +
128            "       color: red;}" +
129            "span.pass {" +
130            "       color: green;}" +
131            "span.time_out {" +
132            "       color: orange;}" +
133            "table.summary {" +
134            "       border: 1px solid black;" +
135            "       margin-top: 20px;}" +
136            "table.summary td {" +
137            "       padding: 3px;}" +
138            "span.listItem {" +
139            "       font-size: 11px;" +
140            "       font-weight: normal;" +
141            "       text-transform: uppercase;" +
142            "       padding: 3px;" +
143            "       -webkit-border-radius: 4px;}" +
144            "span." + AbstractResult.ResultCode.PASS.name() + "{" +
145            "       background-color: #8ee100;" +
146            "       color: black;}" +
147            "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
148            "       background-color: #ccc;" +
149            "       color: black;}" +
150            "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
151            "       background-color: #a700e4;" +
152            "       color: #fff;}" +
153            "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
154            "       background-color: #f3cb00;" +
155            "       color: black;}" +
156            "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
157            "       background-color: #c30000;" +
158            "       color: #fff;}" +
159            "span.noLtc {" +
160            "       background-color: #944000;" +
161            "       color: #fff;}" +
162            "span.noEventSender {" +
163            "       background-color: #815600;" +
164            "       color: #fff;}" +
165            "</style>";
166
167    private static final String SCRIPT =
168            "<script type=\"text/javascript\">" +
169            "    function toggleDisplay(id) {" +
170            "        element = document.getElementById(id);" +
171            "        triangle = document.getElementById('tri.' + id);" +
172            "        if (element.style.display == 'none') {" +
173            "            element.style.display = 'inline';" +
174            "            triangle.innerHTML = '&#x25bc; ';" +
175            "        } else {" +
176            "            element.style.display = 'none';" +
177            "            triangle.innerHTML = '&#x25b6; ';" +
178            "        }" +
179            "    }" +
180            "</script>";
181
182    /** TODO: Make it a setting */
183    private static final String HTML_DETAILS_RELATIVE_PATH = "details.html";
184    private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt";
185
186    private int mCrashedTestsCount = 0;
187    private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>();
188    private List<AbstractResult> mExpectedFailures = new ArrayList<AbstractResult>();
189    private List<String> mExpectedPasses = new ArrayList<String>();
190    private List<String> mUnexpectedPasses = new ArrayList<String>();
191
192    private FileFilter mFileFilter;
193    private String mResultsRootDirPath;
194
195    private String mTestsRelativePath;
196
197    private Date mDate;
198
199    public Summarizer(FileFilter fileFilter, String resultsRootDirPath) {
200        mFileFilter = fileFilter;
201        mResultsRootDirPath = resultsRootDirPath;
202    }
203
204    public static URI getDetailsUri() {
205        return new File(ManagerService.RESULTS_ROOT_DIR_PATH + File.separator +
206                HTML_DETAILS_RELATIVE_PATH).toURI();
207    }
208
209    public void appendTest(AbstractResult result) {
210        String relativePath = result.getRelativePath();
211
212        if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
213            mCrashedTestsCount++;
214        }
215
216        if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
217            if (mFileFilter.isFail(relativePath)) {
218                mUnexpectedPasses.add(relativePath);
219            } else {
220                mExpectedPasses.add(relativePath);
221            }
222        } else {
223            if (mFileFilter.isFail(relativePath)) {
224                mExpectedFailures.add(result);
225            } else {
226                mUnexpectedFailures.add(result);
227            }
228        }
229    }
230
231    public void setTestsRelativePath(String testsRelativePath) {
232        mTestsRelativePath = testsRelativePath;
233    }
234
235    public void summarize() {
236        createHtmlDetails();
237        createTxtSummary();
238    }
239
240    public void reset() {
241        mCrashedTestsCount = 0;
242        mUnexpectedFailures.clear();
243        mExpectedFailures.clear();
244        mExpectedPasses.clear();
245        mDate = new Date();
246    }
247
248    private void createTxtSummary() {
249        StringBuilder txt = new StringBuilder();
250
251        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
252        txt.append(mTestsRelativePath + "\n");
253        txt.append("Date: " + dateFormat.format(mDate) + "\n");
254        txt.append("Build fingerprint: " + Build.FINGERPRINT + "\n");
255        txt.append("WebKit version: " + getWebKitVersionFromUserAgentString() + "\n");
256
257        txt.append("TOTAL:   " + getTotalTestCount() + "\n");
258        if (mCrashedTestsCount > 0) {
259            txt.append("CRASHED (total among all tests): " + mCrashedTestsCount + "\n");
260            txt.append("-------------");
261        }
262        txt.append("UNEXPECTED FAILURES: " + mUnexpectedFailures.size() + "\n");
263        txt.append("UNEXPECTED PASSES:   " + mUnexpectedPasses.size() + "\n");
264        txt.append("EXPECTED FAILURES:   " + mExpectedFailures.size() + "\n");
265        txt.append("EXPECTED PASSES:     " + mExpectedPasses.size() + "\n");
266
267        FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH),
268                txt.toString().getBytes(), false);
269    }
270
271    private void createHtmlDetails() {
272        StringBuilder html = new StringBuilder();
273
274        html.append("<html><head>");
275        html.append(CSS);
276        html.append(SCRIPT);
277        html.append("</head><body>");
278
279        createTopSummaryTable(html);
280
281        createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
282
283        createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
284
285        createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
286
287        createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
288
289        html.append("</body></html>");
290
291        FsUtils.writeDataToStorage(new File(mResultsRootDirPath, HTML_DETAILS_RELATIVE_PATH),
292                html.toString().getBytes(), false);
293    }
294
295    private int getTotalTestCount() {
296        return mUnexpectedFailures.size() +
297                mUnexpectedPasses.size() +
298                mExpectedPasses.size() +
299                mExpectedFailures.size();
300    }
301
302    private String getWebKitVersionFromUserAgentString() {
303        Resources resources = new Resources(new AssetManager(), new DisplayMetrics(),
304                new Configuration());
305        String userAgent =
306                resources.getString(com.android.internal.R.string.web_user_agent);
307
308        Matcher matcher = Pattern.compile("AppleWebKit/([0-9]+?\\.[0-9])").matcher(userAgent);
309        if (matcher.find()) {
310            return matcher.group(1);
311        }
312        return "unknown";
313    }
314
315    private void createTopSummaryTable(StringBuilder html) {
316        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
317        html.append("<h1>" + mTestsRelativePath + "</h1>");
318        html.append("<h3>" + "Date: " + dateFormat.format(new Date()) + "</h3>");
319        html.append("<h3>" + "Build fingerprint: " + Build.FINGERPRINT + "</h3>");
320        html.append("<h3>" + "WebKit version: " + getWebKitVersionFromUserAgentString() + "</h3>");
321
322        html.append("<table class=\"summary\">");
323        createSummaryTableRow(html, "TOTAL", getTotalTestCount());
324        createSummaryTableRow(html, "CRASHED", mCrashedTestsCount);
325        createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailures.size());
326        createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPasses.size());
327        createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailures.size());
328        createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPasses.size());
329        html.append("</table>");
330    }
331
332    private void createSummaryTableRow(StringBuilder html, String caption, int size) {
333        html.append("<tr>");
334        html.append("    <td>" + caption + "</td>");
335        html.append("    <td>" + size + "</td>");
336        html.append("</tr>");
337    }
338
339    private void createResultsListWithDiff(StringBuilder html, String title,
340            List<AbstractResult> resultsList) {
341        String relativePath;
342        String id = "";
343        AbstractResult.ResultCode resultCode;
344
345        Collections.sort(resultsList);
346        html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
347        for (AbstractResult result : resultsList) {
348            relativePath = result.getRelativePath();
349            resultCode = result.getResultCode();
350            assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
351
352            html.append("<h3>");
353
354            /**
355             * Technically, two different paths could end up being the same, because
356             * ':' is a valid  character in a path. However, it is probably not going
357             * to cause any problems in this case
358             */
359            id = relativePath.replace(File.separator, ":");
360            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
361            html.append("return false;\">");
362            html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
363            html.append("<span class=\"path\">" + relativePath + "</span>");
364            html.append("</a>");
365
366            html.append(" <span class=\"listItem " + resultCode.name() + "\">");
367            html.append(resultCode.toString());
368            html.append("</span>");
369
370            /** Detect missing LTC function */
371            String additionalTextOutputString = result.getAdditionalTextOutputString();
372            if (additionalTextOutputString != null &&
373                    additionalTextOutputString.contains("com.android.dumprendertree") &&
374                    additionalTextOutputString.contains("has no method")) {
375                if (additionalTextOutputString.contains("LayoutTestController")) {
376                    html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
377                }
378                if (additionalTextOutputString.contains("EventSender")) {
379                    html.append(" <span class=\"listItem noEventSender\">");
380                    html.append("ES function missing</span>");
381                }
382            }
383
384            html.append("</h3>");
385
386            html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
387            html.append(result.getDiffAsHtml());
388            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
389            html.append("return false;\">Hide</a>");
390            html.append(" | ");
391            html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
392            html.append(" target=\"_blank\">Show source</a>");
393            html.append("</div>");
394
395            html.append("<div class=\"space\"></div>");
396        }
397    }
398
399    private void createResultsListNoDiff(StringBuilder html, String title,
400            List<String> resultsList) {
401        Collections.sort(resultsList);
402        html.append("<h2>" + title + "[" + resultsList.size() + "]</h2>");
403        for (String result : resultsList) {
404            html.append("<h3>");
405            html.append("<a href=\"" + getViewSourceUrl(result).toString() + "\"");
406            html.append(" target=\"_blank\">");
407            html.append("<span class=\"sqr\">&#x25a0; </span>");
408            html.append("<span class=\"path\">" + result + "</span>");
409            html.append("</a>");
410            html.append("</h3>");
411            html.append("<div class=\"space\"></div>");
412        }
413    }
414
415    private static final URL getViewSourceUrl(String relativePath) {
416        URL url = null;
417        try {
418            url = new URL("http", "localhost", ForwarderManager.HTTP_PORT,
419                    "/WebKitTools/DumpRenderTree/android/view_source.php?src=" +
420                    relativePath);
421        } catch (MalformedURLException e) {
422            assert false : "relativePath=" + relativePath;
423        }
424        return url;
425    }
426}