Summarizer.java revision 2ca8acdb5622b03a4ef56159477087adcf87db62
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.dumprendertree2; 18 19import android.content.res.AssetManager; 20import android.content.res.Configuration; 21import android.content.res.Resources; 22import android.os.Build; 23import android.util.DisplayMetrics; 24import android.util.Log; 25 26import com.android.dumprendertree2.forwarder.ForwarderManager; 27 28import java.io.File; 29import java.net.MalformedURLException; 30import java.net.URI; 31import java.net.URL; 32import java.text.SimpleDateFormat; 33import java.util.ArrayList; 34import java.util.Collections; 35import java.util.Date; 36import java.util.List; 37import java.util.regex.Matcher; 38import java.util.regex.Pattern; 39 40/** 41 * A class that collects information about tests that ran and can create HTML 42 * files with summaries and easy navigation. 43 */ 44public class Summarizer { 45 46 private static final String LOG_TAG = "Summarizer"; 47 48 private static final String CSS = 49 "<style type=\"text/css\">" + 50 "* {" + 51 " font-family: Verdana;" + 52 " border: 0;" + 53 " margin: 0;" + 54 " padding: 0;}" + 55 "body {" + 56 " margin: 10px;}" + 57 "h1 {" + 58 " font-size: 24px;" + 59 " margin: 4px 0 4px 0;}" + 60 "h2 {" + 61 " font-size:18px;" + 62 " text-transform: uppercase;" + 63 " margin: 20px 0 3px 0;}" + 64 "h3, h3 a {" + 65 " font-size: 14px;" + 66 " color: black;" + 67 " text-decoration: none;" + 68 " margin-top: 4px;" + 69 " margin-bottom: 2px;}" + 70 "h3 a span.path {" + 71 " text-decoration: underline;}" + 72 "h3 span.tri {" + 73 " text-decoration: none;" + 74 " float: left;" + 75 " width: 20px;}" + 76 "h3 span.sqr {" + 77 " text-decoration: none;" + 78 " color: #8ee100;" + 79 " float: left;" + 80 " width: 20px;}" + 81 "span.source {" + 82 " display: block;" + 83 " font-size: 10px;" + 84 " color: #888;" + 85 " margin-left: 20px;" + 86 " margin-bottom: 1px;}" + 87 "span.source a {" + 88 " font-size: 10px;" + 89 " color: #888;}" + 90 "h3 img {" + 91 " width: 8px;" + 92 " margin-right: 4px;}" + 93 "div.diff {" + 94 " margin-bottom: 25px;}" + 95 "div.diff a {" + 96 " font-size: 12px;" + 97 " color: #888;}" + 98 "table.visual_diff {" + 99 " border-bottom: 0px solid;" + 100 " border-collapse: collapse;" + 101 " width: 100%;" + 102 " margin-bottom: 2px;}" + 103 "table.visual_diff tr.headers td {" + 104 " border-bottom: 1px solid;" + 105 " border-top: 0;" + 106 " padding-bottom: 3px;}" + 107 "table.visual_diff tr.results td {" + 108 " border-top: 1px dashed;" + 109 " border-right: 1px solid;" + 110 " font-size: 15px;" + 111 " vertical-align: top;}" + 112 "table.visual_diff tr.results td.line_count {" + 113 " background-color:#aaa;" + 114 " min-width:20px;" + 115 " text-align: right;" + 116 " border-right: 1px solid;" + 117 " border-left: 1px solid;" + 118 " padding: 2px 1px 2px 0px;}" + 119 "table.visual_diff tr.results td.line {" + 120 " padding: 2px 0px 2px 4px;" + 121 " border-right: 1px solid;" + 122 " width: 49.8%;}" + 123 "table.visual_diff tr.footers td {" + 124 " border-top: 1px solid;" + 125 " border-bottom: 0;}" + 126 "table.visual_diff tr td.space {" + 127 " border: 0;" + 128 " width: 0.4%}" + 129 "div.space {" + 130 " margin-top:4px;}" + 131 "span.eql {" + 132 " background-color: #f3f3f3;}" + 133 "span.del {" + 134 " background-color: #ff8888; }" + 135 "span.ins {" + 136 " background-color: #88ff88; }" + 137 "span.fail {" + 138 " color: red;}" + 139 "span.pass {" + 140 " color: green;}" + 141 "span.time_out {" + 142 " color: orange;}" + 143 "table.summary {" + 144 " border: 1px solid black;" + 145 " margin-top: 20px;}" + 146 "table.summary td {" + 147 " padding: 3px;}" + 148 "span.listItem {" + 149 " font-size: 11px;" + 150 " font-weight: normal;" + 151 " text-transform: uppercase;" + 152 " padding: 3px;" + 153 " -webkit-border-radius: 4px;}" + 154 "span." + AbstractResult.ResultCode.PASS.name() + "{" + 155 " background-color: #8ee100;" + 156 " color: black;}" + 157 "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" + 158 " background-color: #ccc;" + 159 " color: black;}" + 160 "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" + 161 " background-color: #a700e4;" + 162 " color: #fff;}" + 163 "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" + 164 " background-color: #f3cb00;" + 165 " color: black;}" + 166 "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" + 167 " background-color: #c30000;" + 168 " color: #fff;}" + 169 "span.noLtc {" + 170 " background-color: #944000;" + 171 " color: #fff;}" + 172 "span.noEventSender {" + 173 " background-color: #815600;" + 174 " color: #fff;}" + 175 "</style>"; 176 177 private static final String SCRIPT = 178 "<script type=\"text/javascript\">" + 179 " function toggleDisplay(id) {" + 180 " element = document.getElementById(id);" + 181 " triangle = document.getElementById('tri.' + id);" + 182 " if (element.style.display == 'none') {" + 183 " element.style.display = 'inline';" + 184 " triangle.innerHTML = '▼ ';" + 185 " } else {" + 186 " element.style.display = 'none';" + 187 " triangle.innerHTML = '▶ ';" + 188 " }" + 189 " }" + 190 "</script>"; 191 192 /** TODO: Make it a setting */ 193 private static final String HTML_DETAILS_RELATIVE_PATH = "details.html"; 194 private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt"; 195 196 private int mCrashedTestsCount = 0; 197 private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>(); 198 private List<AbstractResult> mExpectedFailures = new ArrayList<AbstractResult>(); 199 private List<AbstractResult> mExpectedPasses = new ArrayList<AbstractResult>(); 200 private List<AbstractResult> mUnexpectedPasses = new ArrayList<AbstractResult>(); 201 202 private FileFilter mFileFilter; 203 private String mResultsRootDirPath; 204 205 private String mTestsRelativePath; 206 207 private Date mDate; 208 209 public Summarizer(FileFilter fileFilter, String resultsRootDirPath) { 210 mFileFilter = fileFilter; 211 mResultsRootDirPath = resultsRootDirPath; 212 } 213 214 public static URI getDetailsUri() { 215 return new File(ManagerService.RESULTS_ROOT_DIR_PATH + File.separator + 216 HTML_DETAILS_RELATIVE_PATH).toURI(); 217 } 218 219 public void appendTest(AbstractResult result) { 220 String relativePath = result.getRelativePath(); 221 222 if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) { 223 mCrashedTestsCount++; 224 } 225 226 if (result.getResultCode() == AbstractResult.ResultCode.PASS) { 227 if (mFileFilter.isFail(relativePath)) { 228 mUnexpectedPasses.add(result); 229 } else { 230 mExpectedPasses.add(result); 231 } 232 } else { 233 if (mFileFilter.isFail(relativePath)) { 234 mExpectedFailures.add(result); 235 } else { 236 mUnexpectedFailures.add(result); 237 } 238 } 239 } 240 241 public void setTestsRelativePath(String testsRelativePath) { 242 mTestsRelativePath = testsRelativePath; 243 } 244 245 public void summarize() { 246 createHtmlDetails(); 247 createTxtSummary(); 248 } 249 250 public void reset() { 251 mCrashedTestsCount = 0; 252 mUnexpectedFailures.clear(); 253 mExpectedFailures.clear(); 254 mExpectedPasses.clear(); 255 mDate = new Date(); 256 } 257 258 private void createTxtSummary() { 259 StringBuilder txt = new StringBuilder(); 260 261 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); 262 txt.append(mTestsRelativePath + "\n"); 263 txt.append("Date: " + dateFormat.format(mDate) + "\n"); 264 txt.append("Build fingerprint: " + Build.FINGERPRINT + "\n"); 265 txt.append("WebKit version: " + getWebKitVersionFromUserAgentString() + "\n"); 266 txt.append("WebKit revision: " + getWebKitRevision() + "\n"); 267 268 txt.append("TOTAL: " + getTotalTestCount() + "\n"); 269 if (mCrashedTestsCount > 0) { 270 txt.append("CRASHED (total among all tests): " + mCrashedTestsCount + "\n"); 271 txt.append("-------------"); 272 } 273 txt.append("UNEXPECTED FAILURES: " + mUnexpectedFailures.size() + "\n"); 274 txt.append("UNEXPECTED PASSES: " + mUnexpectedPasses.size() + "\n"); 275 txt.append("EXPECTED FAILURES: " + mExpectedFailures.size() + "\n"); 276 txt.append("EXPECTED PASSES: " + mExpectedPasses.size() + "\n"); 277 278 FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH), 279 txt.toString().getBytes(), false); 280 } 281 282 private void createHtmlDetails() { 283 StringBuilder html = new StringBuilder(); 284 285 html.append("<html><head>"); 286 html.append(CSS); 287 html.append(SCRIPT); 288 html.append("</head><body>"); 289 290 createTopSummaryTable(html); 291 292 createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures); 293 294 createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses); 295 296 createResultsListWithDiff(html, "Expected failures", mExpectedFailures); 297 298 createResultsListNoDiff(html, "Expected passes", mExpectedPasses); 299 300 html.append("</body></html>"); 301 302 FsUtils.writeDataToStorage(new File(mResultsRootDirPath, HTML_DETAILS_RELATIVE_PATH), 303 html.toString().getBytes(), false); 304 } 305 306 private int getTotalTestCount() { 307 return mUnexpectedFailures.size() + 308 mUnexpectedPasses.size() + 309 mExpectedPasses.size() + 310 mExpectedFailures.size(); 311 } 312 313 private String getWebKitVersionFromUserAgentString() { 314 Resources resources = new Resources(new AssetManager(), new DisplayMetrics(), 315 new Configuration()); 316 String userAgent = 317 resources.getString(com.android.internal.R.string.web_user_agent); 318 319 Matcher matcher = Pattern.compile("AppleWebKit/([0-9]+?\\.[0-9])").matcher(userAgent); 320 if (matcher.find()) { 321 return matcher.group(1); 322 } 323 return "unknown"; 324 } 325 326 private String getWebKitRevision() { 327 URL url = null; 328 try { 329 url = new URL(ForwarderManager.getHostSchemePort(false) + "WEBKIT_MERGE_REVISION"); 330 } catch (MalformedURLException e) { 331 assert false; 332 } 333 334 String webkitMergeRevisionFileContents = new String(FsUtils.readDataFromUrl(url)); 335 Matcher matcher = 336 Pattern.compile("http://svn.webkit.org/repository/webkit/trunk@([0-9]+)").matcher( 337 webkitMergeRevisionFileContents); 338 if (matcher.find()) { 339 return matcher.group(1); 340 } 341 return "unknown"; 342 } 343 344 private void createTopSummaryTable(StringBuilder html) { 345 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); 346 html.append("<h1>" + mTestsRelativePath + "</h1>"); 347 html.append("<h3>" + "Date: " + dateFormat.format(new Date()) + "</h3>"); 348 html.append("<h3>" + "Build fingerprint: " + Build.FINGERPRINT + "</h3>"); 349 html.append("<h3>" + "WebKit version: " + getWebKitVersionFromUserAgentString() + "</h3>"); 350 351 String webkitRevision = getWebKitRevision(); 352 html.append("<h3>" + "WebKit revision: "); 353 html.append("<a href=\"http://trac.webkit.org/browser/trunk?rev=" + webkitRevision + 354 "\" target=\"_blank\"><span class=\"path\">" + webkitRevision + "</span></a>"); 355 html.append("</h3>"); 356 357 html.append("<table class=\"summary\">"); 358 createSummaryTableRow(html, "TOTAL", getTotalTestCount()); 359 createSummaryTableRow(html, "CRASHED", mCrashedTestsCount); 360 createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailures.size()); 361 createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPasses.size()); 362 createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailures.size()); 363 createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPasses.size()); 364 html.append("</table>"); 365 } 366 367 private void createSummaryTableRow(StringBuilder html, String caption, int size) { 368 html.append("<tr>"); 369 html.append(" <td>" + caption + "</td>"); 370 html.append(" <td>" + size + "</td>"); 371 html.append("</tr>"); 372 } 373 374 private void createResultsListWithDiff(StringBuilder html, String title, 375 List<AbstractResult> resultsList) { 376 String relativePath; 377 String id = ""; 378 AbstractResult.ResultCode resultCode; 379 380 Collections.sort(resultsList); 381 html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>"); 382 for (AbstractResult result : resultsList) { 383 relativePath = result.getRelativePath(); 384 resultCode = result.getResultCode(); 385 assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode; 386 387 html.append("<h3>"); 388 389 /** 390 * Technically, two different paths could end up being the same, because 391 * ':' is a valid character in a path. However, it is probably not going 392 * to cause any problems in this case 393 */ 394 id = relativePath.replace(File.separator, ":"); 395 html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); 396 html.append("return false;\">"); 397 html.append("<span class=\"tri\" id=\"tri." + id + "\">▶ </span>"); 398 html.append("<span class=\"path\">" + relativePath + "</span>"); 399 html.append("</a>"); 400 401 html.append(" <span class=\"listItem " + resultCode.name() + "\">"); 402 html.append(resultCode.toString()); 403 html.append("</span>"); 404 405 /** Detect missing LTC function */ 406 String additionalTextOutputString = result.getAdditionalTextOutputString(); 407 if (additionalTextOutputString != null && 408 additionalTextOutputString.contains("com.android.dumprendertree") && 409 additionalTextOutputString.contains("has no method")) { 410 if (additionalTextOutputString.contains("LayoutTestController")) { 411 html.append(" <span class=\"listItem noLtc\">LTC function missing</span>"); 412 } 413 if (additionalTextOutputString.contains("EventSender")) { 414 html.append(" <span class=\"listItem noEventSender\">"); 415 html.append("ES function missing</span>"); 416 } 417 } 418 419 html.append("</h3>"); 420 appendExpectedResultsSources(result, html); 421 422 html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">"); 423 html.append(result.getDiffAsHtml()); 424 html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); 425 html.append("return false;\">Hide</a>"); 426 html.append(" | "); 427 html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\""); 428 html.append(" target=\"_blank\">Show source</a>"); 429 html.append("</div>"); 430 431 html.append("<div class=\"space\"></div>"); 432 } 433 } 434 435 private void createResultsListNoDiff(StringBuilder html, String title, 436 List<AbstractResult> resultsList) { 437 Collections.sort(resultsList); 438 html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>"); 439 for (AbstractResult result : resultsList) { 440 html.append("<h3>"); 441 html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + 442 "\""); 443 html.append(" target=\"_blank\">"); 444 html.append("<span class=\"sqr\">■ </span>"); 445 html.append("<span class=\"path\">" + result.getRelativePath() + "</span>"); 446 html.append("</a>"); 447 html.append("</h3>"); 448 appendExpectedResultsSources(result, html); 449 html.append("<div class=\"space\"></div>"); 450 } 451 } 452 453 private static final void appendExpectedResultsSources(AbstractResult result, 454 StringBuilder html) { 455 String textSource = result.getExpectedTextResultPath(); 456 String imageSource = result.getExpectedImageResultPath(); 457 458 if (textSource != null) { 459 html.append("<span class=\"source\">Expected textual result from: "); 460 html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" + 461 textSource + "\""); 462 html.append(" target=\"_blank\">"); 463 html.append(textSource + "</a></span>"); 464 } 465 if (imageSource != null) { 466 html.append("<span class=\"source\">Expected image result from: "); 467 html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" + 468 imageSource + "\""); 469 html.append(" target=\"_blank\">"); 470 html.append(imageSource + "</a></span>"); 471 } 472 } 473 474 private static final URL getViewSourceUrl(String relativePath) { 475 URL url = null; 476 try { 477 url = new URL("http", "localhost", ForwarderManager.HTTP_PORT, 478 "/WebKitTools/DumpRenderTree/android/view_source.php?src=" + 479 relativePath); 480 } catch (MalformedURLException e) { 481 assert false : "relativePath=" + relativePath; 482 } 483 return url; 484 } 485}