CtsXmlResultReporter.java revision 27484532f14e2e44f899982263489a5467d45956
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.cts.tradefed.result; 18 19import com.android.cts.tradefed.build.CtsBuildHelper; 20import com.android.cts.tradefed.device.DeviceInfoCollector; 21import com.android.cts.tradefed.testtype.CtsTest; 22import com.android.cts.tradefed.util.CtsHostStore; 23import com.android.ddmlib.Log; 24import com.android.ddmlib.Log.LogLevel; 25import com.android.ddmlib.testrunner.TestIdentifier; 26import com.android.tradefed.build.IBuildInfo; 27import com.android.tradefed.build.IFolderBuildInfo; 28import com.android.tradefed.config.Option; 29import com.android.tradefed.config.Option.Importance; 30import com.android.tradefed.log.LogUtil.CLog; 31import com.android.tradefed.result.ITestInvocationListener; 32import com.android.tradefed.result.ITestSummaryListener; 33import com.android.tradefed.result.InputStreamSource; 34import com.android.tradefed.result.LogDataType; 35import com.android.tradefed.result.LogFileSaver; 36import com.android.tradefed.result.TestSummary; 37import com.android.tradefed.util.AbiFormatter; 38import com.android.tradefed.util.FileUtil; 39import com.android.tradefed.util.StreamUtil; 40 41import org.kxml2.io.KXmlSerializer; 42 43import java.io.File; 44import java.io.FileNotFoundException; 45import java.io.FileOutputStream; 46import java.io.IOException; 47import java.io.InputStream; 48import java.io.OutputStream; 49import java.util.List; 50import java.util.Map; 51import java.util.regex.Matcher; 52import java.util.regex.Pattern; 53 54/** 55 * Writes results to an XML files in the CTS format. 56 * <p/> 57 * Collects all test info in memory, then dumps to file when invocation is complete. 58 * <p/> 59 * Outputs xml in format governed by the cts_result.xsd 60 */ 61public class CtsXmlResultReporter implements ITestInvocationListener, ITestSummaryListener { 62 private static final String LOG_TAG = "CtsXmlResultReporter"; 63 64 static final String TEST_RESULT_FILE_NAME = "testResult.xml"; 65 static final String CTS_RESULT_FILE_VERSION = "4.4"; 66 private static final String[] CTS_RESULT_RESOURCES = {"cts_result.xsl", "cts_result.css", 67 "logo.gif", "newrule-green.png"}; 68 69 /** the XML namespace */ 70 static final String ns = null; 71 72 static final String RESULT_TAG = "TestResult"; 73 static final String PLAN_ATTR = "testPlan"; 74 static final String STARTTIME_ATTR = "starttime"; 75 76 @Option(name = "quiet-output", description = "Mute display of test results.") 77 private boolean mQuietOutput = false; 78 79 private static final String REPORT_DIR_NAME = "output-file-path"; 80 @Option(name=REPORT_DIR_NAME, description="root file system path to directory to store xml " + 81 "test results and associated logs. If not specified, results will be stored at " + 82 "<cts root>/repository/results") 83 protected File mReportDir = null; 84 85 // listen in on the plan option provided to CtsTest 86 @Option(name = CtsTest.PLAN_OPTION, description = "the test plan to run.") 87 private String mPlanName = "NA"; 88 89 // listen in on the continue-session option provided to CtsTest 90 @Option(name = CtsTest.CONTINUE_OPTION, description = "the test result session to continue.") 91 private Integer mContinueSessionId = null; 92 93 @Option(name = "result-server", description = "Server to publish test results.") 94 private String mResultServer; 95 96 protected IBuildInfo mBuildInfo; 97 private String mStartTime; 98 private String mDeviceSerial; 99 private TestResults mResults = new TestResults(); 100 private TestPackageResult mCurrentPkgResult = null; 101 private boolean mIsDeviceInfoRun = false; 102 private ResultReporter mReporter; 103 private File mLogDir; 104 private String mSuiteName; 105 private String mReferenceUrl; 106 107 private static final Pattern mCtsLogPattern = Pattern.compile("(.*)\\+\\+\\+\\+(.*)"); 108 109 @Option(name = AbiFormatter.FORCE_ABI_STRING, 110 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 111 importance = Importance.IF_UNSET) 112 private String mForceAbi = null; 113 114 public void setReportDir(File reportDir) { 115 mReportDir = reportDir; 116 } 117 118 /** 119 * {@inheritDoc} 120 */ 121 @Override 122 public void invocationStarted(IBuildInfo buildInfo) { 123 mBuildInfo = buildInfo; 124 if (!(buildInfo instanceof IFolderBuildInfo)) { 125 throw new IllegalArgumentException("build info is not a IFolderBuildInfo"); 126 } 127 IFolderBuildInfo ctsBuild = (IFolderBuildInfo)buildInfo; 128 CtsBuildHelper ctsBuildHelper = getBuildHelper(ctsBuild); 129 mDeviceSerial = buildInfo.getDeviceSerial() == null ? "unknown_device" : 130 buildInfo.getDeviceSerial(); 131 if (mContinueSessionId != null) { 132 CLog.d("Continuing session %d", mContinueSessionId); 133 // reuse existing directory 134 TestResultRepo resultRepo = new TestResultRepo(ctsBuildHelper.getResultsDir()); 135 mResults = resultRepo.getResult(mContinueSessionId); 136 if (mResults == null) { 137 throw new IllegalArgumentException(String.format("Could not find session %d", 138 mContinueSessionId)); 139 } 140 mPlanName = resultRepo.getSummaries().get(mContinueSessionId).getTestPlan(); 141 mStartTime = resultRepo.getSummaries().get(mContinueSessionId).getStartTime(); 142 mReportDir = resultRepo.getReportDir(mContinueSessionId); 143 } else { 144 if (mReportDir == null) { 145 mReportDir = ctsBuildHelper.getResultsDir(); 146 } 147 mReportDir = createUniqueReportDir(mReportDir); 148 149 mStartTime = getTimestamp(); 150 logResult("Created result dir %s", mReportDir.getName()); 151 } 152 mSuiteName = ctsBuildHelper.getSuiteName(); 153 mReporter = new ResultReporter(mResultServer, mSuiteName); 154 155 // TODO: allow customization of log dir 156 // create a unique directory for saving logs, with same name as result dir 157 File rootLogDir = getBuildHelper(ctsBuild).getLogsDir(); 158 mLogDir = new File(rootLogDir, mReportDir.getName()); 159 mLogDir.mkdirs(); 160 } 161 162 /** 163 * Create a unique directory for saving results. 164 * <p/> 165 * Currently using legacy CTS host convention of timestamp directory names. In case of 166 * collisions, will use {@link FileUtil} to generate unique file name. 167 * <p/> 168 * TODO: in future, consider using LogFileSaver to create build-specific directories 169 * 170 * @param parentDir the parent folder to create dir in 171 * @return the created directory 172 */ 173 private static synchronized File createUniqueReportDir(File parentDir) { 174 // TODO: in future, consider using LogFileSaver to create build-specific directories 175 176 File reportDir = new File(parentDir, TimeUtil.getResultTimestamp()); 177 if (reportDir.exists()) { 178 // directory with this timestamp exists already! Choose a unique, although uglier, name 179 try { 180 reportDir = FileUtil.createTempDir(TimeUtil.getResultTimestamp() + "_", parentDir); 181 } catch (IOException e) { 182 CLog.e(e); 183 CLog.e("Failed to create result directory %s", reportDir.getAbsolutePath()); 184 } 185 } else { 186 if (!reportDir.mkdirs()) { 187 // TODO: consider throwing an exception 188 CLog.e("mkdirs failed when attempting to create result directory %s", 189 reportDir.getAbsolutePath()); 190 } 191 } 192 return reportDir; 193 } 194 195 /** 196 * Helper method to retrieve the {@link CtsBuildHelper}. 197 * @param ctsBuild 198 */ 199 CtsBuildHelper getBuildHelper(IFolderBuildInfo ctsBuild) { 200 CtsBuildHelper buildHelper = new CtsBuildHelper(ctsBuild.getRootDir()); 201 try { 202 buildHelper.validateStructure(); 203 } catch (FileNotFoundException e) { 204 // just log an error - it might be expected if we failed to retrieve a build 205 CLog.e("Invalid CTS build %s", ctsBuild.getRootDir()); 206 } 207 return buildHelper; 208 } 209 210 /** 211 * {@inheritDoc} 212 */ 213 @Override 214 public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) { 215 try { 216 File logFile = getLogFileSaver().saveAndZipLogData(dataName, dataType, 217 dataStream.createInputStream()); 218 logResult(String.format("Saved log %s", logFile.getName())); 219 } catch (IOException e) { 220 CLog.e("Failed to write log for %s", dataName); 221 } 222 } 223 224 /** 225 * Return the {@link LogFileSaver} to use. 226 * <p/> 227 * Exposed for unit testing. 228 */ 229 LogFileSaver getLogFileSaver() { 230 return new LogFileSaver(mLogDir); 231 } 232 233 234 @Override 235 public void testRunStarted(String name, int numTests) { 236 mCurrentPkgResult = mResults.getOrCreatePackage(name); 237 mIsDeviceInfoRun = name.equals(DeviceInfoCollector.APP_PACKAGE_NAME); 238 } 239 240 /** 241 * {@inheritDoc} 242 */ 243 @Override 244 public void testStarted(TestIdentifier test) { 245 mCurrentPkgResult.insertTest(test); 246 } 247 248 /** 249 * {@inheritDoc} 250 */ 251 @Override 252 public void testFailed(TestFailure status, TestIdentifier test, String trace) { 253 mCurrentPkgResult.reportTestFailure(test, CtsTestStatus.FAIL, trace); 254 } 255 256 /** 257 * {@inheritDoc} 258 */ 259 @Override 260 public void testEnded(TestIdentifier test, Map<String, String> testMetrics) { 261 collectCtsResults(test, testMetrics); 262 mCurrentPkgResult.reportTestEnded(test); 263 } 264 265 /** 266 * Collect Cts results for both device and host tests to the package result. 267 * @param test test ran 268 * @param testMetrics test metrics which can contain performance result for device tests 269 */ 270 private void collectCtsResults(TestIdentifier test, Map<String, String> testMetrics) { 271 // device test can have performance results in testMetrics 272 String perfResult = CtsReportUtil.getCtsResultFromMetrics(testMetrics); 273 // host test should be checked in CtsHostStore. 274 if (perfResult == null) { 275 perfResult = CtsHostStore.removeCtsResult(mDeviceSerial, test.toString()); 276 } 277 if (perfResult != null) { 278 // CTS result is passed in Summary++++Details format. 279 // Extract Summary and Details, and pass them. 280 Matcher m = mCtsLogPattern.matcher(perfResult); 281 if (m.find()) { 282 mCurrentPkgResult.reportPerformanceResult(test, CtsTestStatus.PASS, m.group(1), 283 m.group(2)); 284 } else { 285 logResult("CTS Result unrecognizable:" + perfResult); 286 } 287 } 288 } 289 290 /** 291 * {@inheritDoc} 292 */ 293 @Override 294 public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) { 295 if (mIsDeviceInfoRun) { 296 mResults.populateDeviceInfoMetrics(runMetrics); 297 } else { 298 mCurrentPkgResult.populateMetrics(runMetrics); 299 } 300 } 301 302 /** 303 * {@inheritDoc} 304 */ 305 @Override 306 public void invocationEnded(long elapsedTime) { 307 if (mReportDir == null || mStartTime == null) { 308 // invocationStarted must have failed, abort 309 CLog.w("Unable to create XML report"); 310 return; 311 } 312 313 File reportFile = getResultFile(mReportDir); 314 createXmlResult(reportFile, mStartTime, elapsedTime); 315 copyFormattingFiles(mReportDir); 316 zipResults(mReportDir); 317 318 try { 319 mReporter.reportResult(reportFile, mReferenceUrl); 320 } catch (IOException e) { 321 CLog.e(e); 322 } 323 } 324 325 private void logResult(String format, Object... args) { 326 if (mQuietOutput) { 327 CLog.i(format, args); 328 } else { 329 Log.logAndDisplay(LogLevel.INFO, mDeviceSerial, String.format(format, args)); 330 } 331 } 332 333 /** 334 * Creates a report file and populates it with the report data from the completed tests. 335 */ 336 private void createXmlResult(File reportFile, String startTimestamp, long elapsedTime) { 337 String endTime = getTimestamp(); 338 OutputStream stream = null; 339 try { 340 stream = createOutputResultStream(reportFile); 341 KXmlSerializer serializer = new KXmlSerializer(); 342 serializer.setOutput(stream, "UTF-8"); 343 serializer.startDocument("UTF-8", false); 344 serializer.setFeature( 345 "http://xmlpull.org/v1/doc/features.html#indent-output", true); 346 serializer.processingInstruction("xml-stylesheet type=\"text/xsl\" " + 347 "href=\"cts_result.xsl\""); 348 serializeResultsDoc(serializer, startTimestamp, endTime); 349 serializer.endDocument(); 350 String msg = String.format("XML test result file generated at %s. Passed %d, " + 351 "Failed %d, Not Executed %d", mReportDir.getName(), 352 mResults.countTests(CtsTestStatus.PASS), 353 mResults.countTests(CtsTestStatus.FAIL), 354 mResults.countTests(CtsTestStatus.NOT_EXECUTED)); 355 logResult(msg); 356 logResult("Time: %s", TimeUtil.formatElapsedTime(elapsedTime)); 357 } catch (IOException e) { 358 Log.e(LOG_TAG, "Failed to generate report data"); 359 } finally { 360 StreamUtil.close(stream); 361 } 362 } 363 364 /** 365 * Output the results XML. 366 * 367 * @param serializer the {@link KXmlSerializer} to use 368 * @param startTime the user-friendly starting time of the test invocation 369 * @param endTime the user-friendly ending time of the test invocation 370 * @throws IOException 371 */ 372 private void serializeResultsDoc(KXmlSerializer serializer, String startTime, String endTime) 373 throws IOException { 374 serializer.startTag(ns, RESULT_TAG); 375 serializer.attribute(ns, PLAN_ATTR, mPlanName); 376 serializer.attribute(ns, STARTTIME_ATTR, startTime); 377 serializer.attribute(ns, "endtime", endTime); 378 serializer.attribute(ns, "version", CTS_RESULT_FILE_VERSION); 379 serializer.attribute(ns, "suite", mSuiteName); 380 if (mForceAbi != null) { 381 serializer.attribute(ns, "abi", mForceAbi); 382 } 383 mResults.serialize(serializer); 384 // TODO: not sure why, but the serializer doesn't like this statement 385 //serializer.endTag(ns, RESULT_TAG); 386 } 387 388 private File getResultFile(File reportDir) { 389 return new File(reportDir, TEST_RESULT_FILE_NAME); 390 } 391 392 /** 393 * Creates the output stream to use for test results. Exposed for mocking. 394 */ 395 OutputStream createOutputResultStream(File reportFile) throws IOException { 396 logResult("Created xml report file at file://%s", reportFile.getAbsolutePath()); 397 return new FileOutputStream(reportFile); 398 } 399 400 /** 401 * Copy the xml formatting files stored in this jar to the results directory 402 * 403 * @param resultsDir 404 */ 405 private void copyFormattingFiles(File resultsDir) { 406 for (String resultFileName : CTS_RESULT_RESOURCES) { 407 InputStream configStream = getClass().getResourceAsStream(String.format("/report/%s", 408 resultFileName)); 409 if (configStream != null) { 410 File resultFile = new File(resultsDir, resultFileName); 411 try { 412 FileUtil.writeToFile(configStream, resultFile); 413 } catch (IOException e) { 414 Log.w(LOG_TAG, String.format("Failed to write %s to file", resultFileName)); 415 } 416 } else { 417 Log.w(LOG_TAG, String.format("Failed to load %s from jar", resultFileName)); 418 } 419 } 420 } 421 422 /** 423 * Zip the contents of the given results directory. 424 * 425 * @param resultsDir 426 */ 427 private void zipResults(File resultsDir) { 428 try { 429 // create a file in parent directory, with same name as resultsDir 430 File zipResultFile = new File(resultsDir.getParent(), String.format("%s.zip", 431 resultsDir.getName())); 432 FileUtil.createZip(resultsDir, zipResultFile); 433 } catch (IOException e) { 434 Log.w(LOG_TAG, String.format("Failed to create zip for %s", resultsDir.getName())); 435 } 436 } 437 438 /** 439 * Get a String version of the current time. 440 * <p/> 441 * Exposed so unit tests can mock. 442 */ 443 String getTimestamp() { 444 return TimeUtil.getTimestamp(); 445 } 446 447 /** 448 * {@inheritDoc} 449 */ 450 @Override 451 public void testRunFailed(String errorMessage) { 452 // ignore 453 } 454 455 /** 456 * {@inheritDoc} 457 */ 458 @Override 459 public void testRunStopped(long elapsedTime) { 460 // ignore 461 } 462 463 /** 464 * {@inheritDoc} 465 */ 466 @Override 467 public void invocationFailed(Throwable cause) { 468 // ignore 469 } 470 471 /** 472 * {@inheritDoc} 473 */ 474 @Override 475 public TestSummary getSummary() { 476 return null; 477 } 478 479 /** 480 * {@inheritDoc} 481 */ 482 @Override 483 public void putSummary(List<TestSummary> summaries) { 484 // By convention, only store the first summary that we see as the summary URL. 485 if (summaries.isEmpty()) { 486 return; 487 } 488 489 mReferenceUrl = summaries.get(0).getSummary().getString(); 490 } 491} 492