1/******************************************************************************* 2 * Copyright (c) 2000, 2009 IBM Corporation and others. 3 * All rights reserved. This program and the accompanying materials 4 * are made available under the terms of the Eclipse Public License v1.0 5 * which accompanies this distribution, and is available at 6 * http://www.eclipse.org/legal/epl-v10.html 7 * 8 * Contributors: 9 * IBM Corporation - initial API and implementation 10 *******************************************************************************/ 11package org.eclipse.test.performance.ui; 12 13import java.io.BufferedOutputStream; 14import java.io.File; 15import java.io.FileNotFoundException; 16import java.io.FileOutputStream; 17import java.io.PrintStream; 18import java.text.SimpleDateFormat; 19import java.util.ArrayList; 20import java.util.Arrays; 21import java.util.Date; 22import java.util.List; 23import java.util.StringTokenizer; 24 25import org.eclipse.core.runtime.IProgressMonitor; 26import org.eclipse.core.runtime.IStatus; 27import org.eclipse.core.runtime.OperationCanceledException; 28import org.eclipse.core.runtime.Status; 29import org.eclipse.core.runtime.SubMonitor; 30import org.eclipse.swt.widgets.Display; 31import org.eclipse.test.internal.performance.results.db.ConfigResults; 32import org.eclipse.test.internal.performance.results.db.DB_Results; 33import org.eclipse.test.internal.performance.results.db.PerformanceResults; 34import org.eclipse.test.internal.performance.results.db.ScenarioResults; 35import org.eclipse.test.internal.performance.results.utils.Util; 36import org.osgi.framework.Bundle; 37 38/** 39 * Main class to generate performance results of all scenarios matching a given pattern 40 * in one HTML page per component. 41 * 42 * @see #printUsage() method to see a detailed parameters usage 43 */ 44public class GenerateResults { 45 46/** 47 * Prefix of baseline builds displayed in data graphs. 48 * This field is set using <b>-baseline.prefix</b> argument. 49 * <p> 50 * Example: 51 * <pre>-baseline.prefix 3.2_200606291905</pre> 52 * 53 * @see #currentBuildPrefixes 54 */ 55String baselinePrefix = null; 56 57/** 58 * Root directory where all files are generated. 59 * This field is set using <b>-output</b> argument. 60 * <p> 61 * Example: 62 * <pre>-output /releng/results/I20070615-1200/performance</pre> 63 */ 64File outputDir; 65 66/** 67 * Root directory where all data are locally stored to speed-up generation. 68 * This field is set using <b>-dataDir</b> argument. 69 * <p> 70 * Example: 71 * <pre>-dataDir /tmp</pre> 72 */ 73File dataDir; 74 75/** 76 * Arrays of 2 strings which contains config information: name and description. 77 * This field is set using <b>-config</b> and/or <b>-config.properties</b> arguments. 78 * <p> 79 * Example: 80 * <pre> 81 * -config eclipseperflnx3_R3.3,eclipseperfwin2_R3.3,eclipseperflnx2_R3.3,eclipseperfwin1_R3.3,eclipseperflnx1_R3.3 82 * -config.properties 83 * "eclipseperfwin1_R3.3,Win XP Sun 1.4.2_08 (2 GHz 512 MB); 84 * eclipseperflnx1_R3.3,RHEL 3.0 Sun 1.4.2_08 (2 GHz 512 MB); 85 * eclipseperfwin2_R3.3,Win XP Sun 1.4.2_08 (3 GHz 2 GB); 86 * eclipseperflnx2_R3.3,RHEL 3.0 Sun 1.4.2_08 (3 GHz 2 GB); 87 * eclipseperflnx3_R3.3,RHEL 4.0 Sun 1.4.2_08 (3 GHz 2.5 GB)" 88 * </pre> 89 * Note that: 90 * <ul> 91 * <li>if only <b>-config</b> is set, then configuration name is used for description </li> 92 * <li>if only <b>-config.properties</b> is set, then all configurations defined with this argument are generated 93 * <li>if both arguments are defined, then only configurations defined by <b>-config</b> argument are generated, 94 * <b>-config.properties</b> argument is only used to set the configuration description.</li> 95 * </ul> 96 */ 97String[][] configDescriptors; 98 99/** 100 * Scenario pattern used to generate performance results. 101 * This field is set using <b>-scenarioPattern</b> argument. 102 * <p> 103 * Note that this pattern uses SQL conventions, not RegEx ones, 104 * which means that '%' is used to match several consecutive characters 105 * and '_' to match a single character. 106 * <p> 107 * Example: 108 * <pre>-scenario.pattern org.eclipse.%.test</pre> 109 */ 110String scenarioPattern; 111 112/** 113 * A list of prefixes for builds displayed in data graphs. 114 * This field is set using <b>-currentPrefix</b> argument. 115 * <p> 116 * Example: 117 * <pre>-current.prefix N, I</pre> 118 * 119 * @see #baselinePrefix 120 */ 121List currentBuildPrefixes; 122 123/** 124 * A list of prefixes of builds to highlight in displayed data graphs. 125 * This field is set using <b>-highlight</b> and/or <b>-highlight.latest</b> arguments. 126 * <p> 127 * Example: 128 * <pre>-higlight 3_2</pre> 129 */ 130List pointsOfInterest; 131 132/** 133 * Tells whether only fingerprints has to be generated. 134 * This field is set to <code>true</code> if <b>-fingerprints</b> argument is specified. 135 * <p> 136 * Default is <code>false</code> which means that scenario data 137 * will also be generated. 138 * 139 * @see #genData 140 * @see #genAll 141 */ 142boolean genFingerPrints = false; 143 144/** 145 * Tells whether only fingerprints has to be generated. 146 * This field is set to <code>true</code> if <b>-data</b> argument is specified. 147 * <p> 148 * Default is <code>false</code> which means that fingerprints 149 * will also be generated. 150 * 151 * @see #genFingerPrints 152 * @see #genAll 153 */ 154boolean genData = false; 155 156/** 157 * Tells whether only fingerprints has to be generated. 158 * This field is set to <code>false</code> 159 * if <b>-fingerprints</b> or <b>-data</b> argument is specified. 160 * <p> 161 * Default is <code>true</code> which means that scenario data 162 * will also be generated. 163 * 164 * @see #genData 165 * @see #genFingerPrints 166 */ 167boolean genAll = true; 168 169/** 170 * Tells whether information should be displayed in the console while generating. 171 * This field is set to <code>true</code> if <b>-print</b> argument is specified. 172 * <p> 173 * Default is <code>false</code> which means that nothing is print during the generation. 174 */ 175PrintStream printStream = null; 176 177/** 178 * Tells what should be the failure percentage threshold. 179 * <p> 180 * Default is 10%. 181 */ 182int failure_threshold = 10; // PerformanceTestPlugin.getDBLocation().startsWith("net://"); 183 184PerformanceResults performanceResults; 185 186public GenerateResults() { 187} 188 189public GenerateResults(PerformanceResults results, String current, String baseline, boolean fingerprints, File data, File output) { 190 this.dataDir = data; 191 this.outputDir = output; 192 this.genFingerPrints = fingerprints; 193 this.genAll = !fingerprints; 194 this.performanceResults = results; 195 this.printStream = System.out; 196 setDefaults(current, baseline); 197} 198 199/* 200 * Parse the command arguments and create corresponding performance 201 * results object. 202 */ 203private void parse(String[] args) { 204 StringBuffer buffer = new StringBuffer("Parameters used to generate performance results ("); 205 buffer.append(new SimpleDateFormat().format(new Date(System.currentTimeMillis()))); 206 buffer.append("):\n"); 207 int i = 0; 208 int argsLength = args.length; 209 if (argsLength == 0) { 210 printUsage(); 211 } 212 213 String currentBuildId = null; 214 String baseline = null; 215 String jvm = null; 216 this.configDescriptors = null; 217 218 while (i < argsLength) { 219 String arg = args[i]; 220 if (!arg.startsWith("-")) { 221 i++; 222 continue; 223 } 224 if (argsLength == i + 1 && i != argsLength - 1) { 225 System.out.println("Missing value for last parameter"); 226 printUsage(); 227 } 228 if (arg.equals("-baseline")) { 229 baseline = args[i + 1]; 230 if (baseline.startsWith("-")) { 231 System.out.println("Missing value for "+arg+" parameter"); 232 printUsage(); 233 } 234 buffer.append(" -baseline = "+baseline+'\n'); 235 i++; 236 continue; 237 } 238 if (arg.equals("-baseline.prefix")) { 239 this.baselinePrefix = args[i + 1]; 240 if (this.baselinePrefix.startsWith("-")) { 241 System.out.println("Missing value for "+arg+" parameter"); 242 printUsage(); 243 } 244 buffer.append(" ").append(arg).append(" = ").append(this.baselinePrefix).append('\n'); 245 i++; 246 continue; 247 } 248 if (arg.equals("-current.prefix")) { 249 String idPrefixList = args[i + 1]; 250 if (idPrefixList.startsWith("-")) { 251 System.out.println("Missing value for "+arg+" parameter"); 252 printUsage(); 253 } 254 buffer.append(" ").append(arg).append(" = "); 255 String[] ids = idPrefixList.split(","); 256 this.currentBuildPrefixes = new ArrayList(); 257 for (int j = 0; j < ids.length; j++) { 258 this.currentBuildPrefixes.add(ids[j]); 259 buffer.append(ids[j]); 260 } 261 buffer.append('\n'); 262 i++; 263 continue; 264 } 265 if (arg.equals("-highlight") || arg.equals("-highlight.latest")) { 266 if (args[i + 1].startsWith("-")) { 267 System.out.println("Missing value for "+arg+" parameter"); 268 printUsage(); 269 } 270 buffer.append(" ").append(arg).append(" = "); 271 String[] ids = args[i + 1].split(","); 272 this.pointsOfInterest = new ArrayList(); 273 for (int j = 0; j < ids.length; j++) { 274 this.pointsOfInterest.add(ids[j]); 275 buffer.append(ids[j]); 276 } 277 buffer.append('\n'); 278 i++; 279 continue; 280 } 281 if (arg.equals("-current")) { 282 currentBuildId = args[i + 1]; 283 if (currentBuildId.startsWith("-")) { 284 System.out.println("Missing value for "+arg+" parameter"); 285 printUsage(); 286 } 287 buffer.append(" ").append(arg).append(" = ").append(currentBuildId).append('\n'); 288 i++; 289 continue; 290 } 291 if (arg.equals("-jvm")) { 292 jvm = args[i + 1]; 293 if (jvm.startsWith("-")) { 294 System.out.println("Missing value for "+arg+" parameter"); 295 printUsage(); 296 } 297 buffer.append(" ").append(arg).append(" = ").append(jvm).append('\n'); 298 i++; 299 continue; 300 } 301 if (arg.equals("-output")) { 302 String dir = args[++i]; 303 if (dir.startsWith("-")) { 304 System.out.println("Missing value for "+arg+" parameter"); 305 printUsage(); 306 } 307 this.outputDir = new File(dir); 308 if (!this.outputDir.exists() && !this.outputDir.mkdirs()) { 309 System.err.println("Cannot create directory "+dir+" to write results in!"); 310 System.exit(2); 311 } 312 buffer.append(" ").append(arg).append(" = ").append(dir).append('\n'); 313 continue; 314 } 315 if (arg.equals("-dataDir")) { 316 String dir = args[++i]; 317 if (dir.startsWith("-")) { 318 System.out.println("Missing value for "+arg+" parameter"); 319 printUsage(); 320 } 321 this.dataDir = new File(dir); 322 if (!this.dataDir.exists() && !this.dataDir.mkdirs()) { 323 System.err.println("Cannot create directory "+dir+" to save data locally!"); 324 System.exit(2); 325 } 326 buffer.append(" ").append(arg).append(" = ").append(dir).append('\n'); 327 continue; 328 } 329 if (arg.equals("-config")) { 330 String configs = args[i + 1]; 331 if (configs.startsWith("-")) { 332 System.out.println("Missing value for "+arg+" parameter"); 333 printUsage(); 334 } 335 String[] names = configs.split(","); 336 int length = names.length; 337 buffer.append(" ").append(arg).append(" = "); 338 for (int j=0; j<length; j++) { 339 if (j>0) buffer.append(','); 340 buffer.append(names[j]); 341 } 342 if (this.configDescriptors == null) { 343 this.configDescriptors = new String[length][2]; 344 for (int j=0; j<length; j++) { 345 this.configDescriptors[j][0] = names[j]; 346 this.configDescriptors[j][1] = names[j]; 347 } 348 } else { 349 int confLength = this.configDescriptors[0].length; 350 int newLength = confLength; 351 mainLoop: for (int j=0; j<confLength; j++) { 352 for (int k=0; k<length; k++) { 353 if (this.configDescriptors[j][0].equals(names[k])) { 354 continue mainLoop; 355 } 356 } 357 this.configDescriptors[j][0] = null; 358 this.configDescriptors[j][1] = null; 359 newLength--; 360 } 361 if (newLength < confLength) { 362 String[][] newDescriptors = new String[newLength][2]; 363 for (int j=0, c=0; j<newLength; j++) { 364 if (this.configDescriptors[c] != null) { 365 newDescriptors[j][0] = this.configDescriptors[c][0]; 366 newDescriptors[j][1] = this.configDescriptors[c][1]; 367 } else { 368 c++; 369 } 370 } 371 this.configDescriptors = newDescriptors; 372 } 373 } 374 buffer.append('\n'); 375 i++; 376 continue; 377 } 378 if (arg.equals("-config.properties")) { 379 String configProperties = args[i + 1]; 380 if (configProperties.startsWith("-")) { 381 System.out.println("Missing value for "+arg+" parameter"); 382 printUsage(); 383 } 384 if (this.configDescriptors == null) { 385 System.out.println("Missing -config parameter"); 386 printUsage(); 387 } 388 int length = this.configDescriptors.length; 389 StringTokenizer tokenizer = new StringTokenizer(configProperties, ";"); 390 buffer.append('\t').append(arg).append(" = '").append(configProperties).append("' splitted in ").append(length).append(" configs:"); 391 while (tokenizer.hasMoreTokens()) { 392 String labelDescriptor = tokenizer.nextToken(); 393 String[] elements = labelDescriptor.trim().split(","); 394 for (int j=0; j<length; j++) { 395 if (elements[0].equals(this.configDescriptors[j][0])) { 396 this.configDescriptors[j][1] = elements[1]; 397 buffer.append("\n\t\t+ "); 398 buffer.append(elements[0]); 399 buffer.append(" -> "); 400 buffer.append(elements[1]); 401 } 402 } 403 } 404 buffer.append('\n'); 405 i++; 406 continue; 407 } 408 if (arg.equals("-scenario.filter") || arg.equals("-scenario.pattern")) { 409 this.scenarioPattern= args[i + 1]; 410 if (this.scenarioPattern.startsWith("-")) { 411 System.out.println("Missing value for "+arg+" parameter"); 412 printUsage(); 413 } 414 buffer.append(" ").append(arg).append(" = ").append(this.scenarioPattern).append('\n'); 415 i++; 416 continue; 417 } 418 if (arg.equals("-fingerprints")) { 419 this.genFingerPrints = true; 420 this.genAll = false; 421 buffer.append(" ").append(arg).append('\n'); 422 i++; 423 continue; 424 } 425 if (arg.equals("-data")) { 426 this.genData = true; 427 this.genAll = false; 428 buffer.append(" ").append(arg).append('\n'); 429 i++; 430 continue; 431 } 432 if (arg.equals("-print")) { 433 this.printStream = System.out; // default is to print to console 434 buffer.append(" ").append(arg); 435 i++; 436 String printFile = i==argsLength ? null : args[i]; 437 if (printFile==null ||printFile.startsWith("-")) { 438 buffer.append(" (to the console)").append('\n'); 439 } else { 440 try { 441 this.printStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(printFile))); 442 } 443 catch (FileNotFoundException fnfe) { 444 // use the console if the output file cannot be created 445 } 446 buffer.append(" (to file: ").append(printFile).append(")\n"); 447 } 448 continue; 449 } 450 if (arg.equals("-failure.threshold")) { 451 String value = args[i + 1]; 452 try { 453 this.failure_threshold = Integer.parseInt(value); 454 if (this.failure_threshold < 0) { 455 System.out.println("Value for "+arg+" parameter must be positive."); 456 printUsage(); 457 } 458 } 459 catch (NumberFormatException nfe) { 460 System.out.println("Invalid value for "+arg+" parameter"); 461 printUsage(); 462 } 463 buffer.append(" ").append(arg).append(" = ").append(value).append('\n'); 464 i++; 465 continue; 466 } 467 i++; 468 } 469 if (this.printStream != null) { 470 this.printStream.print(buffer.toString()); 471 } 472 473 // Stop if some mandatory parameters are missing 474 if (this.outputDir == null || this.configDescriptors == null || jvm == null) { 475 printUsage(); 476 } 477 478 // Set performance results 479 setPerformanceResults(currentBuildId, baseline); 480} 481 482/* 483 * Print component PHP file 484 */ 485private void printComponent(/*PerformanceResults performanceResults, */String component) throws FileNotFoundException { 486 if (this.printStream != null) this.printStream.print("."); 487 File outputFile = new File(this.outputDir, component + ".php"); 488 PrintStream stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile))); 489 490 // Print header 491 boolean isGlobal = component.startsWith("global"); 492 if (isGlobal) { 493 File globalFile = new File(this.outputDir, "global.php"); 494 PrintStream gStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(globalFile))); 495 gStream.print(Utils.HTML_OPEN); 496 gStream.print("</head>\n"); 497 gStream.print("<body>\n"); 498 gStream.print("<?php\n"); 499 gStream.print(" include(\"global_fp.php\");\n"); 500 gStream.print("?>\n"); 501 gStream.print("<table border=0 cellpadding=2 cellspacing=5 width=\"100%\">\n"); 502 gStream.print("<tbody><tr> <td colspan=3 align=\"left\" bgcolor=\"#0080c0\" valign=\"top\"><b><font color=\"#ffffff\" face=\"Arial,Helvetica\">\n"); 503 gStream.print("Detailed performance data grouped by scenario prefix</font></b></td></tr></tbody></table>\n"); 504 gStream.print("<a href=\"org.eclipse.ant.php?\">org.eclipse.ant*</a><br>\n"); 505 gStream.print("<a href=\"org.eclipse.compare.php?\">org.eclipse.compare*</a><br>\n"); 506 gStream.print("<a href=\"org.eclipse.core.php?\">org.eclipse.core*</a><br>\n"); 507 gStream.print("<a href=\"org.eclipse.jdt.core.php?\">org.eclipse.jdt.core*</a><br>\n"); 508 gStream.print("<a href=\"org.eclipse.jdt.debug.php?\">org.eclipse.jdt.debug*</a><br>\n"); 509 gStream.print("<a href=\"org.eclipse.jdt.text.php?\">org.eclipse.jdt.text*</a><br>\n"); 510 gStream.print("<a href=\"org.eclipse.jdt.ui.php?\">org.eclipse.jdt.ui*</a><br>\n"); 511 gStream.print("<a href=\"org.eclipse.jface.php?\">org.eclipse.jface*</a><br>\n"); 512 gStream.print("<a href=\"org.eclipse.osgi.php?\">org.eclipse.osgi*</a><br>\n"); 513 gStream.print("<a href=\"org.eclipse.pde.api.tools.php?\">org.eclipse.pde.api.tools*</a><br>\n"); 514 gStream.print("<a href=\"org.eclipse.pde.ui.php?\">org.eclipse.pde.ui*</a><br>\n"); 515 gStream.print("<a href=\"org.eclipse.swt.php?\">org.eclipse.swt*</a><br>\n"); 516 gStream.print("<a href=\"org.eclipse.team.php?\">org.eclipse.team*</a><br>\n"); 517 gStream.print("<a href=\"org.eclipse.ua.php?\">org.eclipse.ua*</a><br>\n"); 518 gStream.print("<a href=\"org.eclipse.ui.php?\">org.eclipse.ui*</a><br><p><br><br>\n"); 519 gStream.print("</body>\n"); 520 gStream.print(Utils.HTML_CLOSE); 521 gStream.close(); 522 } else { 523 stream.print(Utils.HTML_OPEN); 524 } 525 stream.print("<link href=\""+Utils.TOOLTIP_STYLE+"\" rel=\"stylesheet\" type=\"text/css\">\n"); 526 stream.print("<script src=\""+Utils.TOOLTIP_SCRIPT+"\"></script>\n"); 527 stream.print("<script src=\""+Utils.FINGERPRINT_SCRIPT+"\"></script>\n"); 528 stream.print(Utils.HTML_DEFAULT_CSS); 529 530 // Print title 531 stream.print("<body>"); 532 printComponentTitle(/*performanceResults, */component, isGlobal, stream); 533 534 // print the html representation of fingerprint for each config 535 Display display = Display.getDefault(); 536 if (this.genFingerPrints || this.genAll) { 537 final FingerPrint fingerprint = new FingerPrint(component, stream, this.outputDir); 538 display.syncExec( 539 new Runnable() { 540 public void run(){ 541 try { 542 fingerprint.print(GenerateResults.this.performanceResults); 543 } catch (Exception ex) { 544 ex.printStackTrace(); 545 } 546 } 547 } 548 ); 549 } 550// FingerPrint fingerprint = new FingerPrint(component, stream, this.outputDir); 551// fingerprint.print(performanceResults); 552 553 // print scenario status table 554 if (!isGlobal) { 555 // print the component scenario status table beneath the fingerprint 556 final ScenarioStatusTable sst = new ScenarioStatusTable(component, stream); 557 display.syncExec( 558 new Runnable() { 559 public void run(){ 560 try { 561 sst.print(GenerateResults.this.performanceResults); 562 } catch (Exception ex) { 563 ex.printStackTrace(); 564 } 565 } 566 } 567 ); 568// ScenarioStatusTable sst = new ScenarioStatusTable(component, stream); 569// sst.print(performanceResults); 570 } 571 572 stream.print(Utils.HTML_CLOSE); 573 stream.close(); 574} 575 576private void printComponentTitle(/*PerformanceResults performanceResults, */String component, boolean isGlobal, PrintStream stream) { 577 String baselineName = this.performanceResults.getBaselineName(); 578 String currentName = this.performanceResults.getName(); 579 580 // Print title line 581 stream.print("<h3>Performance of "); 582 if (!isGlobal) { 583 stream.print(component); 584 stream.print(": "); 585 } 586 stream.print(currentName); 587 stream.print(" relative to "); 588 int index = baselineName.indexOf('_'); 589 if (index > 0) { 590 stream.print(baselineName.substring(0, index)); 591 stream.print(" ("); 592 index = baselineName.lastIndexOf('_'); 593 stream.print(baselineName.substring(index+1, baselineName.length())); 594 stream.print(')'); 595 } else { 596 stream.print(baselineName); 597 } 598 stream.print("</h3>\n"); 599 600 // Print reference to global results 601 if (!isGlobal) { 602 stream.print("<?php\n"); 603 stream.print(" $type=$_SERVER['QUERY_STRING'];\n"); 604 stream.print(" if ($type==\"\") {\n"); 605 stream.print(" $type=\"fp_type=0\";\n"); 606 stream.print(" }\n"); 607 stream.print(" $href=\"<a href=\\\"performance.php?\";\n"); 608 stream.print(" $href=$href . $type . \"\\\">Back to global results</a><br><br>\";\n"); 609 stream.print(" echo $href;\n"); 610 stream.print("?>\n"); 611 } 612} 613 614/* 615 * Print summary of coefficient of variation for each scenario of the given pattern 616 * both for baseline and current builds. 617 */ 618private void printSummary(/*PerformanceResults performanceResults*/) { 619 long start = System.currentTimeMillis(); 620 if (this.printStream != null) this.printStream.print("Print scenarios variations summary..."); 621 File outputFile = new File(this.outputDir, "cvsummary.html"); 622 PrintStream stream = null; 623 try { 624 stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile))); 625 printSummaryPresentation(stream); 626// List scenarioNames = DB_Results.getScenarios(); 627// int size = scenarioNames.size(); 628 String[] components = this.performanceResults.getComponents(); 629 int componentsLength = components.length; 630 printSummaryColumnsTitle(stream/*, performanceResults*/); 631 String[] configs = this.performanceResults.getConfigNames(true/*sorted*/); 632 int configsLength = configs.length; 633 for (int i=0; i<componentsLength; i++) { 634 String componentName = components[i]; 635 List scenarioNames = this.performanceResults.getComponentScenarios(componentName); 636 int size = scenarioNames.size(); 637 for (int s=0; s<size; s++) { 638 String scenarioName = ((ScenarioResults) scenarioNames.get(s)).getName(); 639 if (scenarioName == null) continue; 640 ScenarioResults scenarioResults = this.performanceResults.getScenarioResults(scenarioName); 641 if (scenarioResults != null) { 642 stream.print("<tr>\n"); 643 for (int j=0; j<2; j++) { 644 for (int c=0; c<configsLength; c++) { 645 printSummaryScenarioLine(j, configs[c], scenarioResults, stream); 646 } 647 } 648 stream.print("<td>"); 649 stream.print(scenarioName); 650 stream.print("</td></tr>\n"); 651 } 652 } 653 } 654 } catch (Exception e) { 655 e.printStackTrace(); 656 } finally { 657 stream.print("</table></body></html>\n"); 658 stream.flush(); 659 stream.close(); 660 } 661 if (this.printStream != null) this.printStream.println("done in "+(System.currentTimeMillis()-start)+"ms"); 662} 663 664/* 665 * Print summary presentation (eg. file start and text presenting the purpose of this file contents).. 666 */ 667private void printSummaryPresentation(PrintStream stream) { 668 stream.print(Utils.HTML_OPEN); 669 stream.print(Utils.HTML_DEFAULT_CSS); 670 stream.print("<title>Summary of Elapsed Process Variation Coefficients</title></head>\n"); 671 stream.print("<body><h3>Summary of Elapsed Process Variation Coefficients</h3>\n"); 672 stream.print("<p> This table provides a bird's eye view of variability in elapsed process times\n"); 673 stream.print("for baseline and current build stream performance scenarios."); 674 stream.print(" This summary is provided to facilitate the identification of scenarios that should be examined due to high variability."); 675 stream.print("The variability for each scenario is expressed as a <a href=\"http://en.wikipedia.org/wiki/Coefficient_of_variation\">coefficient\n"); 676 stream.print("of variation</a> (CV). The CV is calculated by dividing the <b>standard deviation\n"); 677 stream.print("of the elapse process time over builds</b> by the <b>average elapsed process\n"); 678 stream.print("time over builds</b> and multiplying by 100.\n"); 679 stream.print("</p><p>High CV values may be indicative of any of the following:<br></p>\n"); 680 stream.print("<ol><li> an unstable performance test. </li>\n"); 681 stream.print("<ul><li>may be evidenced by an erratic elapsed process line graph.<br><br></li></ul>\n"); 682 stream.print("<li>performance regressions or improvements at some time in the course of builds.</li>\n"); 683 stream.print("<ul><li>may be evidenced by plateaus in elapsed process line graphs.<br><br></li></ul>\n"); 684 stream.print("<li>unstable testing hardware.\n"); 685 stream.print("<ul><li>consistent higher CV values for one test configuration as compared to others across"); 686 stream.print(" scenarios may be related to hardward problems.</li></ul></li></ol>\n"); 687 stream.print("<p> Scenarios are listed in alphabetical order in the far right column. A scenario's\n"); 688 stream.print("variation coefficients (CVs) are in columns to the left for baseline and current\n"); 689 stream.print("build streams for each test configuration. Scenarios with CVs > 10% are highlighted\n"); 690 stream.print("in yellow (10%<CV><CV<20%) and orange(CV>20%). </p>\n"); 691 stream.print("<p> Each CV value links to the scenario's detailed results to allow viewers to\n"); 692 stream.print("investigate the variability.</p>\n"); 693} 694 695/* 696 * Print columns titles of the summary table. 697 */ 698private void printSummaryColumnsTitle(PrintStream stream/*, PerformanceResults performanceResults*/) { 699 String[] configBoxes = this.performanceResults.getConfigBoxes(true/*sorted*/); 700 int length = configBoxes.length; 701 stream.print("<table border=\"1\"><tr><td colspan=\""); 702 stream.print(length); 703 stream.print("\"><b>Baseline CVs</b></td><td colspan=\""); 704 stream.print(length); 705 stream.print("\"><b>Current Build Stream CVs</b></td><td rowspan=\"2\"><b>Scenario Name</b></td></tr>\n"); 706 stream.print("<tr>"); 707 for (int n=0; n<2; n++) { 708 for (int c=0; c<length; c++) { 709 stream.print("<td>"); 710 stream.print(configBoxes[c]); 711 stream.print("</td>"); 712 } 713 } 714 stream.print("</tr>\n"); 715} 716 717/* 718 * Print a scenario line in the summary table. 719 */ 720private void printSummaryScenarioLine(int i, String config, ScenarioResults scenarioResults, PrintStream stream) { 721 ConfigResults configResults = scenarioResults.getConfigResults(config); 722 if (configResults == null || !configResults.isValid()) { 723 stream.print("<td>n/a</td>"); 724 return; 725 } 726 String url = config + "/" + scenarioResults.getFileName()+".html"; 727 double[] stats = null; 728 if (i==0) { // baseline results 729 List baselinePrefixes; 730 if (this.baselinePrefix == null) { 731 baselinePrefixes = Util.BASELINE_BUILD_PREFIXES; 732 } else { 733 baselinePrefixes = new ArrayList(); 734 baselinePrefixes.add(this.baselinePrefix); 735 } 736 stats = configResults.getStatistics(baselinePrefixes); 737 } else { 738 stats = configResults.getStatistics(this.currentBuildPrefixes); 739 } 740 double variation = stats[3]; 741 if (variation > 0.1 && variation < 0.2) { 742 stream.print("<td bgcolor=\"yellow\">"); 743 } else if (variation >= 0.2) { 744 stream.print("<td bgcolor=\"FF9900\">"); 745 } else { 746 stream.print("<td>"); 747 } 748 stream.print("<a href=\""); 749 stream.print(url); 750 stream.print("\"/>"); 751 stream.print(Util.PERCENTAGE_FORMAT.format(variation)); 752 stream.print("</a></td>"); 753} 754 755/* 756 * Print usage in case one of the argument of the line was incorrect. 757 * Note that calling this method ends the program run due to final System.exit() 758 */ 759private void printUsage() { 760 System.out.println( 761 "Usage:\n\n" + 762 "-baseline\n" + 763 " Build id against which to compare results.\n" + 764 " Same as value specified for the \"build\" key in the eclipse.perf.config system property.\n\n" + 765 766 "[-baseline.prefix]\n" + 767 " Optional. Build id prefix used in baseline test builds and reruns. Used to plot baseline historical data.\n" + 768 " A common prefix used for the value of the \"build\" key in the eclipse.perf.config system property when rerunning baseline tests.\n\n" + 769 770 "-current\n" + 771 " build id for which to generate results. Compared to build id specified in -baseline parameter above.\n" + 772 " Same as value specified for the \"build\" key in the eclipse.perf.config system property. \n\n" + 773 774 "[-current.prefix]\n" + 775 " Optional. Comma separated list of build id prefixes used in current build stream.\n" + 776 " Used to plot current build stream historical data. Defaults to \"N,I\".\n" + 777 " Prefixes for values specified for the \"build\" key in the eclipse.perf.config system property. \n\n" + 778 779 "-jvm\n" + 780 " Value specified in \"jvm\" key in eclipse.perf.config system property for current build.\n\n" + 781 782 "-config\n" + 783 " Comma separated list of config names for which to generate results.\n" + 784 " Same as values specified in \"config\" key in eclipse.perf.config system property.\n\n" + 785 786 "-output\n" + 787 " Path to default output directory.\n\n" + 788 789 "[-config.properties]\n" + 790 " Optional. Used by scenario status table to provide the following:\n" + 791 " alternate descriptions of config values to use in columns.\n" + 792 " The value should be specified in the following format:\n" + 793 " name1,description1;name2,description2;etc..\n\n" + 794 795 "[-highlight]\n" + 796 " Optional. Comma-separated list of build Id prefixes used to find most recent matching for each entry.\n" + 797 " Result used to highlight points in line graphs.\n\n" + 798 799 "[-scenario.pattern]\n" + 800 " Optional. Scenario prefix pattern to query database. If not specified,\n" + 801 " default of % used in query.\n\n" + 802 803 "[-fingerprints]\n" + 804 " Optional. Use to generate fingerprints only.\n\n" + 805 806 "[-data]\n" + 807 " Optional. Generates table of scenario reference and current data with line graphs.\n\n" + 808 809 "[-print]\n" + 810 " Optional. Display output in the console while generating.\n" + 811 812 "[-nophp]\n" + 813 " Optional. Generate files for non-php server.\n" + 814 815 "[-failure.threshold]\n" + 816 " Optional. Set the failure percentage threshold (default is 10%).\n" 817 ); 818 819 System.exit(1); 820} 821 822/** 823 * Run the generation from a list of arguments. 824 * Typically used to generate results from an application. 825 */ 826public IStatus run(String[] args) { 827 parse(args); 828 return run((IProgressMonitor) null); 829} 830 831/** 832 * Run the generation using a progress monitor. 833 * Note that all necessary information to generate properly must be set before 834 * calling this method 835 * 836 * @see #run(String[]) 837 */ 838public IStatus run(final IProgressMonitor monitor) { 839 long begin = System.currentTimeMillis(); 840 int work = 1100; 841 int dataWork = 1000 * this.performanceResults.getConfigBoxes(false).length; 842 if (this.genAll || this.genData) { 843 work += dataWork; 844 } 845 SubMonitor subMonitor = SubMonitor.convert(monitor, work); 846 try { 847 848 // Print whole scenarios summary 849 if (this.printStream != null) this.printStream.println(); 850 printSummary(/*performanceResults*/); 851 852 // Copy images and scripts to output dir 853 Bundle bundle = UiPlugin.getDefault().getBundle(); 854// URL images = bundle.getEntry("images"); 855// if (images != null) { 856// images = FileLocator.resolve(images); 857// Utils.copyImages(new File(images.getPath()), this.outputDir); 858// } 859 /* New way to get images 860 File content = FileLocator.getBundleFile(bundle); 861 BundleFile bundleFile; 862 if (content.isDirectory()) { 863 bundleFile = new DirBundleFile(content); 864 Utils.copyImages(bundleFile.getFile("images", true), this.outputDir); 865 } else { 866 bundleFile = new ZipBundleFile(content, null); 867 Enumeration imageFiles = bundle.findEntries("images", "*.gif", false); 868 while (imageFiles.hasMoreElements()) { 869 URL url = (URL) imageFiles.nextElement(); 870 Utils.copyFile(bundleFile.getFile("images"+File.separator+, true), this.outputDir); 871 } 872 } 873 */ 874 // Copy bundle files 875 Utils.copyBundleFiles(bundle, "images", "*.gif", this.outputDir); // images 876 Utils.copyBundleFiles(bundle, "scripts", "*.js", this.outputDir); // java scripts 877 Utils.copyBundleFiles(bundle, "scripts", "*.css", this.outputDir); // styles 878 Utils.copyBundleFiles(bundle, "doc", "*.html", this.outputDir); // doc 879 Utils.copyBundleFiles(bundle, "doc/images", "*.png", this.outputDir); // images for doc 880 /* 881 URL doc = bundle.getEntry("doc"); 882 if (doc != null) { 883 doc = FileLocator.resolve(doc); 884 File docDir = new File(doc.getPath()); 885 FileFilter filter = new FileFilter() { 886 public boolean accept(File pathname) { 887 return !pathname.getName().equals("CVS"); 888 } 889 }; 890 File[] docFiles = docDir.listFiles(filter); 891 for (int i=0; i<docFiles.length; i++) { 892 File file = docFiles[i]; 893 if (file.isDirectory()) { 894 File subdir = new File(this.outputDir, file.getName()); 895 subdir.mkdir(); 896 File[] subdirFiles = file.listFiles(filter); 897 for (int j=0; j<subdirFiles.length; j++) { 898 if (subdirFiles[i].isDirectory()) { 899 // expect only one sub-directory 900 } else { 901 Util.copyFile(subdirFiles[j], new File(subdir, subdirFiles[j].getName())); 902 } 903 } 904 } else { 905 Util.copyFile(file, new File(this.outputDir, file.getName())); 906 } 907 } 908 } 909 */ 910 911 // Print HTML pages and all linked files 912 if (this.printStream != null) { 913 this.printStream.println("Print performance results HTML pages:"); 914 this.printStream.print(" - components main page"); 915 } 916 long start = System.currentTimeMillis(); 917 subMonitor.setTaskName("Write fingerprints: 0%"); 918 subMonitor.subTask("Global..."); 919 printComponent(/*performanceResults, */"global_fp"); 920 subMonitor.worked(100); 921 if (subMonitor.isCanceled()) throw new OperationCanceledException(); 922 String[] components = this.performanceResults.getComponents(); 923 int length = components.length; 924 int step = 1000 / length; 925 int progress = 0; 926 for (int i=0; i<length; i++) { 927 int percentage = (int) ((progress / ((double) length)) * 100); 928 subMonitor.setTaskName("Write fingerprints: "+percentage+"%"); 929 subMonitor.subTask(components[i]+"..."); 930 printComponent(/*performanceResults, */components[i]); 931 subMonitor.worked(step); 932 if (subMonitor.isCanceled()) throw new OperationCanceledException(); 933 progress++; 934 } 935 if (this.printStream != null) { 936 String duration = Util.timeString(System.currentTimeMillis()-start); 937 this.printStream.println(" done in "+duration); 938 } 939 940 // Print the scenarios data 941 if (this.genData || this.genAll) { 942 start = System.currentTimeMillis(); 943 if (this.printStream != null) this.printStream.println(" - all scenarios data:"); 944 ScenarioData data = new ScenarioData(this.baselinePrefix, this.pointsOfInterest, this.currentBuildPrefixes, this.outputDir); 945 try { 946 data.print(this.performanceResults, this.printStream, subMonitor.newChild(dataWork)); 947 } catch (Exception ex) { 948 ex.printStackTrace(); 949 } 950 if (this.printStream != null) { 951 String duration = Util.timeString(System.currentTimeMillis()-start); 952 this.printStream.println(" => done in "+duration); 953 } 954 } 955 if (this.printStream != null) { 956 long time = System.currentTimeMillis(); 957 this.printStream.println("End of generation: "+new SimpleDateFormat("H:mm:ss").format(new Date(time))); 958 String duration = Util.timeString(System.currentTimeMillis()-begin); 959 this.printStream.println("=> done in "+duration); 960 } 961 return new Status(IStatus.OK, UiPlugin.getDefault().toString(), "Everything is OK"); 962 } 963 catch (OperationCanceledException oce) { 964 return new Status(IStatus.OK, UiPlugin.getDefault().toString(), "Generation was cancelled!"); 965 } 966 catch (Exception ex) { 967 return new Status(IStatus.ERROR, UiPlugin.getDefault().toString(), "An unexpected exception occurred!", ex); 968 } 969 finally { 970 if (this.printStream != null) { 971 this.printStream.flush(); 972 if (this.printStream != System.out) { 973 this.printStream.close(); 974 } 975 } 976 } 977} 978 979private void setDefaults(String buildName, String baseline) { 980 if (buildName == null) { 981 buildName = this.performanceResults.getName(); 982 } 983 984 // Set default output dir if not set 985 if (this.outputDir.getPath().indexOf(buildName) == -1) { 986 File dir = new File(this.outputDir, buildName); 987 if (dir.exists() || dir.mkdir()) { 988 this.outputDir = dir; 989 if (this.printStream != null) { 990 this.printStream.println(" + changed output dir to: "+dir.getPath()); 991 } 992 } 993 } 994 995 // Verify that build is known 996 String[] builds = this.performanceResults.getAllBuildNames(); 997 if (builds == null || builds.length == 0) { 998 System.err.println("Cannot connect to database to generate results build '"+buildName+"'"); 999 System.exit(1); 1000 } 1001 if (Arrays.binarySearch(builds, buildName, Util.BUILD_DATE_COMPARATOR) < 0) { 1002 throw new RuntimeException("No results in database for build '"+buildName+"'"); 1003 } 1004 if (this.printStream != null) { 1005 this.printStream.println(); 1006 this.printStream.flush(); 1007 } 1008 1009 // Init baseline prefix if not set 1010 if (this.baselinePrefix == null) { 1011 int index = baseline.lastIndexOf('_'); 1012 if (index > 0) { 1013 this.baselinePrefix = baseline.substring(0, index); 1014 } else { 1015 this.baselinePrefix = DB_Results.getDbBaselinePrefix(); 1016 } 1017 } 1018 1019 // Init current build prefixes if not set 1020 if (this.currentBuildPrefixes == null) { 1021 this.currentBuildPrefixes = new ArrayList(); 1022 if (buildName.charAt(0) == 'M') { 1023 this.currentBuildPrefixes.add("M"); 1024 } else { 1025 this.currentBuildPrefixes.add("N"); 1026 } 1027 this.currentBuildPrefixes.add("I"); 1028 } 1029} 1030 1031private void setPerformanceResults(String buildName, String baselineName) { 1032 1033 // Set performance results 1034 this.performanceResults = new PerformanceResults(buildName, baselineName, this.baselinePrefix, this.printStream); 1035 1036 // Set defaults 1037 setDefaults(buildName, this.performanceResults.getBaselineName()); 1038 1039 // Read performance results data 1040 this.performanceResults.readAll(buildName, this.configDescriptors, this.scenarioPattern, this.dataDir, this.failure_threshold, null); 1041} 1042 1043/* (non-Javadoc) 1044 * @see org.eclipse.equinox.app.IApplication#stop() 1045 */ 1046public void stop() { 1047 // Do nothing 1048} 1049 1050}