/******************************************************************************* * Copyright (c) 2000, 2009 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * * Contributors: * IBM Corporation - initial API and implementation *******************************************************************************/ package org.eclipse.test.performance.ui; import java.io.PrintStream; import java.util.List; import java.util.StringTokenizer; import org.eclipse.test.internal.performance.results.db.BuildResults; import org.eclipse.test.internal.performance.results.db.ConfigResults; import org.eclipse.test.internal.performance.results.db.PerformanceResults; import org.eclipse.test.internal.performance.results.db.ScenarioResults; /** * Class used to print a scenario status table. */ public class ScenarioStatusTable { private String component; private PrintStream stream; private int jsIdCount; /** * Creates an HTML table of red x/green check for a scenario for each * configuration. */ public ScenarioStatusTable(String name, PrintStream stream) { this.component = name; this.stream = stream; } /** * Prints the HTML representation of scenario status table into the given stream. */ public void print(PerformanceResults performanceResults) { String baselineName = performanceResults.getBaselineName(); List scenarios = performanceResults.getComponentScenarios(this.component); int size = scenarios.size(); // Print titles printTitle(); this.stream.print("\n"); this.stream.print("\n"); this.stream.print("\n"); printColumnsTitle(size, performanceResults); // Print one line per scenario results this.jsIdCount = 0; for (int i=0; i\n"); this.stream.print(""); return; } BuildResults currentBuildResults = configResults.getCurrentBuildResults(); String failure = currentBuildResults.getFailure(); double[] deviation = configResults.getCurrentBuildDeltaInfo(); int confidence = Utils.confidenceLevel(deviation); boolean hasFailure = failure != null; String comment = currentBuildResults.getComment(); String image = Utils.getImage(confidence, hasFailure, comment != null); this.stream.print("

All "); this.stream.print(computeSize(scenarios)); this.stream.print(" scenarios

"); boolean hasSummary = scenarioResults.hasSummary(); if (hasSummary) this.stream.print(""); String scenarioBaseline = scenarioResults.getBaselineBuildName(); boolean hasBaseline = baselineName.equals(scenarioBaseline); if (!hasBaseline) { this.stream.print("*"); this.stream.print(scenarioResults.getShortName()); this.stream.print(" (vs. "); this.stream.print(scenarioBaseline); this.stream.print(")"); } else { this.stream.print(scenarioResults.getShortName()); } if (hasSummary) this.stream.print(""); this.stream.print("\n"); String[] configs = performanceResults.getConfigNames(true/*sort*/); int length = configs.length; for (int j=0; j\n"); } private int computeSize(List scenarios) { int size = scenarios.size(); int n = 0; for (int i=0; i
"); this.stream.print(columnTitle); this.stream.print("
\n"); } } /* * Print the scenario statistics value for the given configuration. */ private void printConfigStats(ScenarioResults scenarioResults, String config) { ConfigResults configResults = scenarioResults.getConfigResults(config); if (configResults == null || !configResults.isValid()) { this.stream.print("
n/a\n"); this.stream.print("\n"); } else { // create message with tooltip text including deviation with error plus failure message this.jsIdCount+=1; this.stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip"); this.stream.print(this.jsIdCount); this.stream.print("')\" onMouseout=\"hide_element('toolTip"); this.stream.print(this.jsIdCount); this.stream.print("')\" \nhref=\""); this.stream.print(configResults.getName()); this.stream.print('/'); this.stream.print(scenarioResults.getFileName()); this.stream.print(".html\">\n"); this.stream.print("\n"); this.stream.print(""); this.stream.print(failure); this.stream.print("\n"); } String result = Utils.failureMessage(deviation, false); this.stream.print(result); this.stream.print("\n"); } /* * Print the status table explanationtitle. */ private void printTitle() { this.stream.print("

Scenario Status

\n"); this.stream.print("The following table gives a complete but compact view of performance results for the component.
\n"); this.stream.print("Each line of the table shows the results for one scenario on all machines.

\n"); this.stream.print("The name of the scenario is in bold when its results are also displayed in the fingerprints
\n"); this.stream.print("and starts with an '*' when the scenario has no results in the last baseline run.

\n"); this.stream.print("Here are information displayed for each test (ie. in each cell):\n"); this.stream.print("
    \n"); this.stream.print("
  • an icon showing whether the test fails or passes and whether it's reliable or not.
    \n"); this.stream.print("The legend for this icon is:\n"); this.stream.print("
      \n"); this.stream.print("
    • Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
    • "); this.stream.print("
    • Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
    • \n"); this.stream.print("
    • Gray (): mark a failing result (see above) with a comment explaining this degradation.
    • \n"); this.stream.print("
    • Yellow ( or ): mark a failing or successful result with a significant standard error (more than "); this.stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING); this.stream.print(")
    • \n"); this.stream.print("
    • Black (): mark an undefined result, which means that deviation on this test is not a number (NaN) or is infinite (happens when the reference value is equals to 0!)
    • "); this.stream.print("
    • \"n/a\": mark a test for with no performance results
    • \n"); this.stream.print("
  • \n"); this.stream.print("
  • the value of the deviation from the baseline as a percentage (ie. formula is: (build_test_time - baseline_test_time) / baseline_test_time)
  • \n"); this.stream.print("
  • the value of the standard error of this deviation as a percentage (ie. formula is: sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time)
    \n"); this.stream.print("When test only has one measure, the standard error cannot be computed and is replaced with a '[n/a]'.
  • \n"); this.stream.print("
\n"); this.stream.print("Hints:
    \n"); this.stream.print("
  • fly over image of failing tests to see the complete error message
  • \n"); this.stream.print("
  • to look at the complete and detailed test results, click on its image
  • \n"); this.stream.print("
\n"); } }