1/******************************************************************************* 2 * Copyright (c) 2000, 2009 IBM Corporation and others. 3 * All rights reserved. This program and the accompanying materials 4 * are made available under the terms of the Eclipse Public License v1.0 5 * which accompanies this distribution, and is available at 6 * http://www.eclipse.org/legal/epl-v10.html 7 * 8 * Contributors: 9 * IBM Corporation - initial API and implementation 10 *******************************************************************************/ 11package org.eclipse.test.internal.performance.results.model; 12 13import java.util.ArrayList; 14import java.util.List; 15import java.util.Vector; 16 17import org.eclipse.core.runtime.IAdaptable; 18import org.eclipse.core.runtime.preferences.IEclipsePreferences; 19import org.eclipse.core.runtime.preferences.InstanceScope; 20import org.eclipse.jface.resource.ImageDescriptor; 21import org.eclipse.swt.graphics.Image; 22import org.eclipse.test.internal.performance.eval.StatisticsUtil; 23import org.eclipse.test.internal.performance.results.db.AbstractResults; 24import org.eclipse.test.internal.performance.results.db.BuildResults; 25import org.eclipse.test.internal.performance.results.db.ConfigResults; 26import org.eclipse.test.internal.performance.results.db.DB_Results; 27import org.eclipse.test.internal.performance.results.utils.IPerformancesConstants; 28import org.eclipse.test.internal.performance.results.utils.Util; 29import org.eclipse.ui.ISharedImages; 30import org.eclipse.ui.PlatformUI; 31import org.eclipse.ui.model.IWorkbenchAdapter; 32import org.eclipse.ui.views.properties.ComboBoxPropertyDescriptor; 33import org.eclipse.ui.views.properties.IPropertyDescriptor; 34import org.eclipse.ui.views.properties.IPropertySource; 35import org.eclipse.ui.views.properties.PropertyDescriptor; 36import org.eclipse.ui.views.properties.TextPropertyDescriptor; 37import org.osgi.service.prefs.BackingStoreException; 38 39/** 40 * An Organization Element 41 */ 42public abstract class ResultsElement implements IAdaptable, IPropertySource, IWorkbenchAdapter, Comparable { 43 44 // Image descriptors 45 private static final ISharedImages WORKBENCH_SHARED_IMAGES = PlatformUI.getWorkbench().getSharedImages(); 46 public static final Image ERROR_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_ERROR_TSK); 47 public static final ImageDescriptor ERROR_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_ERROR_TSK); 48 public static final Image WARN_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_WARN_TSK); 49 public static final ImageDescriptor WARN_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_WARN_TSK); 50 public static final Image INFO_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_INFO_TSK); 51 public static final ImageDescriptor INFO_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_INFO_TSK); 52 public static final Image HELP_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_LCL_LINKTO_HELP); 53 public static final ImageDescriptor HELP_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_LCL_LINKTO_HELP); 54 public static final ImageDescriptor FOLDER_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJ_FOLDER); 55 public static final ImageDescriptor CONNECT_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_ELCL_SYNCED); 56 57 // Model 58 ResultsElement parent; 59 AbstractResults results; 60 ResultsElement[] children; 61 String name; 62 int status = -1; 63 64 // Stats 65 double[] statistics; 66 67 // Status constants 68 // state 69 static final int UNKNOWN = 0x01; 70 static final int UNREAD = 0x02; 71 static final int READ = 0x04; 72 static final int MISSING = 0x08; 73 public static final int STATE_MASK = 0x0F; 74 // info 75 static final int SMALL_VALUE = 0x0010; 76 static final int STUDENT_TTEST = 0x0020; 77 public static final int INFO_MASK = 0x0030; 78 // warning 79 static final int NO_BASELINE = 0x0040; 80 static final int SINGLE_RUN = 0x0080; 81 static final int BIG_ERROR = 0x0100; 82 static final int NOT_STABLE = 0x0200; 83 static final int NOT_RELIABLE = 0x0400; 84 public static final int WARNING_MASK = 0x0FC0; 85 // error 86 static final int BIG_DELTA = 0x1000; 87 public static final int ERROR_MASK = 0xF000; 88 89 // Property descriptors 90 static final String P_ID_STATUS_INFO = "ResultsElement.status_info"; //$NON-NLS-1$ 91 static final String P_ID_STATUS_WARNING = "ResultsElement.status_warning"; //$NON-NLS-1$ 92 static final String P_ID_STATUS_ERROR = "ResultsElement.status_error"; //$NON-NLS-1$ 93 static final String P_ID_STATUS_COMMENT = "ResultsElement.status_comment"; //$NON-NLS-1$ 94 95 static final String P_STR_STATUS_INFO = " info"; //$NON-NLS-1$ 96 static final String P_STR_STATUS_WARNING = "warning"; //$NON-NLS-1$ 97 static final String P_STR_STATUS_ERROR = "error"; //$NON-NLS-1$ 98 static final String P_STR_STATUS_COMMENT = "comment"; //$NON-NLS-1$ 99 static final String[] NO_VALUES = new String[0]; 100 101 private static Vector DESCRIPTORS; 102 static final TextPropertyDescriptor COMMENT_DESCRIPTOR = new TextPropertyDescriptor(P_ID_STATUS_COMMENT, P_STR_STATUS_COMMENT); 103 static final TextPropertyDescriptor ERROR_DESCRIPTOR = new TextPropertyDescriptor(P_ID_STATUS_ERROR, P_STR_STATUS_ERROR); 104 static Vector initDescriptors(int status) { 105 DESCRIPTORS = new Vector(); 106 // Status category 107 DESCRIPTORS.add(getInfosDescriptor(status)); 108 DESCRIPTORS.add(getWarningsDescriptor(status)); 109 DESCRIPTORS.add(ERROR_DESCRIPTOR); 110 ERROR_DESCRIPTOR.setCategory("Status"); 111 // Survey category 112 DESCRIPTORS.add(COMMENT_DESCRIPTOR); 113 COMMENT_DESCRIPTOR.setCategory("Survey"); 114 return DESCRIPTORS; 115 } 116 static Vector getDescriptors() { 117 return DESCRIPTORS; 118 } 119 static ComboBoxPropertyDescriptor getInfosDescriptor(int status) { 120 List list = new ArrayList(); 121 if ((status & SMALL_VALUE) != 0) { 122 list.add("Some builds have tests with small values"); 123 } 124 if ((status & STUDENT_TTEST) != 0) { 125 list.add("Some builds have student-t test error over the threshold"); 126 } 127 String[] infos = new String[list.size()]; 128 if (list.size() > 0) { 129 list.toArray(infos); 130 } 131 ComboBoxPropertyDescriptor infoDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_INFO, P_STR_STATUS_INFO, infos); 132 infoDescriptor.setCategory("Status"); 133 return infoDescriptor; 134 } 135 static PropertyDescriptor getWarningsDescriptor(int status) { 136 List list = new ArrayList(); 137 if ((status & BIG_ERROR) != 0) { 138 list.add("Some builds have tests with error over 3%"); 139 } 140 if ((status & NOT_RELIABLE) != 0) { 141 list.add("Some builds have no reliable tests"); 142 } 143 if ((status & NOT_STABLE) != 0) { 144 list.add("Some builds have no stable tests"); 145 } 146 if ((status & NO_BASELINE) != 0) { 147 list.add("Some builds have no baseline to compare with"); 148 } 149 if ((status & SINGLE_RUN) != 0) { 150 list.add("Some builds have single run tests"); 151 } 152 String[] warnings = new String[list.size()]; 153 if (list.size() > 0) { 154 list.toArray(warnings); 155 } 156 ComboBoxPropertyDescriptor warningDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_WARNING, P_STR_STATUS_WARNING, warnings); 157 warningDescriptor.setCategory("Status"); 158 return warningDescriptor; 159 } 160 161ResultsElement() { 162} 163 164ResultsElement(AbstractResults results, ResultsElement parent) { 165 this.parent = parent; 166 this.results = results; 167} 168 169ResultsElement(String name, ResultsElement parent) { 170 this.parent = parent; 171 this.name = name; 172} 173 174public int compareTo(Object o) { 175 if (this.results == null) { 176 if (o instanceof ResultsElement && this.name != null) { 177 ResultsElement element = (ResultsElement) o; 178 return this.name.compareTo(element.getName()); 179 } 180 return -1; 181 } 182 if (o instanceof ResultsElement) { 183 return this.results.compareTo(((ResultsElement)o).results); 184 } 185 return -1; 186} 187 188abstract ResultsElement createChild(AbstractResults testResults); 189 190/* (non-Javadoc) 191 * Method declared on IAdaptable 192 */ 193public Object getAdapter(Class adapter) { 194 if (adapter == IPropertySource.class) { 195 return this; 196 } 197 if (adapter == IWorkbenchAdapter.class) { 198 return this; 199 } 200 return null; 201} 202 203/** 204 * Iterate the element children. 205 */ 206public ResultsElement[] getChildren() { 207 if (this.results == null) { 208 return new ResultsElement[0]; 209 } 210 if (this.children == null) { 211 initChildren(); 212 } 213 return this.children; 214} 215 216/* (non-Javadoc) 217 * Method declared on IWorkbenchAdapter 218 */ 219public Object[] getChildren(Object o) { 220 if (this.results == null) { 221 return new Object[0]; 222 } 223 if (this.children == null) { 224 initChildren(); 225 } 226 return this.children; 227} 228 229/* (non-Javadoc) 230 * Method declared on IPropertySource 231 */ 232public Object getEditableValue() { 233 return this; 234} 235 236final String getId() { 237 return getId(new StringBuffer()).toString(); 238} 239 240private StringBuffer getId(StringBuffer buffer) { 241 if (this.parent != null) { 242 return this.parent.getId(buffer).append('/').append(getName()); 243 } 244 return buffer.append(DB_Results.getDbName()); 245} 246 247/* (non-Javadoc) 248 * Method declared on IWorkbenchAdapter 249 */ 250public ImageDescriptor getImageDescriptor(Object object) { 251 if (object instanceof ResultsElement) { 252 ResultsElement resultsElement = (ResultsElement) object; 253// DEBUG 254// if (resultsElement.getName().equals("I20090806-0100")) { 255// if (resultsElement.results != null) { 256// String toString = resultsElement.results.getParent().toString(); 257// String toString = resultsElement.results.toString(); 258// if (toString.indexOf("testStoreExists")>0 && toString.indexOf("eplnx2")>0) { 259// System.out.println("stop"); 260// } 261// } 262// } 263 int elementStatus = resultsElement.getStatus(); 264 if (elementStatus == MISSING) { 265 return HELP_IMAGE_DESCRIPTOR; 266 } 267 if ((elementStatus & ResultsElement.ERROR_MASK) != 0) { 268 return ERROR_IMAGE_DESCRIPTOR; 269 } 270 if ((elementStatus & ResultsElement.WARNING_MASK) != 0) { 271 return WARN_IMAGE_DESCRIPTOR; 272 } 273 if ((elementStatus & ResultsElement.INFO_MASK) != 0) { 274 return INFO_IMAGE_DESCRIPTOR; 275 } 276 } 277 return null; 278} 279 280/* (non-Javadoc) 281 * Method declared on IWorkbenchAdapter 282 */ 283public String getLabel(Object o) { 284 return getName(); 285} 286 287/** 288 * Returns the name 289 */ 290public String getName() { 291 if (this.name == null && this.results != null) { 292 this.name = this.results.getName(); 293 } 294 return this.name; 295} 296 297/** 298 * Returns the parent 299 */ 300public Object getParent(Object o) { 301 return this.parent; 302} 303 304/* (non-Javadoc) 305 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyDescriptors() 306 */ 307public IPropertyDescriptor[] getPropertyDescriptors() { 308 Vector descriptors = getDescriptors(); 309 if (descriptors == null) { 310 descriptors = initDescriptors(getStatus()); 311 } 312 int size = descriptors.size(); 313 IPropertyDescriptor[] descriptorsArray = new IPropertyDescriptor[size]; 314 descriptorsArray[0] = getInfosDescriptor(getStatus()); 315 descriptorsArray[1] = getWarningsDescriptor(getStatus()); 316 for (int i=2; i<size; i++) { 317 descriptorsArray[i] = (IPropertyDescriptor) descriptors.get(i); 318 } 319 return descriptorsArray; 320} 321 322/* (non-Javadoc) 323 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyValue(java.lang.Object) 324 */ 325public Object getPropertyValue(Object propKey) { 326 if (propKey.equals(P_ID_STATUS_INFO)) { 327 if ((getStatus() & INFO_MASK) != 0) { 328 return new Integer(0); 329 } 330 } 331 if (propKey.equals(P_ID_STATUS_WARNING)) { 332 if ((getStatus() & WARNING_MASK) != 0) { 333 return new Integer(0); 334 } 335 } 336 if (propKey.equals(P_ID_STATUS_ERROR)) { 337 if ((getStatus() & BIG_DELTA) != 0) { 338 return "Some builds have tests with regression"; 339 } 340 } 341 if (propKey.equals(P_ID_STATUS_COMMENT)) { 342 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID); 343 return preferences.get(getId(), ""); 344 } 345 return null; 346} 347 348public ResultsElement getResultsElement(String resultName) { 349 int length = getChildren(null).length; 350 for (int i=0; i<length; i++) { 351 ResultsElement searchedResults = this.children[i]; 352 if (searchedResults.getName().equals(resultName)) { 353 return searchedResults; 354 } 355 } 356 return null; 357} 358 359/** 360 * Return the status of the element. 361 * 362 * The status is a bit mask pattern where digits are 363 * allowed as follow: 364 * <ul> 365 * <li>0-3: bits for state showing whether the element is 366 * <ul> 367 * <li>{@link #UNKNOWN} : not connected to a db</li> 368 * <li>{@link #UNREAD} : is not valid (e.g. NaN results)</li> 369 * <li>{@link #MISSING} : no results (e.g. the perf machine crashed and didn't store any results)</li> 370 * <li>{@link #READ} : has valid results</li> 371 * </ul> 372 * </li> 373 * <li>4-5: bits for information. Current possible information are 374 * <ul> 375 * <li>{@link #SMALL_VALUE} : build results or delta with baseline value is under 100ms</li> 376 * <li>{@link #STUDENT_TTEST} : the Student T-test is over the threshold (old yellow color for test results)</li> 377 * </ul> 378 * </li> 379 * <li>6-11: bits for warnings. Current possible warnings are 380 * <ul> 381 * <li>{@link #NO_BASELINE} : no baseline for the current build</li> 382 * <li>{@link #SINGLE_RUN} : the test has only one run (i.e. no error could be computed), hence its reliability cannot be evaluated</li> 383 * <li>{@link #BIG_ERROR} : the test result is over the 3% threshold</li> 384 * <li>{@link #NOT_STABLE} : the test history shows a deviation between 10% and 20% (may mean that this test is not so reliable)</li> 385 * <li>{@link #NOT_RELIABLE} : the test history shows a deviation over 20% (surely means that this test is too erratic to be reliable)</li> 386 * </ul> 387 * </li> 388 * <li>12-15: bits for errors. Current possible errors are 389 * <ul> 390 * <li>{@link #BIG_DELTA} : the delta for the test is over the 10% threshold</li> 391 * </ul> 392 * </li> 393 * </ul> 394 * 395 * Note that these explanation applied to {@link BuildResultsElement}, and {@link DimResultsElement}. 396 * For {@link ComponentResultsElement}, and {@link ScenarioResultsElement}, it's the merge of all the children status 397 * and means "Some tests have..." instead of "The test has...". For {@link ConfigResultsElement}, it means the status 398 * of the most recent build compared to its most recent baseline. 399 * 400 * @return An int with each bit set when the corresponding symptom applies. 401 */ 402public final int getStatus() { 403 if (this.status < 0) { 404 initStatus(); 405 } 406 return this.status; 407} 408 409/** 410 * Return the statistics of the build along its history. 411 * 412 * @return An array of double built as follows: 413 * <ul> 414 * <li>0: numbers of values</li> 415 * <li>1: mean of values</li> 416 * <li>2: standard deviation of these values</li> 417 * <li>3: coefficient of variation of these values</li> 418 * </ul> 419 */ 420double[] getStatistics() { 421 return this.statistics; 422} 423 424/** 425 * Returns whether the element (or one in its hierarchy) has an error. 426 * 427 * @return <code> true</code> if the element or one in its hierarchy has an error, 428 * <code> false</code> otherwise 429 */ 430public final boolean hasError() { 431 return (getStatus() & ERROR_MASK) != 0; 432} 433 434void initChildren() { 435 AbstractResults[] resultsChildren = this.results.getChildren(); 436 int length = resultsChildren.length; 437 this.children = new ResultsElement[length]; 438 int count = 0; 439 for (int i=0; i<length; i++) { 440 ResultsElement childElement = createChild(resultsChildren[i]); 441 if (childElement != null) { 442 this.children[count++] = childElement; 443 } 444 } 445 if (count < length) { 446 System.arraycopy(this.children, 0, this.children = new ResultsElement[count], 0, count); 447 } 448} 449void initStatus() { 450 this.status = READ; 451 if (this.results != null) { 452 if (this.children == null) initChildren(); 453 int length = this.children.length; 454 for (int i=0; i<length; i++) { 455 this.status |= this.children[i].getStatus(); 456 } 457 } 458} 459 460int initStatus(BuildResults buildResults) { 461 this.status = READ; 462 463 // Get values 464 double buildValue = buildResults.getValue(); 465 ConfigResults configResults = (ConfigResults) buildResults.getParent(); 466 BuildResults baselineResults = configResults.getBaselineBuildResults(buildResults.getName()); 467 double baselineValue = baselineResults.getValue(); 468 double delta = (baselineValue - buildValue) / baselineValue; 469 470 // Store if there's no baseline 471 if (Double.isNaN(delta)) { 472 this.status |= NO_BASELINE; 473 } 474 475 // Store if there's only one run 476 long baselineCount = baselineResults.getCount(); 477 long currentCount = buildResults.getCount(); 478 double error = Double.NaN; 479 if (baselineCount == 1 || currentCount == 1) { 480 this.status |= SINGLE_RUN; 481 } 482 483 // Store if the T-test is not good 484 double ttestValue = Util.computeTTest(baselineResults, buildResults); 485 int degreeOfFreedom = (int) (baselineResults.getCount()+buildResults.getCount()-2); 486 if (ttestValue >= 0 && StatisticsUtil.getStudentsT(degreeOfFreedom, StatisticsUtil.T90) >= ttestValue) { 487 this.status |= STUDENT_TTEST; 488 } 489 490 // Store if there's a big error (over 3%) 491 double baselineError = baselineResults.getError(); 492 double currentError = buildResults.getError(); 493 error = Double.isNaN(baselineError) 494 ? currentError / baselineValue 495 : Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue; 496 if (error > 0.03) { 497 this.status |= BIG_ERROR; 498 } 499 500 // Store if there's a big delta (over 10%) 501 if (delta <= -0.1) { 502 this.status |= BIG_DELTA; 503 double currentBuildValue = buildResults.getValue(); 504 double diff = Math.abs(baselineValue - currentBuildValue); 505 if (currentBuildValue < 100 || diff < 100) { // moderate the status when 506 // diff is less than 100ms 507 this.status |= SMALL_VALUE; 508 } else { 509 double[] stats = getStatistics(); 510 if (stats != null) { 511 if (stats[3] > 0.2) { // invalidate the status when the test 512 // historical deviation is over 20% 513 this.status |= NOT_RELIABLE; 514 } else if (stats[3] > 0.1) { // moderate the status when the test 515 // historical deviation is between 10% 516 // and 20% 517 this.status |= NOT_STABLE; 518 } 519 } 520 } 521 } 522 523 return this.status; 524} 525 526public boolean isInitialized() { 527 return this.results != null; 528} 529 530/* (non-Javadoc) 531 * Method declared on IPropertySource 532 */ 533public boolean isPropertySet(Object property) { 534 return false; 535} 536 537boolean onlyFingerprints() { 538 if (this.parent != null) { 539 return this.parent.onlyFingerprints(); 540 } 541 return ((PerformanceResultsElement)this).fingerprints; 542} 543 544/* (non-Javadoc) 545 * Method declared on IPropertySource 546 */ 547public void resetPropertyValue(Object property) { 548} 549 550void resetStatus() { 551 this.status = -1; 552 if (this.results != null) { 553 if (this.children == null) initChildren(); 554 int length = this.children.length; 555 for (int i=0; i<length; i++) { 556 this.children[i].resetStatus(); 557 } 558 } 559} 560 561public void setPropertyValue(Object name, Object value) { 562 if (name.equals(P_ID_STATUS_COMMENT)) { 563 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID); 564 preferences.put(getId(), (String) value); 565 try { 566 preferences.flush(); 567 } catch (BackingStoreException e) { 568 // skip 569 } 570 } 571} 572 573/** 574 * Sets the image descriptor 575 */ 576void setImageDescriptor(ImageDescriptor desc) { 577// this.imageDescriptor = desc; 578} 579 580public String toString() { 581 if (this.results == null) { 582 return getName(); 583 } 584 return this.results.toString(); 585} 586 587/* 588 * Write the element status in the given stream 589 */ 590StringBuffer writableStatus(StringBuffer buffer, int kind, StringBuffer excluded) { 591 int length = this.children.length; 592 for (int i=0; i<length; i++) { 593 this.children[i].writableStatus(buffer, kind, excluded); 594 } 595 return buffer; 596} 597 598 599} 600