Invoker.java revision db04b1679d3367e54e61d7e6f8c225354e90081c
1package org.testng.internal; 2 3import static org.testng.internal.invokers.InvokedMethodListenerMethod.AFTER_INVOCATION; 4import static org.testng.internal.invokers.InvokedMethodListenerMethod.BEFORE_INVOCATION; 5 6import org.testng.IClass; 7import org.testng.IConfigurable; 8import org.testng.IConfigurationListener; 9import org.testng.IConfigurationListener2; 10import org.testng.IHookable; 11import org.testng.IInvokedMethod; 12import org.testng.IInvokedMethodListener; 13import org.testng.IRetryAnalyzer; 14import org.testng.ITestClass; 15import org.testng.ITestContext; 16import org.testng.ITestListener; 17import org.testng.ITestNGMethod; 18import org.testng.ITestResult; 19import org.testng.Reporter; 20import org.testng.SkipException; 21import org.testng.SuiteRunState; 22import org.testng.TestException; 23import org.testng.TestNGException; 24import org.testng.annotations.IConfigurationAnnotation; 25import org.testng.annotations.NoInjection; 26import org.testng.collections.Lists; 27import org.testng.collections.Maps; 28import org.testng.internal.InvokeMethodRunnable.TestNGRuntimeException; 29import org.testng.internal.ParameterHolder.ParameterOrigin; 30import org.testng.internal.annotations.AnnotationHelper; 31import org.testng.internal.annotations.IAnnotationFinder; 32import org.testng.internal.annotations.Sets; 33import org.testng.internal.invokers.InvokedMethodListenerInvoker; 34import org.testng.internal.invokers.InvokedMethodListenerMethod; 35import org.testng.internal.thread.ThreadExecutionException; 36import org.testng.internal.thread.ThreadUtil; 37import org.testng.internal.thread.graph.IWorker; 38import org.testng.xml.XmlClass; 39import org.testng.xml.XmlSuite; 40import org.testng.xml.XmlTest; 41 42import java.lang.annotation.Annotation; 43import java.lang.reflect.InvocationTargetException; 44import java.lang.reflect.Method; 45import java.util.Arrays; 46import java.util.Iterator; 47import java.util.List; 48import java.util.Map; 49import java.util.Set; 50import java.util.regex.Pattern; 51 52/** 53 * This class is responsible for invoking methods: 54 * - test methods 55 * - configuration methods 56 * - possibly in a separate thread 57 * and then for notifying the result listeners. 58 * 59 * @author <a href="mailto:cedric@beust.com">Cedric Beust</a> 60 * @author <a href='mailto:the_mindstorm@evolva.ro'>Alexandru Popescu</a> 61 */ 62public class Invoker implements IInvoker { 63 private final ITestContext m_testContext; 64 private final ITestResultNotifier m_notifier; 65 private final IAnnotationFinder m_annotationFinder; 66 private final SuiteRunState m_suiteState; 67 private final boolean m_skipFailedInvocationCounts; 68 private final List<IInvokedMethodListener> m_invokedMethodListeners; 69 private final boolean m_continueOnFailedConfiguration; 70 71 /** Group failures must be synced as the Invoker is accessed concurrently */ 72 private Map<String, Boolean> m_beforegroupsFailures = Maps.newHashtable(); 73 74 /** Class failures must be synced as the Invoker is accessed concurrently */ 75 private Map<Class<?>, Set<Object>> m_classInvocationResults = Maps.newHashtable(); 76 77 /** Test methods whose configuration methods have failed. */ 78 private Map<ITestNGMethod, Set<Object>> m_methodInvocationResults = Maps.newHashtable(); 79 private IConfiguration m_configuration; 80 81 /** Predicate to filter methods */ 82 private static Predicate CAN_RUN_FROM_CLASS = new CanRunFromClassPredicate(); 83 /** Predicate to filter methods */ 84 private static final Predicate SAME_CLASS = new SameClassNamePredicate(); 85 86 private void setClassInvocationFailure(Class<?> clazz, Object instance) { 87 Set<Object> instances = m_classInvocationResults.get( clazz ); 88 if (instances == null) { 89 instances = Sets.newHashSet(); 90 m_classInvocationResults.put(clazz, instances); 91 } 92 instances.add(instance); 93 } 94 95 private void setMethodInvocationFailure(ITestNGMethod method, Object instance) { 96 Set<Object> instances = m_methodInvocationResults.get(method); 97 if (instances == null) { 98 instances = Sets.newHashSet(); 99 m_methodInvocationResults.put(method, instances); 100 } 101 instances.add(instance); 102 } 103 104 public Invoker(IConfiguration configuration, 105 ITestContext testContext, 106 ITestResultNotifier notifier, 107 SuiteRunState state, 108 boolean skipFailedInvocationCounts, 109 List<IInvokedMethodListener> invokedMethodListeners) { 110 m_configuration = configuration; 111 m_testContext= testContext; 112 m_suiteState= state; 113 m_notifier= notifier; 114 m_annotationFinder= configuration.getAnnotationFinder(); 115 m_skipFailedInvocationCounts = skipFailedInvocationCounts; 116 m_invokedMethodListeners = invokedMethodListeners; 117 m_continueOnFailedConfiguration = XmlSuite.CONTINUE.equals(testContext.getSuite().getXmlSuite().getConfigFailurePolicy()); 118 } 119 120 /** 121 * Invoke configuration methods if they belong to the same TestClass passed 122 * in parameter.. <p/>TODO: Calculate ahead of time which methods should be 123 * invoked for each class. Might speed things up for users who invoke the 124 * same test class with different parameters in the same suite run. 125 * 126 * If instance is non-null, the configuration will be run on it. If it is null, 127 * the configuration methods will be run on all the instances retrieved 128 * from the ITestClass. 129 */ 130 @Override 131 public void invokeConfigurations(IClass testClass, 132 ITestNGMethod[] allMethods, 133 XmlSuite suite, 134 Map<String, String> params, 135 Object[] parameterValues, 136 Object instance) 137 { 138 invokeConfigurations(testClass, null, allMethods, suite, params, parameterValues, instance, 139 null); 140 } 141 142 private void invokeConfigurations(IClass testClass, 143 ITestNGMethod currentTestMethod, 144 ITestNGMethod[] allMethods, 145 XmlSuite suite, 146 Map<String, String> params, 147 Object[] parameterValues, 148 Object instance, 149 ITestResult testMethodResult) 150 { 151 if(null == allMethods) { 152 log(5, "No configuration methods found"); 153 154 return; 155 } 156 157 ITestNGMethod[] methods= filterMethods(testClass, allMethods, SAME_CLASS); 158 159 for(ITestNGMethod tm : methods) { 160 if(null == testClass) { 161 testClass= tm.getTestClass(); 162 } 163 164 ITestResult testResult= new TestResult(testClass, 165 instance, 166 tm, 167 null, 168 System.currentTimeMillis(), 169 System.currentTimeMillis()); 170 171 IConfigurationAnnotation configurationAnnotation= null; 172 try { 173 Object[] instances= tm.getInstances(); 174 if (instances == null || instances.length == 0) { 175 instances = new Object[] { instance }; 176 } 177 Class<?> objectClass= instances[0].getClass(); 178 Method method= tm.getMethod(); 179 180 // Only run the configuration if 181 // - the test is enabled and 182 // - the Configuration method belongs to the same class or a parent 183 if(MethodHelper.isEnabled(objectClass, m_annotationFinder)) { 184 configurationAnnotation = AnnotationHelper.findConfiguration(m_annotationFinder, method); 185 186 if (MethodHelper.isEnabled(configurationAnnotation)) { 187 boolean isClassConfiguration = isClassConfiguration(configurationAnnotation); 188 boolean isSuiteConfiguration = isSuiteConfiguration(configurationAnnotation); 189 boolean alwaysRun= isAlwaysRun(configurationAnnotation); 190 191 if (!confInvocationPassed(tm, currentTestMethod, testClass, instance) && !alwaysRun) { 192 handleConfigurationSkip(tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 193 continue; 194 } 195 196 log(3, "Invoking " + Utils.detailedMethodName(tm, true)); 197 198 Object[] parameters = Parameters.createConfigurationParameters(tm.getMethod(), 199 params, 200 parameterValues, 201 currentTestMethod, 202 m_annotationFinder, 203 suite, 204 m_testContext, 205 testMethodResult); 206 testResult.setParameters(parameters); 207 208 Object[] newInstances= (null != instance) ? new Object[] { instance } : instances; 209 210 runConfigurationListeners(testResult, true /* before */); 211 212 invokeConfigurationMethod(newInstances, tm, 213 parameters, isClassConfiguration, isSuiteConfiguration, testResult); 214 215 // TODO: probably we should trigger the event for each instance??? 216 testResult.setEndMillis(System.currentTimeMillis()); 217 runConfigurationListeners(testResult, false /* after */); 218 } 219 else { 220 log(3, 221 "Skipping " 222 + Utils.detailedMethodName(tm, true) 223 + " because it is not enabled"); 224 } 225 } // if is enabled 226 else { 227 log(3, 228 "Skipping " 229 + Utils.detailedMethodName(tm, true) 230 + " because " 231 + objectClass.getName() 232 + " is not enabled"); 233 } 234 } 235 catch(InvocationTargetException ex) { 236 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 237 } 238 catch(TestNGException ex) { 239 // Don't wrap TestNGExceptions, it could be a missing parameter on a 240 // @Configuration method 241 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 242 } 243 catch(Throwable ex) { // covers the non-wrapper exceptions 244 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 245 } 246 } // for methods 247 } 248 249 /** 250 * Marks the current <code>TestResult</code> as skipped and invokes the listeners. 251 */ 252 private void handleConfigurationSkip(ITestNGMethod tm, 253 ITestResult testResult, 254 IConfigurationAnnotation annotation, 255 ITestNGMethod currentTestMethod, 256 Object instance, 257 XmlSuite suite) { 258 recordConfigurationInvocationFailed(tm, testResult.getTestClass(), annotation, currentTestMethod, instance, suite); 259 testResult.setStatus(ITestResult.SKIP); 260 runConfigurationListeners(testResult, false /* after */); 261 } 262 263 /** 264 * Is the current <code>IConfiguration</code> a class-level method. 265 */ 266 private boolean isClassConfiguration(IConfigurationAnnotation configurationAnnotation) { 267 if (null == configurationAnnotation) { 268 return false; 269 } 270 271 boolean before = configurationAnnotation.getBeforeTestClass(); 272 boolean after = configurationAnnotation.getAfterTestClass(); 273 274 return before || after; 275 } 276 277 /** 278 * Is the current <code>IConfiguration</code> a suite level method. 279 */ 280 private boolean isSuiteConfiguration(IConfigurationAnnotation configurationAnnotation) { 281 if (null == configurationAnnotation) { 282 return false; 283 } 284 285 boolean before = configurationAnnotation.getBeforeSuite(); 286 boolean after = configurationAnnotation.getAfterSuite(); 287 288 return before || after; 289 } 290 291 /** 292 * Is the <code>IConfiguration</code> marked as alwaysRun. 293 */ 294 private boolean isAlwaysRun(IConfigurationAnnotation configurationAnnotation) { 295 if(null == configurationAnnotation) { 296 return false; 297 } 298 299 boolean alwaysRun= false; 300 if ((configurationAnnotation.getAfterSuite() 301 || configurationAnnotation.getAfterTest() 302 || configurationAnnotation.getAfterTestClass() 303 || configurationAnnotation.getAfterTestMethod()) 304 && configurationAnnotation.getAlwaysRun()) 305 { 306 alwaysRun= true; 307 } 308 309 return alwaysRun; 310 } 311 312 private void handleConfigurationFailure(Throwable ite, 313 ITestNGMethod tm, 314 ITestResult testResult, 315 IConfigurationAnnotation annotation, 316 ITestNGMethod currentTestMethod, 317 Object instance, 318 XmlSuite suite) 319 { 320 Throwable cause= ite.getCause() != null ? ite.getCause() : ite; 321 322 if(SkipException.class.isAssignableFrom(cause.getClass())) { 323 SkipException skipEx= (SkipException) cause; 324 if(skipEx.isSkip()) { 325 testResult.setThrowable(skipEx); 326 handleConfigurationSkip(tm, testResult, annotation, currentTestMethod, instance, suite); 327 return; 328 } 329 } 330 Utils.log("", 3, "Failed to invoke configuration method " 331 + tm.getRealClass().getName() + "." + tm.getMethodName() + ":" + cause.getMessage()); 332 handleException(cause, tm, testResult, 1); 333 runConfigurationListeners(testResult, false /* after */); 334 335 // 336 // If in TestNG mode, need to take a look at the annotation to figure out 337 // what kind of @Configuration method we're dealing with 338 // 339 if (null != annotation) { 340 recordConfigurationInvocationFailed(tm, testResult.getTestClass(), annotation, currentTestMethod, instance, suite); 341 } 342 } 343 344 /** 345 * @return All the classes that belong to the same <test> tag as @param cls 346 */ 347 private XmlClass[] findClassesInSameTest(Class<?> cls, XmlSuite suite) { 348 Map<String, XmlClass> vResult= Maps.newHashMap(); 349 String className= cls.getName(); 350 for(XmlTest test : suite.getTests()) { 351 for(XmlClass testClass : test.getXmlClasses()) { 352 if(testClass.getName().equals(className)) { 353 354 // Found it, add all the classes in this test in the result 355 for(XmlClass thisClass : test.getXmlClasses()) { 356 vResult.put(thisClass.getName(), thisClass); 357 } 358 // Note: we need to iterate through the entire suite since the same 359 // class might appear in several <test> tags 360 } 361 } 362 } 363 364 XmlClass[] result= vResult.values().toArray(new XmlClass[vResult.size()]); 365 366 return result; 367 } 368 369 /** 370 * Record internally the failure of a Configuration, so that we can determine 371 * later if @Test should be skipped. 372 */ 373 private void recordConfigurationInvocationFailed(ITestNGMethod tm, 374 IClass testClass, 375 IConfigurationAnnotation annotation, 376 ITestNGMethod currentTestMethod, 377 Object instance, 378 XmlSuite suite) { 379 // If beforeTestClass or afterTestClass failed, mark either the config method's 380 // entire class as failed, or the class under tests as failed, depending on 381 // the configuration failure policy 382 if (annotation.getBeforeTestClass() || annotation.getAfterTestClass()) { 383 // tm is the configuration method, and currentTestMethod is null for BeforeClass 384 // methods, so we need testClass 385 if (m_continueOnFailedConfiguration) { 386 setClassInvocationFailure(testClass.getRealClass(), instance); 387 } else { 388 setClassInvocationFailure(tm.getRealClass(), instance); 389 } 390 } 391 392 // If before/afterTestMethod failed, mark either the config method's entire 393 // class as failed, or just the current test method as failed, depending on 394 // the configuration failure policy 395 else if (annotation.getBeforeTestMethod() || annotation.getAfterTestMethod()) { 396 if (m_continueOnFailedConfiguration) { 397 setMethodInvocationFailure(currentTestMethod, instance); 398 } else { 399 setClassInvocationFailure(tm.getRealClass(), instance); 400 } 401 } 402 403 // If beforeSuite or afterSuite failed, mark *all* the classes as failed 404 // for configurations. At this point, the entire Suite is screwed 405 else if (annotation.getBeforeSuite() || annotation.getAfterSuite()) { 406 m_suiteState.failed(); 407 } 408 409 // beforeTest or afterTest: mark all the classes in the same 410 // <test> stanza as failed for configuration 411 else if (annotation.getBeforeTest() || annotation.getAfterTest()) { 412 setClassInvocationFailure(tm.getRealClass(), instance); 413 XmlClass[] classes= findClassesInSameTest(tm.getRealClass(), suite); 414 for(XmlClass xmlClass : classes) { 415 setClassInvocationFailure(xmlClass.getSupportClass(), instance); 416 } 417 } 418 String[] beforeGroups= annotation.getBeforeGroups(); 419 if(null != beforeGroups && beforeGroups.length > 0) { 420 for(String group: beforeGroups) { 421 m_beforegroupsFailures.put(group, Boolean.FALSE); 422 } 423 } 424 } 425 426 /** 427 * @return true if this class has successfully run all its @Configuration 428 * method or false if at least one of these methods failed. 429 */ 430 private boolean confInvocationPassed(ITestNGMethod method, ITestNGMethod currentTestMethod, IClass testClass, Object instance) { 431 boolean result= true; 432 433 // If continuing on config failure, check invocation results for the class 434 // under test, otherwise use the method's declaring class 435 Class<?> cls = m_continueOnFailedConfiguration ? 436 testClass.getRealClass() : method.getMethod().getDeclaringClass(); 437 438 if(m_suiteState.isFailed()) { 439 result= false; 440 } 441 else { 442 if (m_classInvocationResults.containsKey(cls)) { 443 if (! m_continueOnFailedConfiguration) { 444 result = !m_classInvocationResults.containsKey(cls); 445 } else { 446 result = !m_classInvocationResults.get(cls).contains(instance); 447 } 448 } 449 // if method is BeforeClass, currentTestMethod will be null 450 else if (m_continueOnFailedConfiguration && 451 currentTestMethod != null && 452 m_methodInvocationResults.containsKey(currentTestMethod)) { 453 result = !m_methodInvocationResults.get(currentTestMethod).contains(instance); 454 } 455 else if (! m_continueOnFailedConfiguration) { 456 for(Class<?> clazz: m_classInvocationResults.keySet()) { 457// if (clazz == cls) { 458 if(clazz.isAssignableFrom(cls)) { 459 result= false; 460 break; 461 } 462 } 463 } 464 } 465 466 // check if there are failed @BeforeGroups 467 String[] groups= method.getGroups(); 468 if(null != groups && groups.length > 0) { 469 for(String group: groups) { 470 if(m_beforegroupsFailures.containsKey(group)) { 471 result= false; 472 break; 473 } 474 } 475 } 476 return result; 477 } 478 479 /** 480 * Effectively invokes a configuration method on all passed in instances. 481 * TODO: Should change this method to be more like invokeMethod() so that we can 482 * handle calls to {@code IInvokedMethodListener} better. 483 * 484 * @param instances the instances to invoke the configuration method on 485 * @param tm the configuration method 486 * @param params the parameters needed for method invocation 487 * @param isClass flag if the configuration method is a class level method // FIXME: this looks like a missusage 488 * @param testResult 489 * @throws InvocationTargetException 490 * @throws IllegalAccessException 491 */ 492 private void invokeConfigurationMethod(Object[] instances, 493 ITestNGMethod tm, 494 Object[] params, 495 boolean isClass, 496 boolean isSuite, 497 ITestResult testResult) 498 throws InvocationTargetException, IllegalAccessException 499 { 500 // Mark this method with the current thread id 501 tm.setId(ThreadUtil.currentThreadInfo()); 502 503 // Only a @BeforeMethod/@AfterMethod needs to be run before each instance, all the other 504 // configuration methods only need to be run once 505 List<Object> actualInstances = Lists.newArrayList(); 506 if (tm.isBeforeMethodConfiguration() || tm.isAfterMethodConfiguration()) { 507 actualInstances.addAll(Arrays.asList(instances)); 508 } else { 509 actualInstances.add(instances[0]); 510 } 511 for(Object targetInstance : actualInstances) { 512 InvokedMethod invokedMethod= new InvokedMethod(targetInstance, 513 tm, 514 params, 515 false, /* isTest */ 516 isClass, /* ??? */ 517 System.currentTimeMillis()); 518 519 runInvokedMethodListeners(BEFORE_INVOCATION, invokedMethod, testResult); 520 m_notifier.addInvokedMethod(invokedMethod); 521 try { 522 Reporter.setCurrentTestResult(testResult); 523 Method method = tm.getMethod(); 524 525 // 526 // If this method is a IHookable, invoke its run() method 527 // 528 IConfigurable configurableInstance = 529 IConfigurable.class.isAssignableFrom(tm.getMethod().getDeclaringClass()) ? 530 (IConfigurable) targetInstance : m_configuration.getConfigurable(); 531 if (configurableInstance != null) { 532 // 533 // If this method is a IConfigurable, invoke its run() method 534 // 535 MethodInvocationHelper.invokeConfigurable(targetInstance, params, configurableInstance, method, 536 testResult); 537 } 538 else { 539 // 540 // Not a IConfigurable, invoke directly 541 // 542 if (MethodHelper.calculateTimeOut(tm) <= 0) { 543 MethodInvocationHelper.invokeMethod(method, targetInstance, params); 544 } 545 else { 546 MethodInvocationHelper.invokeWithTimeout(tm, targetInstance, params, testResult); 547 if (!testResult.isSuccess()) { 548 // A time out happened 549 throwConfigurationFailure(testResult, testResult.getThrowable()); 550 throw testResult.getThrowable(); 551 } 552 } 553 } 554 // Only run the method once if it's @BeforeSuite or @AfterSuite 555 if (isSuite) { 556 break; 557 } 558 } 559 catch (InvocationTargetException ex) { 560 throwConfigurationFailure(testResult, ex); 561 throw ex; 562 } 563 catch (IllegalAccessException ex) { 564 throwConfigurationFailure(testResult, ex); 565 throw ex; 566 } 567 catch (NoSuchMethodException ex) { 568 throwConfigurationFailure(testResult, ex); 569 throw new TestNGException(ex); 570 } 571 catch (Throwable ex) { 572 throwConfigurationFailure(testResult, ex); 573 throw new TestNGException(ex); 574 } 575 finally { 576 Reporter.setCurrentTestResult(testResult); 577 runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult); 578 } 579 } 580 } 581 582 private void throwConfigurationFailure(ITestResult testResult, Throwable ex) 583 { 584 testResult.setStatus(ITestResult.FAILURE);; 585 testResult.setThrowable(ex.getCause() == null ? ex : ex.getCause()); 586 } 587 588 private void runInvokedMethodListeners(InvokedMethodListenerMethod listenerMethod, IInvokedMethod invokedMethod, 589 ITestResult testResult) 590 { 591 if ( noListenersPresent() ) { 592 return; 593 } 594 595 InvokedMethodListenerInvoker invoker = new InvokedMethodListenerInvoker(listenerMethod, testResult, m_testContext); 596 for (IInvokedMethodListener currentListener : m_invokedMethodListeners) { 597 invoker.invokeListener(currentListener, invokedMethod); 598 } 599 } 600 601 private boolean noListenersPresent() { 602 return (m_invokedMethodListeners == null) || (m_invokedMethodListeners.size() == 0); 603 } 604 605 // pass both paramValues and paramIndex to be thread safe in case parallel=true + dataprovider. 606 private ITestResult invokeMethod(Object[] instances, 607 int instanceIndex, 608 final ITestNGMethod tm, 609 Object[] parameterValues, 610 int parametersIndex, 611 XmlSuite suite, 612 Map<String, String> params, 613 ITestClass testClass, 614 ITestNGMethod[] beforeMethods, 615 ITestNGMethod[] afterMethods, 616 ConfigurationGroupMethods groupMethods) { 617 TestResult testResult = new TestResult(); 618 619 // 620 // Invoke beforeGroups configurations 621 // 622 Object instance = instances[instanceIndex]; 623 invokeBeforeGroupsConfigurations(testClass, tm, groupMethods, suite, params, 624 instance); 625 626 // 627 // Invoke beforeMethods only if 628 // - firstTimeOnly is not set 629 // - firstTimeOnly is set, and we are reaching at the first invocationCount 630 // 631 invokeConfigurations(testClass, tm, 632 filterConfigurationMethods(tm, beforeMethods, true /* beforeMethods */), 633 suite, params, parameterValues, 634 instance, testResult); 635 636 // 637 // Create the ExtraOutput for this method 638 // 639 InvokedMethod invokedMethod = null; 640 try { 641 testResult.init(testClass, instance, 642 tm, 643 null, 644 System.currentTimeMillis(), 645 0); 646 testResult.setParameters(parameterValues); 647 testResult.setHost(m_testContext.getHost()); 648 testResult.setStatus(ITestResult.STARTED); 649 650 invokedMethod= new InvokedMethod(instance, 651 tm, 652 parameterValues, 653 true /* isTest */, 654 false /* isConfiguration */, 655 System.currentTimeMillis()); 656 657 // Fix from ansgarkonermann 658 // invokedMethod is used in the finally, which can be invoked if 659 // any of the test listeners throws an exception, therefore, 660 // invokedMethod must have a value before we get here 661 runTestListeners(testResult); 662 663 runInvokedMethodListeners(BEFORE_INVOCATION, invokedMethod, testResult); 664 665 m_notifier.addInvokedMethod(invokedMethod); 666 667 Method thisMethod= tm.getMethod(); 668 669 if(confInvocationPassed(tm, tm, testClass, instance)) { 670 log(3, "Invoking " + thisMethod.getDeclaringClass().getName() + "." + 671 thisMethod.getName()); 672 673 // If no timeOut, just invoke the method 674 if (MethodHelper.calculateTimeOut(tm) <= 0) { 675 try { 676 Reporter.setCurrentTestResult(testResult); 677 // 678 // If this method is a IHookable, invoke its run() method 679 // 680 IHookable hookableInstance = 681 IHookable.class.isAssignableFrom(thisMethod.getDeclaringClass()) ? 682 (IHookable) instance : m_configuration.getHookable(); 683 if (hookableInstance != null) { 684 MethodInvocationHelper.invokeHookable(instance, 685 parameterValues, hookableInstance, thisMethod, testResult); 686 } 687 // 688 // Not a IHookable, invoke directly 689 // 690 else { 691 MethodInvocationHelper.invokeMethod(thisMethod, instance, 692 parameterValues); 693 } 694 testResult.setStatus(ITestResult.SUCCESS); 695 } 696 finally { 697 Reporter.setCurrentTestResult(null); 698 } 699 } 700 else { 701 // 702 // Method with a timeout 703 // 704 try { 705 Reporter.setCurrentTestResult(testResult); 706 MethodInvocationHelper.invokeWithTimeout(tm, instance, parameterValues, testResult); 707 } 708 finally { 709 Reporter.setCurrentTestResult(null); 710 } 711 } 712 } 713 else { 714 testResult.setStatus(ITestResult.SKIP); 715 } 716 } 717 catch(InvocationTargetException ite) { 718 testResult.setThrowable(ite.getCause()); 719 testResult.setStatus(ITestResult.FAILURE); 720 } 721 catch(ThreadExecutionException tee) { // wrapper for TestNGRuntimeException 722 Throwable cause= tee.getCause(); 723 if(TestNGRuntimeException.class.equals(cause.getClass())) { 724 testResult.setThrowable(cause.getCause()); 725 } 726 else { 727 testResult.setThrowable(cause); 728 } 729 testResult.setStatus(ITestResult.FAILURE); 730 } 731 catch(Throwable thr) { // covers the non-wrapper exceptions 732 testResult.setThrowable(thr); 733 testResult.setStatus(ITestResult.FAILURE); 734 } 735 finally { 736 ExpectedExceptionsHolder expectedExceptionClasses 737 = MethodHelper.findExpectedExceptions(m_annotationFinder, tm.getMethod()); 738 List<ITestResult> results = Lists.newArrayList(); 739 results.add(testResult); 740 handleInvocationResults(tm, results, null, 0, expectedExceptionClasses, false, 741 false /* collect results */); 742 743 // If this method has a data provider and just failed, memorize the number 744 // at which it failed. 745 // Note: we're not exactly testing that this method has a data provider, just 746 // that it has parameters, so might have to revisit this if bugs get reported 747 // for the case where this method has parameters that don't come from a data 748 // provider 749 if (testResult.getThrowable() != null && parameterValues.length > 0) { 750 tm.addFailedInvocationNumber(parametersIndex); 751 } 752 753 // 754 // Increment the invocation count for this method 755 // 756 tm.incrementCurrentInvocationCount(); 757 758 if (testResult != null) { 759 testResult.setEndMillis(System.currentTimeMillis()); 760 } 761 762 // Run invokedMethodListeners after updating TestResult 763 runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult); 764 runTestListeners(testResult); 765 collectResults(tm, results, testResult); 766 767 // 768 // Invoke afterMethods only if 769 // - lastTimeOnly is not set 770 // - lastTimeOnly is set, and we are reaching the last invocationCount 771 // 772 invokeConfigurations(testClass, tm, 773 filterConfigurationMethods(tm, afterMethods, false /* beforeMethods */), 774 suite, params, parameterValues, 775 instance, 776 testResult); 777 778 // 779 // Invoke afterGroups configurations 780 // 781 invokeAfterGroupsConfigurations(testClass, tm, groupMethods, suite, 782 params, instance); 783 } 784 785 return testResult; 786 } 787 788 private void collectResults(ITestNGMethod testMethod, List<ITestResult> results, TestResult testResult) { 789 for (int i = 0; i < results.size(); i++) { 790 // Collect the results 791 int status = results.get(i).getStatus(); 792 if(ITestResult.SUCCESS == status) { 793 m_notifier.addPassedTest(testMethod, testResult); 794 } 795 else if(ITestResult.SKIP == status) { 796 m_notifier.addSkippedTest(testMethod, testResult); 797 } 798 else if(ITestResult.FAILURE == status) { 799 m_notifier.addFailedTest(testMethod, testResult); 800 } 801 else if(ITestResult.SUCCESS_PERCENTAGE_FAILURE == status) { 802 m_notifier.addFailedButWithinSuccessPercentageTest(testMethod, testResult); 803 } 804 else { 805 assert false : "UNKNOWN STATUS:" + status; 806 } 807 } 808 } 809 810 /** 811 * The array of methods contains @BeforeMethods if isBefore if true, @AfterMethods 812 * otherwise. This function removes all the methods that should not be run at this 813 * point because they are either firstTimeOnly or lastTimeOnly and we haven't reached 814 * the current invocationCount yet 815 */ 816 private ITestNGMethod[] filterConfigurationMethods(ITestNGMethod tm, 817 ITestNGMethod[] methods, boolean isBefore) 818 { 819 List<ITestNGMethod> result = Lists.newArrayList(); 820 for (ITestNGMethod m : methods) { 821 ConfigurationMethod cm = (ConfigurationMethod) m; 822 if (isBefore) { 823 if (! cm.isFirstTimeOnly() || 824 (cm.isFirstTimeOnly() && tm.getCurrentInvocationCount() == 0)) 825 { 826 result.add(m); 827 } 828 } 829 else { 830 int current = tm.getCurrentInvocationCount(); 831 boolean isLast = false; 832 // If we have parameters, set the boolean if we are about to run 833 // the last invocation 834 if (tm.getParameterInvocationCount() > 0) { 835 isLast = current == tm.getParameterInvocationCount(); 836 } 837 // If we have invocationCount > 1, set the boolean if we are about to 838 // run the last invocation 839 else if (tm.getInvocationCount() > 1) { 840 isLast = current == tm.getInvocationCount(); 841 } 842 if (! cm.isLastTimeOnly() || (cm.isLastTimeOnly() && isLast)) { 843 result.add(m); 844 } 845 } 846 } 847 848 return result.toArray(new ITestNGMethod[result.size()]); 849 } 850 851 /** 852 * {@link #invokeTestMethods()} eventually converge here to invoke a single @Test method. 853 * <p/> 854 * This method is responsible for actually invoking the method. It decides if the invocation 855 * must be done: 856 * <ul> 857 * <li>through an <code>IHookable</code></li> 858 * <li>directly (through reflection)</li> 859 * <li>in a separate thread (in case it needs to timeout) 860 * </ul> 861 * 862 * <p/> 863 * This method is also reponsible for invoking @BeforeGroup, @BeforeMethod, @AfterMethod, @AfterGroup 864 * if it is the case for the passed in @Test method. 865 */ 866 protected List<ITestResult> invokeTestMethod(Object[] instances, 867 final ITestNGMethod tm, 868 Object[] parameterValues, 869 int parametersIndex, 870 XmlSuite suite, 871 Map<String, String> params, 872 ITestClass testClass, 873 ITestNGMethod[] beforeMethods, 874 ITestNGMethod[] afterMethods, 875 ConfigurationGroupMethods groupMethods) 876 { 877 List<ITestResult> results = Lists.newArrayList(); 878 879 // Mark this method with the current thread id 880 tm.setId(ThreadUtil.currentThreadInfo()); 881 882 for(int i= 0; i < instances.length; i++) { 883 results.add(invokeMethod(instances, i, tm, parameterValues, parametersIndex, suite, params, 884 testClass, beforeMethods, afterMethods, groupMethods)); 885 } 886 887 return results; 888 } 889 890 /** 891 * Filter all the beforeGroups methods and invoke only those that apply 892 * to the current test method 893 */ 894 private void invokeBeforeGroupsConfigurations(ITestClass testClass, 895 ITestNGMethod currentTestMethod, 896 ConfigurationGroupMethods groupMethods, 897 XmlSuite suite, 898 Map<String, String> params, 899 Object instance) 900 { 901 synchronized(groupMethods) { 902 List<ITestNGMethod> filteredMethods = Lists.newArrayList(); 903 String[] groups = currentTestMethod.getGroups(); 904 Map<String, List<ITestNGMethod>> beforeGroupMap = groupMethods.getBeforeGroupsMap(); 905 906 for (String group : groups) { 907 List<ITestNGMethod> methods = beforeGroupMap.get(group); 908 if (methods != null) { 909 filteredMethods.addAll(methods); 910 } 911 } 912 913 ITestNGMethod[] beforeMethodsArray = filteredMethods.toArray(new ITestNGMethod[filteredMethods.size()]); 914 // 915 // Invoke the right groups methods 916 // 917 if(beforeMethodsArray.length > 0) { 918 // don't pass the IClass or the instance as the method may be external 919 // the invocation must be similar to @BeforeTest/@BeforeSuite 920 invokeConfigurations(null, beforeMethodsArray, suite, params, 921 null, /* no parameter values */ 922 null); 923 } 924 925 // 926 // Remove them so they don't get run again 927 // 928 groupMethods.removeBeforeGroups(groups); 929 } 930 } 931 932 private void invokeAfterGroupsConfigurations(ITestClass testClass, 933 ITestNGMethod currentTestMethod, 934 ConfigurationGroupMethods groupMethods, 935 XmlSuite suite, 936 Map<String, String> params, 937 Object instance) 938 { 939 // Skip this if the current method doesn't belong to any group 940 // (only a method that belongs to a group can trigger the invocation 941 // of afterGroups methods) 942 if (currentTestMethod.getGroups().length == 0) { 943 return; 944 } 945 946 // See if the currentMethod is the last method in any of the groups 947 // it belongs to 948 Map<String, String> filteredGroups = Maps.newHashMap(); 949 String[] groups = currentTestMethod.getGroups(); 950 synchronized(groupMethods) { 951 for (String group : groups) { 952 if (groupMethods.isLastMethodForGroup(group, currentTestMethod)) { 953 filteredGroups.put(group, group); 954 } 955 } 956 957 if(filteredGroups.isEmpty()) { 958 return; 959 } 960 961 // The list of afterMethods to run 962 Map<ITestNGMethod, ITestNGMethod> afterMethods = Maps.newHashMap(); 963 964 // Now filteredGroups contains all the groups for which we need to run the afterGroups 965 // method. Find all the methods that correspond to these groups and invoke them. 966 Map<String, List<ITestNGMethod>> map = groupMethods.getAfterGroupsMap(); 967 for (String g : filteredGroups.values()) { 968 List<ITestNGMethod> methods = map.get(g); 969 // Note: should put them in a map if we want to make sure the same afterGroups 970 // doesn't get run twice 971 if (methods != null) { 972 for (ITestNGMethod m : methods) { 973 afterMethods.put(m, m); 974 } 975 } 976 } 977 978 // Got our afterMethods, invoke them 979 ITestNGMethod[] afterMethodsArray = afterMethods.keySet().toArray(new ITestNGMethod[afterMethods.size()]); 980 // don't pass the IClass or the instance as the method may be external 981 // the invocation must be similar to @BeforeTest/@BeforeSuite 982 invokeConfigurations(null, afterMethodsArray, suite, params, 983 null, /* no parameter values */ 984 null); 985 986 // Remove the groups so they don't get run again 987 groupMethods.removeAfterGroups(filteredGroups.keySet()); 988 } 989 } 990 991 private Object[] getParametersFromIndex(Iterator<Object[]> parametersValues, int index) { 992 while (parametersValues.hasNext()) { 993 Object[] parameters = parametersValues.next(); 994 995 if (index == 0) { 996 return parameters; 997 } 998 index--; 999 } 1000 return null; 1001 } 1002 1003 int retryFailed(Object[] instances, 1004 int instanceIndex, 1005 final ITestNGMethod tm, 1006 XmlSuite suite, 1007 ITestClass testClass, 1008 ITestNGMethod[] beforeMethods, 1009 ITestNGMethod[] afterMethods, 1010 ConfigurationGroupMethods groupMethods, 1011 List<ITestResult> result, 1012 int failureCount, 1013 ExpectedExceptionsHolder expectedExceptionHolder, 1014 ITestContext testContext, 1015 Map<String, String> parameters, 1016 int parametersIndex) { 1017 List<Object> failedInstances; 1018 1019 do { 1020 failedInstances = Lists.newArrayList(); 1021 Map<String, String> allParameters = Maps.newHashMap(); 1022 /** 1023 * TODO: This recreates all the parameters every time when we only need 1024 * one specific set. Should optimize it by only recreating the set needed. 1025 */ 1026 ParameterBag bag = createParameters(tm, parameters, 1027 allParameters, null, suite, testContext, null /* fedInstance */, null /* testResult */); 1028 Object[] parameterValues = 1029 getParametersFromIndex(bag.parameterHolder.parameters, parametersIndex); 1030 1031 result.add(invokeMethod(instances, instanceIndex, tm, parameterValues,parametersIndex, suite, 1032 allParameters, testClass, beforeMethods, afterMethods, groupMethods)); 1033 failureCount = handleInvocationResults(tm, result, failedInstances, 1034 failureCount, expectedExceptionHolder, true, true /* collect results */); 1035 } 1036 while (!failedInstances.isEmpty()); 1037 return failureCount; 1038 } 1039 1040 private ParameterBag createParameters(ITestNGMethod testMethod, 1041 Map<String, String> parameters, 1042 Map<String, String> allParameterNames, 1043 Object[] parameterValues, 1044 XmlSuite suite, 1045 ITestContext testContext, 1046 Object fedInstance, 1047 ITestResult testResult) 1048 { 1049 Object instance; 1050 if (fedInstance != null) { 1051 instance = fedInstance; 1052 } 1053 else { 1054 instance = testMethod.getInstance(); 1055 } 1056 1057 ParameterBag bag= handleParameters(testMethod, 1058 instance, allParameterNames, parameters, parameterValues, suite, testContext, fedInstance, 1059 testResult); 1060 1061 return bag; 1062 } 1063 1064 /** 1065 * Invoke all the test methods. Note the plural: the method passed in 1066 * parameter might be invoked several times if the test class it belongs 1067 * to has more than one instance (i.e., if an @Factory method has been 1068 * declared somewhere that returns several instances of this TestClass). 1069 * If no @Factory method was specified, testMethod will only be invoked 1070 * once. 1071 * <p/> 1072 * Note that this method also takes care of invoking the beforeTestMethod 1073 * and afterTestMethod, if any. 1074 * 1075 * Note (alex): this method can be refactored to use a SingleTestMethodWorker that 1076 * directly invokes 1077 * {@link #invokeTestMethod(Object[], ITestNGMethod, Object[], XmlSuite, Map, ITestClass, ITestNGMethod[], ITestNGMethod[], ConfigurationGroupMethods)} 1078 * and this would simplify the implementation (see how DataTestMethodWorker is used) 1079 */ 1080 @Override 1081 public List<ITestResult> invokeTestMethods(ITestNGMethod testMethod, 1082 ITestNGMethod[] allTestMethods, 1083 int testMethodIndex, 1084 XmlSuite suite, 1085 Map<String, String> parameters, 1086 ConfigurationGroupMethods groupMethods, 1087 Object[] instances, 1088 ITestContext testContext) 1089 { 1090 // Potential bug here if the test method was declared on a parent class 1091 assert null != testMethod.getTestClass() 1092 : "COULDN'T FIND TESTCLASS FOR " + testMethod.getMethod().getDeclaringClass(); 1093 1094 List<ITestResult> result = Lists.newArrayList(); 1095 1096 if (!MethodHelper.isEnabled(testMethod.getMethod(), m_annotationFinder)) { 1097 /* 1098 * return if the method is not enabled. No need to do any more calculations 1099 */ 1100 return result; 1101 } 1102 1103 ITestClass testClass= testMethod.getTestClass(); 1104 long start = System.currentTimeMillis(); 1105 1106 // For invocationCount > 1 and threadPoolSize > 1 the method will be invoked on a thread pool 1107 long timeOutInvocationCount = testMethod.getInvocationTimeOut(); 1108 //FIXME: Is this correct? 1109 boolean onlyOne = testMethod.getThreadPoolSize() > 1 || 1110 timeOutInvocationCount > 0; 1111 1112 int invocationCount = onlyOne ? 1 : testMethod.getInvocationCount(); 1113 int failureCount = 0; 1114 1115 ExpectedExceptionsHolder expectedExceptionHolder = 1116 MethodHelper.findExpectedExceptions(m_annotationFinder, testMethod.getMethod()); 1117 while(invocationCount-- > 0) { 1118 boolean okToProceed = checkDependencies(testMethod, allTestMethods); 1119 1120 if (!okToProceed) { 1121 // 1122 // Not okToProceed. Test is being skipped 1123 // 1124 ITestResult testResult = new TestResult(testClass, null /* instance */, 1125 testMethod, 1126 null /* cause */, 1127 start, 1128 System.currentTimeMillis()); 1129 String missingGroup = testMethod.getMissingGroup(); 1130 if (missingGroup != null) { 1131 testResult.setThrowable(new Throwable("Method " + testMethod 1132 + " depends on nonexistent group \"" + missingGroup + "\"")); 1133 } 1134 1135 testResult.setStatus(ITestResult.SKIP); 1136 result.add(testResult); 1137 m_notifier.addSkippedTest(testMethod, testResult); 1138 runTestListeners(testResult); 1139 return result; 1140 } 1141 1142 // 1143 // If threadPoolSize specified, run this method in its own pool thread. 1144 // 1145 if (testMethod.getThreadPoolSize() > 1 && testMethod.getInvocationCount() > 1) { 1146 return invokePooledTestMethods(testMethod, allTestMethods, suite, 1147 parameters, groupMethods, testContext); 1148 } 1149 // 1150 // No threads, regular invocation 1151 // 1152 else { 1153 ITestNGMethod[] beforeMethods = filterMethods(testClass, testClass.getBeforeTestMethods(), 1154 CAN_RUN_FROM_CLASS); 1155 ITestNGMethod[] afterMethods = filterMethods(testClass, testClass.getAfterTestMethods(), 1156 CAN_RUN_FROM_CLASS); 1157 1158 Map<String, String> allParameterNames = Maps.newHashMap(); 1159 ParameterBag bag = createParameters(testMethod, 1160 parameters, allParameterNames, null, suite, testContext, instances[0], 1161 null); 1162 1163 if (bag.hasErrors()) { 1164 failureCount = handleInvocationResults(testMethod, 1165 bag.errorResults, null, failureCount, expectedExceptionHolder, true, 1166 true /* collect results */); 1167 ITestResult tr = registerSkippedTestResult(testMethod, instances[0], start, 1168 bag.errorResults.get(0).getThrowable()); 1169 result.add(tr); 1170 continue; 1171 } 1172 1173 Iterator<Object[]> allParameterValues = bag.parameterHolder.parameters; 1174 int parametersIndex = 0; 1175 1176 try { 1177 List<TestMethodWithDataProviderMethodWorker> workers = Lists.newArrayList(); 1178 1179 if (bag.parameterHolder.origin == ParameterOrigin.ORIGIN_DATA_PROVIDER && 1180 bag.parameterHolder.dataProviderHolder.annotation.isParallel()) { 1181 while (allParameterValues.hasNext()) { 1182 Object[] parameterValues = injectParameters(allParameterValues.next(), 1183 testMethod.getMethod(), testContext, null /* test result */); 1184 TestMethodWithDataProviderMethodWorker w = 1185 new TestMethodWithDataProviderMethodWorker(this, 1186 testMethod, parametersIndex, 1187 parameterValues, instances, suite, parameters, testClass, 1188 beforeMethods, afterMethods, groupMethods, 1189 expectedExceptionHolder, testContext, m_skipFailedInvocationCounts, 1190 invocationCount, failureCount, m_notifier); 1191 workers.add(w); 1192 // testng387: increment the param index in the bag. 1193 parametersIndex++; 1194 } 1195 PoolService ps = PoolService.getInstance(); 1196 List<ITestResult> r = ps.submitTasksAndWait(testMethod, workers); 1197 result.addAll(r); 1198 1199 } else { 1200 while (allParameterValues.hasNext()) { 1201 Object[] parameterValues = injectParameters(allParameterValues.next(), 1202 testMethod.getMethod(), testContext, null /* test result */); 1203 1204 List<ITestResult> tmpResults = Lists.newArrayList(); 1205 1206 try { 1207 tmpResults.addAll(invokeTestMethod(instances, 1208 testMethod, 1209 parameterValues, 1210 parametersIndex, 1211 suite, 1212 parameters, 1213 testClass, 1214 beforeMethods, 1215 afterMethods, 1216 groupMethods)); 1217 } 1218 finally { 1219 List<Object> failedInstances = Lists.newArrayList(); 1220 1221 failureCount = handleInvocationResults(testMethod, tmpResults, 1222 failedInstances, failureCount, expectedExceptionHolder, true, 1223 false /* don't collect results */); 1224 if (failedInstances.isEmpty()) { 1225 result.addAll(tmpResults); 1226 } else { 1227 for (int i = 0; i < failedInstances.size(); i++) { 1228 List<ITestResult> retryResults = Lists.newArrayList(); 1229 1230 failureCount = 1231 retryFailed(failedInstances.toArray(), 1232 i, testMethod, suite, testClass, beforeMethods, 1233 afterMethods, groupMethods, retryResults, 1234 failureCount, expectedExceptionHolder, 1235 testContext, parameters, parametersIndex); 1236 result.addAll(retryResults); 1237 } 1238 } 1239 1240 // 1241 // If we have a failure, skip all the 1242 // other invocationCounts 1243 // 1244 if (failureCount > 0 1245 && (m_skipFailedInvocationCounts 1246 || testMethod.skipFailedInvocations())) { 1247 while (invocationCount-- > 0) { 1248 result.add(registerSkippedTestResult(testMethod, instances[0], start, null)); 1249 } 1250 break; 1251 } 1252 }// end finally 1253 parametersIndex++; 1254 } 1255 } 1256 } 1257 catch (Throwable cause) { 1258 ITestResult r = 1259 new TestResult(testMethod.getTestClass(), 1260 instances[0], 1261 testMethod, 1262 cause, 1263 start, 1264 System.currentTimeMillis()); 1265 r.setStatus(TestResult.FAILURE); 1266 result.add(r); 1267 runTestListeners(r); 1268 m_notifier.addFailedTest(testMethod, r); 1269 } // catch 1270 } 1271 } 1272 1273 return result; 1274 1275 } // invokeTestMethod 1276 1277 private ITestResult registerSkippedTestResult(ITestNGMethod testMethod, Object instance, 1278 long start, Throwable throwable) { 1279 ITestResult result = 1280 new TestResult(testMethod.getTestClass(), 1281 instance, 1282 testMethod, 1283 throwable, 1284 start, 1285 System.currentTimeMillis()); 1286 result.setStatus(TestResult.SKIP); 1287 runTestListeners(result); 1288 1289 return result; 1290 } 1291 1292 /** 1293 * Gets an array of parameter values returned by data provider or the ones that 1294 * are injected based on parameter type. The method also checks for {@code NoInjection} 1295 * annotation 1296 * @param parameterValues parameter values from a data provider 1297 * @param method method to be invoked 1298 * @param context test context 1299 * @param testResult test result 1300 * @return 1301 */ 1302 private Object[] injectParameters(Object[] parameterValues, Method method, 1303 ITestContext context, ITestResult testResult) 1304 throws TestNGException { 1305 List<Object> vResult = Lists.newArrayList(); 1306 int i = 0; 1307 int numValues = parameterValues.length; 1308 int numParams = method.getParameterTypes().length; 1309 1310 if (numValues > numParams && ! method.isVarArgs()) { 1311 throw new TestNGException("The data provider is trying to pass " + numValues 1312 + " parameters but the method " 1313 + method.getDeclaringClass().getName() + "#" + method.getName() 1314 + " takes " + numParams); 1315 } 1316 1317 // beyond this, numValues <= numParams 1318 for (Class<?> cls : method.getParameterTypes()) { 1319 Annotation[] annotations = method.getParameterAnnotations()[i]; 1320 boolean noInjection = false; 1321 for (Annotation a : annotations) { 1322 if (a instanceof NoInjection) { 1323 noInjection = true; 1324 break; 1325 } 1326 } 1327 Object injected = Parameters.getInjectedParameter(cls, method, context, testResult); 1328 if (injected != null && ! noInjection) { 1329 vResult.add(injected); 1330 } else { 1331 try { 1332 if (method.isVarArgs()) vResult.add(parameterValues); 1333 else vResult.add(parameterValues[i++]); 1334 } catch (ArrayIndexOutOfBoundsException ex) { 1335 throw new TestNGException("The data provider is trying to pass " + numValues 1336 + " parameters but the method " 1337 + method.getDeclaringClass().getName() + "#" + method.getName() 1338 + " takes " + numParams 1339 + " and TestNG is unable in inject a suitable object", ex); 1340 } 1341 } 1342 } 1343 return vResult.toArray(new Object[vResult.size()]); 1344 } 1345 1346 private ParameterBag handleParameters(ITestNGMethod testMethod, 1347 Object instance, 1348 Map<String, String> allParameterNames, 1349 Map<String, String> parameters, 1350 Object[] parameterValues, 1351 XmlSuite suite, 1352 ITestContext testContext, 1353 Object fedInstance, 1354 ITestResult testResult) 1355 { 1356 try { 1357 return new ParameterBag( 1358 Parameters.handleParameters(testMethod, 1359 allParameterNames, 1360 instance, 1361 new Parameters.MethodParameters(parameters, parameterValues, 1362 testMethod.getMethod(), testContext, testResult), 1363 suite, 1364 m_annotationFinder, 1365 fedInstance), 1366 null /* TestResult */); 1367 } 1368// catch(TestNGException ex) { 1369// throw ex; 1370// } 1371 catch(Throwable cause) { 1372 return new ParameterBag(null /* ParameterHolder */, 1373 new TestResult( 1374 testMethod.getTestClass(), 1375 instance, 1376 testMethod, 1377 cause, 1378 System.currentTimeMillis(), 1379 System.currentTimeMillis())); 1380 } 1381 } 1382 1383 /** 1384 * Invokes a method that has a specified threadPoolSize. 1385 */ 1386 private List<ITestResult> invokePooledTestMethods(ITestNGMethod testMethod, 1387 ITestNGMethod[] allTestMethods, 1388 XmlSuite suite, 1389 Map<String, String> parameters, 1390 ConfigurationGroupMethods groupMethods, 1391 ITestContext testContext) 1392 { 1393 // 1394 // Create the workers 1395 // 1396 List<IWorker<ITestNGMethod>> workers = Lists.newArrayList(); 1397 1398 // Create one worker per invocationCount 1399 for (int i = 0; i < testMethod.getInvocationCount(); i++) { 1400 // we use clones for reporting purposes 1401 ITestNGMethod clonedMethod= testMethod.clone(); 1402 clonedMethod.setInvocationCount(1); 1403 clonedMethod.setThreadPoolSize(1); 1404 1405 MethodInstance mi = new MethodInstance(clonedMethod); 1406 workers.add(new SingleTestMethodWorker(this, 1407 mi, 1408 suite, 1409 parameters, 1410 allTestMethods, 1411 testContext)); 1412 } 1413 1414 return runWorkers(testMethod, workers, testMethod.getThreadPoolSize(), groupMethods, suite, parameters); 1415 } 1416 1417 /** 1418 * @param testMethod 1419 * @param result 1420 * @param failureCount 1421 * @param expectedExceptionsHolder 1422 * @return 1423 */ 1424 int handleInvocationResults(ITestNGMethod testMethod, 1425 List<ITestResult> result, 1426 List<Object> failedInstances, 1427 int failureCount, 1428 ExpectedExceptionsHolder expectedExceptionsHolder, 1429 boolean triggerListeners, 1430 boolean collectResults) 1431 { 1432 // 1433 // Go through all the results and create a TestResult for each of them 1434 // 1435 List<ITestResult> resultsToRetry = Lists.newArrayList(); 1436 1437 for (int i = 0; i < result.size(); i++) { 1438 ITestResult testResult = result.get(i); 1439 Throwable ite= testResult.getThrowable(); 1440 int status= testResult.getStatus(); 1441 1442 // Exception thrown? 1443 if (ite != null) { 1444 1445 // Invocation caused an exception, see if the method was annotated with @ExpectedException 1446 if (isExpectedException(ite, expectedExceptionsHolder)) { 1447 if (messageRegExpMatches(expectedExceptionsHolder.messageRegExp, ite)) { 1448 testResult.setStatus(ITestResult.SUCCESS); 1449 status= ITestResult.SUCCESS; 1450 } 1451 else { 1452 testResult.setThrowable( 1453 new TestException("The exception was thrown with the wrong message:" + 1454 " expected \"" + expectedExceptionsHolder.messageRegExp + "\"" + 1455 " but got \"" + ite.getMessage() + "\"", ite)); 1456 status= ITestResult.FAILURE; 1457 } 1458 } else if (ite != null && expectedExceptionsHolder != null) { 1459 testResult.setThrowable( 1460 new TestException("Expected exception " 1461 + expectedExceptionsHolder.expectedClasses[0].getName() 1462 + " but got " + ite, ite)); 1463 status= ITestResult.FAILURE; 1464 } 1465 else if (SkipException.class.isAssignableFrom(ite.getClass())){ 1466 SkipException skipEx= (SkipException) ite; 1467 if(skipEx.isSkip()) { 1468 status = ITestResult.SKIP; 1469 } 1470 else { 1471 handleException(ite, testMethod, testResult, failureCount++); 1472 status = ITestResult.FAILURE; 1473 } 1474 } 1475 else { 1476 handleException(ite, testMethod, testResult, failureCount++); 1477 status= testResult.getStatus(); 1478 } 1479 } 1480 1481 // No exception thrown, make sure we weren't expecting one 1482 else if(status != ITestResult.SKIP && expectedExceptionsHolder != null) { 1483 Class<?>[] classes = expectedExceptionsHolder.expectedClasses; 1484 if (classes != null && classes.length > 0) { 1485 testResult.setThrowable( 1486 new TestException("Method " + testMethod + " should have thrown an exception of " 1487 + expectedExceptionsHolder.expectedClasses[0])); 1488 status= ITestResult.FAILURE; 1489 } 1490 } 1491 1492 testResult.setStatus(status); 1493 1494 boolean retry = false; 1495 1496 if (testResult.getStatus() == ITestResult.FAILURE) { 1497 IRetryAnalyzer retryAnalyzer = testMethod.getRetryAnalyzer(); 1498 1499 if (retryAnalyzer != null && failedInstances != null) { 1500 retry = retryAnalyzer.retry(testResult); 1501 } 1502 1503 if (retry) { 1504 resultsToRetry.add(testResult); 1505 if (failedInstances != null) { 1506 failedInstances.add(testResult.getInstance()); 1507 } 1508 } 1509 } 1510 if (collectResults) { 1511 // Collect the results 1512 if(ITestResult.SUCCESS == status) { 1513 m_notifier.addPassedTest(testMethod, testResult); 1514 } 1515 else if(ITestResult.SKIP == status) { 1516 m_notifier.addSkippedTest(testMethod, testResult); 1517 } 1518 else if(ITestResult.FAILURE == status) { 1519 m_notifier.addFailedTest(testMethod, testResult); 1520 } 1521 else if(ITestResult.SUCCESS_PERCENTAGE_FAILURE == status) { 1522 m_notifier.addFailedButWithinSuccessPercentageTest(testMethod, testResult); 1523 } 1524 else { 1525 assert false : "UNKNOWN STATUS:" + status; 1526 } 1527// if (triggerListeners && status != ITestResult.SUCCESS) { 1528// runTestListeners(testResult); 1529// } 1530 } 1531 } // for results 1532 1533 return removeResultsToRetryFromResult(resultsToRetry, result, failureCount); 1534 } 1535 1536 /** 1537 * message / regEx .* other 1538 * null true false 1539 * non-null true match 1540 */ 1541 private boolean messageRegExpMatches(String messageRegExp, Throwable ite) { 1542 if (".*".equals(messageRegExp)) { 1543 return true; 1544 } else { 1545 if (ite.getMessage() == null) { 1546 return false; 1547 } else { 1548 return Pattern.matches(messageRegExp, ite.getMessage()); 1549 } 1550 } 1551 } 1552 1553 private int removeResultsToRetryFromResult(List<ITestResult> resultsToRetry, 1554 List<ITestResult> result, int failureCount) { 1555 if (resultsToRetry != null) { 1556 for (ITestResult res : resultsToRetry) { 1557 result.remove(res); 1558 failureCount--; 1559 } 1560 } 1561 return failureCount; 1562 } 1563 1564 /** 1565 * To reduce thread contention and also to correctly handle thread-confinement 1566 * this method invokes the @BeforeGroups and @AfterGroups corresponding to the current @Test method. 1567 */ 1568 private List<ITestResult> runWorkers(ITestNGMethod testMethod, 1569 List<IWorker<ITestNGMethod>> workers, 1570 int threadPoolSize, 1571 ConfigurationGroupMethods groupMethods, 1572 XmlSuite suite, 1573 Map<String, String> parameters) 1574 { 1575 // Invoke @BeforeGroups on the original method (reduce thread contention, 1576 // and also solve thread confinement) 1577 ITestClass testClass= testMethod.getTestClass(); 1578 Object[] instances = testClass.getInstances(true); 1579 for(Object instance: instances) { 1580 invokeBeforeGroupsConfigurations(testClass, testMethod, groupMethods, suite, parameters, instance); 1581 } 1582 1583 1584 long maxTimeOut= -1; // 10 seconds 1585 1586 for(IWorker<ITestNGMethod> tmw : workers) { 1587 long mt= tmw.getTimeOut(); 1588 if(mt > maxTimeOut) { 1589 maxTimeOut= mt; 1590 } 1591 } 1592 1593 ThreadUtil.execute(workers, threadPoolSize, maxTimeOut, true); 1594 1595 // 1596 // Collect all the TestResults 1597 // 1598 List<ITestResult> result = Lists.newArrayList(); 1599 for (IWorker<ITestNGMethod> tmw : workers) { 1600 if (tmw instanceof TestMethodWorker) { 1601 result.addAll(((TestMethodWorker)tmw).getTestResults()); 1602 } 1603 } 1604 1605 for(Object instance: instances) { 1606 invokeAfterGroupsConfigurations(testClass, testMethod, groupMethods, suite, parameters, instance); 1607 } 1608 1609 return result; 1610 } 1611 1612 /** 1613 * Checks to see of the test method has certain dependencies that prevents 1614 * TestNG from executing it 1615 * @param testMethod test method being checked for 1616 * @param testClass 1617 * @return dependencies have been run successfully 1618 */ 1619 private boolean checkDependencies(ITestNGMethod testMethod, 1620 ITestNGMethod[] allTestMethods) 1621 { 1622 boolean result = true; 1623 1624 // If this method is marked alwaysRun, no need to check for its dependencies 1625 if (testMethod.isAlwaysRun()) { 1626 return true; 1627 } 1628 1629 // Any missing group? 1630 if (testMethod.getMissingGroup() != null 1631 && !testMethod.ignoreMissingDependencies()) { 1632 return false; 1633 } 1634 1635 // If this method depends on groups, collect all the methods that 1636 // belong to these groups and make sure they have been run successfully 1637 if (dependsOnGroups(testMethod)) { 1638 String[] groupsDependedUpon = testMethod.getGroupsDependedUpon(); 1639 1640 // Get all the methods that belong to the group depended upon 1641 for (String element : groupsDependedUpon) { 1642 ITestNGMethod[] methods = 1643 MethodGroupsHelper.findMethodsThatBelongToGroup(testMethod, 1644 m_testContext.getAllTestMethods(), 1645 element); 1646 1647 result = result && haveBeenRunSuccessfully(testMethod, methods); 1648 } 1649 } // depends on groups 1650 1651 // If this method depends on other methods, make sure all these other 1652 // methods have been run successfully 1653 if (result && dependsOnMethods(testMethod)) { 1654 ITestNGMethod[] methods = 1655 MethodHelper.findDependedUponMethods(testMethod, allTestMethods); 1656 1657 result = result && haveBeenRunSuccessfully(testMethod, methods); 1658 } 1659 1660 return result; 1661 } 1662 1663 /** 1664 * @return the test results that apply to one of the instances of the testMethod. 1665 */ 1666 private Set<ITestResult> keepSameInstances(ITestNGMethod method, Set<ITestResult> results) { 1667 Set<ITestResult> result = Sets.newHashSet(); 1668 for (ITestResult r : results) { 1669 for (Object o : method.getInstances()) { 1670 // Keep this instance if 1) It's on a different class or 2) It's on the same class 1671 // and on the same instance 1672 Object instance = r.getInstance() != null 1673 ? r.getInstance() : r.getMethod().getInstances()[0]; 1674 if (r.getTestClass() != method.getTestClass() || instance == o) result.add(r); 1675 } 1676 } 1677 return result; 1678 } 1679 1680 /** 1681 * @return true if all the methods have been run successfully 1682 */ 1683 private boolean haveBeenRunSuccessfully(ITestNGMethod testMethod, ITestNGMethod[] methods) { 1684 // Make sure the method has been run successfully 1685 for (ITestNGMethod method : methods) { 1686 Set<ITestResult> results = keepSameInstances(testMethod, m_notifier.getPassedTests(method)); 1687 Set<ITestResult> failedAndSkippedMethods = Sets.newHashSet(); 1688 failedAndSkippedMethods.addAll(m_notifier.getFailedTests(method)); 1689 failedAndSkippedMethods.addAll(m_notifier.getSkippedTests(method)); 1690 Set<ITestResult> failedresults = keepSameInstances(testMethod, failedAndSkippedMethods); 1691 1692 // If failed results were returned on the same instance, then these tests didn't pass 1693 if (failedresults != null && failedresults.size() > 0) { 1694 return false; 1695 } 1696 1697 for (ITestResult result : results) { 1698 if(!result.isSuccess()) { 1699 return false; 1700 } 1701 } 1702 } 1703 1704 return true; 1705 } 1706 1707// private boolean containsInstance(Set<ITestResult> failedresults, Object[] instances) { 1708// for (ITestResult tr : failedresults) { 1709// for (Object o : instances) { 1710// if (o == tr.getInstance()) { 1711// return true; 1712// } 1713// } 1714// } 1715// return false; 1716// } 1717 1718 /** 1719 * An exception was thrown by the test, determine if this method 1720 * should be marked as a failure or as failure_but_within_successPercentage 1721 */ 1722 private void handleException(Throwable throwable, 1723 ITestNGMethod testMethod, 1724 ITestResult testResult, 1725 int failureCount) { 1726 testResult.setThrowable(throwable); 1727 int successPercentage= testMethod.getSuccessPercentage(); 1728 int invocationCount= testMethod.getInvocationCount(); 1729 float numberOfTestsThatCanFail= ((100 - successPercentage) * invocationCount) / 100f; 1730 1731 if(failureCount < numberOfTestsThatCanFail) { 1732 testResult.setStatus(ITestResult.SUCCESS_PERCENTAGE_FAILURE); 1733 } 1734 else { 1735 testResult.setStatus(ITestResult.FAILURE); 1736 } 1737 1738 } 1739 1740 /** 1741 * @param ite The exception that was just thrown 1742 * @param expectedExceptions The list of expected exceptions for this 1743 * test method 1744 * @return true if the exception that was just thrown is part of the 1745 * expected exceptions 1746 */ 1747 private boolean isExpectedException(Throwable ite, ExpectedExceptionsHolder exceptionHolder) { 1748 if (exceptionHolder == null) { 1749 return false; 1750 } 1751 1752 // TestException is the wrapper exception that TestNG will be throwing when an exception was 1753 // expected but not thrown 1754 if (ite.getClass() == TestException.class) { 1755 return false; 1756 } 1757 1758 Class<?>[] exceptions = exceptionHolder.expectedClasses; 1759 Class<?> realExceptionClass= ite.getClass(); 1760 1761 for (Class<?> exception : exceptions) { 1762 if (exception.isAssignableFrom(realExceptionClass)) { 1763 return true; 1764 } 1765 } 1766 1767 return false; 1768 } 1769 1770 static interface Predicate<K, T> { 1771 boolean isTrue(K k, T v); 1772 } 1773 1774 static class CanRunFromClassPredicate implements Predicate <ITestNGMethod, IClass> { 1775 @Override 1776 public boolean isTrue(ITestNGMethod m, IClass v) { 1777 return m.canRunFromClass(v); 1778 } 1779 } 1780 1781 static class SameClassNamePredicate implements Predicate<ITestNGMethod, IClass> { 1782 @Override 1783 public boolean isTrue(ITestNGMethod m, IClass c) { 1784 return c == null || m.getTestClass().getName().equals(c.getName()); 1785 } 1786 } 1787 1788 /** 1789 * @return Only the ITestNGMethods applicable for this testClass 1790 */ 1791 private ITestNGMethod[] filterMethods(IClass testClass, ITestNGMethod[] methods, 1792 Predicate<ITestNGMethod, IClass> predicate) { 1793 List<ITestNGMethod> vResult= Lists.newArrayList(); 1794 1795 for(ITestNGMethod tm : methods) { 1796 if (predicate.isTrue(tm, testClass)) { 1797 log(10, "Keeping method " + tm + " for class " + testClass); 1798 vResult.add(tm); 1799 } else { 1800 log(10, "Filtering out method " + tm + " for class " + testClass); 1801 } 1802 } 1803 1804 ITestNGMethod[] result= vResult.toArray(new ITestNGMethod[vResult.size()]); 1805 1806 return result; 1807 } 1808 1809 /** 1810 * @return true if this method depends on certain groups. 1811 */ 1812 private boolean dependsOnGroups(ITestNGMethod tm) { 1813 String[] groups = tm.getGroupsDependedUpon(); 1814 boolean result = (null != groups) && (groups.length > 0); 1815 return result; 1816 } 1817 1818 /** 1819 * @return true if this method depends on certain groups. 1820 */ 1821 private boolean dependsOnMethods(ITestNGMethod tm) { 1822 String[] methods = tm.getMethodsDependedUpon(); 1823 boolean result = (null != methods) && (methods.length > 0); 1824 return result; 1825 } 1826 1827 private void runConfigurationListeners(ITestResult tr, boolean before) { 1828 if (before) { 1829 for(IConfigurationListener icl: m_notifier.getConfigurationListeners()) { 1830 if (icl instanceof IConfigurationListener2) { 1831 ((IConfigurationListener2) icl).beforeConfiguration(tr); 1832 } 1833 } 1834 } else { 1835 for(IConfigurationListener icl: m_notifier.getConfigurationListeners()) { 1836 switch(tr.getStatus()) { 1837 case ITestResult.SKIP: 1838 icl.onConfigurationSkip(tr); 1839 break; 1840 case ITestResult.FAILURE: 1841 icl.onConfigurationFailure(tr); 1842 break; 1843 case ITestResult.SUCCESS: 1844 icl.onConfigurationSuccess(tr); 1845 break; 1846 } 1847 } 1848 } 1849 } 1850 1851 void runTestListeners(ITestResult tr) { 1852 runTestListeners(tr, m_notifier.getTestListeners()); 1853 } 1854 1855 // TODO: move this from here as it is directly called from TestNG 1856 public static void runTestListeners(ITestResult tr, List<ITestListener> listeners) { 1857 for (ITestListener itl : listeners) { 1858 switch(tr.getStatus()) { 1859 case ITestResult.SKIP: { 1860 itl.onTestSkipped(tr); 1861 break; 1862 } 1863 case ITestResult.SUCCESS_PERCENTAGE_FAILURE: { 1864 itl.onTestFailedButWithinSuccessPercentage(tr); 1865 break; 1866 } 1867 case ITestResult.FAILURE: { 1868 itl.onTestFailure(tr); 1869 break; 1870 } 1871 case ITestResult.SUCCESS: { 1872 itl.onTestSuccess(tr); 1873 break; 1874 } 1875 1876 case ITestResult.STARTED: { 1877 itl.onTestStart(tr); 1878 break; 1879 } 1880 1881 default: { 1882 assert false : "UNKNOWN STATUS:" + tr; 1883 } 1884 } 1885 } 1886 } 1887 1888 private void log(int level, String s) { 1889 Utils.log("Invoker " + Thread.currentThread().hashCode(), level, s); 1890 } 1891 1892 /** 1893 * This class holds a {@code ParameterHolder} and in case of an error, a non-null 1894 * {@code TestResult} containing the cause 1895 */ 1896 private static class ParameterBag { 1897 final ParameterHolder parameterHolder; 1898 final List<ITestResult> errorResults = Lists.newArrayList(); 1899 1900 public ParameterBag(ParameterHolder params, TestResult tr) { 1901 parameterHolder = params; 1902 if (tr != null) { 1903 errorResults.add(tr); 1904 } 1905 } 1906 1907 public boolean hasErrors() { 1908 return !errorResults.isEmpty(); 1909 } 1910 } 1911 1912} 1913