private void fireExecutionStarted(TestDescriptor testDescriptor) { testRun.markStarted(testDescriptor); listener.executionStarted(testDescriptor); }
private void reportUnexpectedFailure(TestRun testRun, RunnerTestDescriptor runnerTestDescriptor, TestExecutionResult result) { if (testRun.isNotStarted(runnerTestDescriptor)) { engineExecutionListener.executionStarted(runnerTestDescriptor); } engineExecutionListener.executionFinished(runnerTestDescriptor, result); }
@Override public void execute(ExecutionRequest request) { EngineExecutionListener engineExecutionListener = request.getEngineExecutionListener(); TestDescriptor engineTestDescriptor = request.getRootTestDescriptor(); engineExecutionListener.executionStarted(engineTestDescriptor); RunnerExecutor runnerExecutor = new RunnerExecutor(engineExecutionListener); executeAllChildren(runnerExecutor, engineTestDescriptor); engineExecutionListener.executionFinished(engineTestDescriptor, successful()); }
@Override public void startBenchmark(BenchmarkParams benchParams) { log.clear(); lastKnownBenchmark = benchParams; TestDescriptor descriptor = descriptionResolver.apply(benchParams); listener.executionStarted(descriptor); delegate.startBenchmark(benchParams); }
@Override public void execute(ExecutionRequest request) { // Use ExecutionRequest to execute TestDescriptor TestDescriptor rootTestDescriptor = request.getRootTestDescriptor(); request.getEngineExecutionListener() .executionStarted(rootTestDescriptor); }
@Example void previouslyFailedTestsAreRunFirst() { TestDescriptor descriptor = TestDescriptorBuilder.forClass(TestContainer.class, "test1", "test2", "test3").build(); EngineExecutionListener listener = Mockito.mock(EngineExecutionListener.class); previouslyFailedTests.add(testId(descriptor, "test2()")); previouslyFailedTests.add(testId(descriptor, "test3()")); executor.execute(descriptor, listener); InOrder events = Mockito.inOrder(listener); events.verify(listener).executionStarted(isPropertyDescriptorFor(TestContainer.class, "test2")); events.verify(listener).executionStarted(isPropertyDescriptorFor(TestContainer.class, "test3")); events.verify(listener).executionStarted(isPropertyDescriptorFor(TestContainer.class, "test1")); }
private void executeRecursively() { taskContext.getListener().executionStarted(testDescriptor); started = true; throwableCollector.execute(() -> { node.around(context, ctx -> { context = ctx; throwableCollector.execute(() -> { // @formatter:off List<NodeTestTask<C>> children = testDescriptor.getChildren().stream() .map(descriptor -> new NodeTestTask<C>(taskContext, descriptor)) .collect(toCollection(ArrayList::new)); // @formatter:on context = node.before(context); final DynamicTestExecutor dynamicTestExecutor = new DefaultDynamicTestExecutor(); context = node.execute(context, dynamicTestExecutor); if (!children.isEmpty()) { children.forEach(child -> child.setParentContext(context)); taskContext.getExecutorService().invokeAll(children); } throwableCollector.execute(dynamicTestExecutor::awaitFinished); }); throwableCollector.execute(() -> node.after(context)); }); }); }
@Example void emptyEngine() { TestDescriptor engineDescriptor = forEngine(testEngine).build(); executeTests(engineDescriptor); InOrder events = Mockito.inOrder(eventRecorder); events.verify(eventRecorder).executionStarted(engineDescriptor); events.verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void engineWithEmptyClass() { TestDescriptor engineDescriptor = forEngine(testEngine).with(ContainerClass.class).build(); executeTests(engineDescriptor); InOrder events = Mockito.inOrder(eventRecorder); events.verify(eventRecorder).executionStarted(engineDescriptor); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(ContainerClass.class)); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(ContainerClass.class), isSuccessful()); events.verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void engineWithClassWithTests() { TestDescriptor engineDescriptor = forEngine(testEngine).with(forClass(ContainerClass.class, "succeeding", "failing")).build(); executeTests(engineDescriptor); InOrder events = Mockito.inOrder(eventRecorder); events.verify(eventRecorder).executionStarted(engineDescriptor); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(ContainerClass.class)); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(ContainerClass.class, "succeeding")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(ContainerClass.class, "succeeding"), isSuccessful()); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(ContainerClass.class, "failing")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(ContainerClass.class, "failing"), isFailed("expected fail")); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(ContainerClass.class), isSuccessful()); events.verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void engineWithTwoClasses() { TestDescriptor engineDescriptor = forEngine(testEngine) .with(forClass(ContainerClass.class, "succeeding", "failing"), forClass(SecondContainerClass.class, "succeeding")).build(); executeTests(engineDescriptor); InOrder events = Mockito.inOrder(eventRecorder); events.verify(eventRecorder).executionStarted(engineDescriptor); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(ContainerClass.class)); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(ContainerClass.class, "succeeding")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(ContainerClass.class, "succeeding"), isSuccessful()); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(ContainerClass.class, "failing")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(ContainerClass.class, "failing"), isFailed("expected fail")); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(ContainerClass.class), isSuccessful()); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(SecondContainerClass.class)); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(SecondContainerClass.class, "succeeding")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(SecondContainerClass.class, "succeeding"), isSuccessful()); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(SecondContainerClass.class), isSuccessful()); events.verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void runTestsFromMethod() { LauncherDiscoveryRequest discoveryRequest = request().selectors(selectMethod(SimpleExampleTests.class, "succeeding")).build(); TestDescriptor engineDescriptor = runTests(discoveryRequest); verify(eventRecorder).executionStarted(engineDescriptor); verify(eventRecorder).executionStarted(TestDescriptorMatchers.isClassDescriptorFor(SimpleExampleTests.class)); verify(eventRecorder).executionStarted(TestDescriptorMatchers.isPropertyDescriptorFor(SimpleExampleTests.class, "succeeding")); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isPropertyDescriptorFor(SimpleExampleTests.class, "succeeding"), TestExecutionResultMatchers.isSuccessful() ); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isClassDescriptorFor(SimpleExampleTests.class), TestExecutionResultMatchers.isSuccessful() ); verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
private void verifyRunOfSimpleExampleTests() { verify(eventRecorder).executionStarted(TestDescriptorMatchers.isClassDescriptorFor(SimpleExampleTests.class)); // SimpleExampleTests.failing() verify(eventRecorder).executionStarted(TestDescriptorMatchers.isPropertyDescriptorFor(SimpleExampleTests.class, "failing")); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isPropertyDescriptorFor(SimpleExampleTests.class, "failing"), TestExecutionResultMatchers.isFailed() ); // SimpleExampleTests.succeeding() verify(eventRecorder).executionStarted(TestDescriptorMatchers.isPropertyDescriptorFor(SimpleExampleTests.class, "succeeding")); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isPropertyDescriptorFor(SimpleExampleTests.class, "succeeding"), TestExecutionResultMatchers.isSuccessful() ); // SimpleExampleTests.withJupiterAnnotation() verify(eventRecorder).executionSkipped( TestDescriptorMatchers.isSkipDecoratorFor(SimpleExampleTests.class, "withJupiterAnnotation"), anyString() ); // SimpleExampleTests.staticExample() verify(eventRecorder).executionSkipped( TestDescriptorMatchers.isSkipDecoratorFor(SimpleExampleTests.class, "staticExample"), anyString() ); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isClassDescriptorFor(SimpleExampleTests.class), TestExecutionResultMatchers.isSuccessful() ); }
@Example void engineWithNestedGroups() { TestDescriptor engineDescriptor = forEngine(testEngine).with(forClass(TopLevelContainer.class, "topLevelSuccess").with( forClass(TopLevelContainer.InnerGroup.class, "innerGroupSuccess") .with(forClass(TopLevelContainer.InnerGroup.InnerInnerGroup.class, "innerInnerGroupSuccess")), forClass(TopLevelContainer.AnotherGroup.class))).build(); executeTests(engineDescriptor); InOrder events = Mockito.inOrder(eventRecorder); events.verify(eventRecorder).executionStarted(engineDescriptor); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(TopLevelContainer.class)); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(TopLevelContainer.InnerGroup.class)); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(TopLevelContainer.InnerGroup.InnerInnerGroup.class)); events.verify(eventRecorder) .executionStarted(isPropertyDescriptorFor(TopLevelContainer.InnerGroup.InnerInnerGroup.class, "innerInnerGroupSuccess")); events.verify(eventRecorder).executionFinished( isPropertyDescriptorFor(TopLevelContainer.InnerGroup.InnerInnerGroup.class, "innerInnerGroupSuccess"), isSuccessful()); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(TopLevelContainer.InnerGroup.InnerInnerGroup.class), isSuccessful()); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(TopLevelContainer.InnerGroup.class, "innerGroupSuccess")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(TopLevelContainer.InnerGroup.class, "innerGroupSuccess"), isSuccessful()); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(TopLevelContainer.InnerGroup.class), isSuccessful()); events.verify(eventRecorder).executionStarted(isClassDescriptorFor(TopLevelContainer.AnotherGroup.class)); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(TopLevelContainer.AnotherGroup.class), isSuccessful()); events.verify(eventRecorder).executionStarted(isPropertyDescriptorFor(TopLevelContainer.class, "topLevelSuccess")); events.verify(eventRecorder).executionFinished(isPropertyDescriptorFor(TopLevelContainer.class, "topLevelSuccess"), isSuccessful()); events.verify(eventRecorder).executionFinished(isClassDescriptorFor(TopLevelContainer.class), isSuccessful()); events.verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void runTestsFromPackage() { LauncherDiscoveryRequest discoveryRequest = request().selectors(selectPackage("examples.packageWithSingleContainer")).build(); TestDescriptor engineDescriptor = runTests(discoveryRequest); verify(eventRecorder).executionStarted(engineDescriptor); verifyRunOfSimpleExampleTests(); verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
private void reportCompletion() { if (throwableCollector.isEmpty() && skipResult.isSkipped()) { try { node.nodeSkipped(context, testDescriptor, skipResult); } catch (Throwable throwable) { BlacklistedExceptions.rethrowIfBlacklisted(throwable); logger.debug(throwable, () -> String.format("Failed to invoke nodeSkipped() on Node %s", testDescriptor.getUniqueId())); } taskContext.getListener().executionSkipped(testDescriptor, skipResult.getReason().orElse("<unknown>")); return; } if (!started) { // Call executionStarted first to comply with the contract of EngineExecutionListener. taskContext.getListener().executionStarted(testDescriptor); } try { node.nodeFinished(context, testDescriptor, throwableCollector.toTestExecutionResult()); } catch (Throwable throwable) { BlacklistedExceptions.rethrowIfBlacklisted(throwable); logger.debug(throwable, () -> String.format("Failed to invoke nodeFinished() on Node %s", testDescriptor.getUniqueId())); } taskContext.getListener().executionFinished(testDescriptor, throwableCollector.toTestExecutionResult()); throwableCollector = null; }
@Example void runTestsFromClass() { LauncherDiscoveryRequest discoveryRequest = request().selectors(selectClass(SimpleExampleTests.class)).build(); TestDescriptor engineDescriptor = runTests(discoveryRequest); verify(eventRecorder).executionStarted(engineDescriptor); verifyRunOfSimpleExampleTests(); verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void runDisabledTests() { LauncherDiscoveryRequest discoveryRequest = request().selectors(selectPackage("examples.packageWithDisabledTests")).build(); TestDescriptor engineDescriptor = runTests(discoveryRequest); verify(eventRecorder).executionStarted(engineDescriptor); // DisabledTests verify(eventRecorder).executionStarted(TestDescriptorMatchers.isClassDescriptorFor(DisabledTests.class)); verify(eventRecorder).executionStarted(TestDescriptorMatchers.isPropertyDescriptorFor(DisabledTests.class, "success")); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isPropertyDescriptorFor(DisabledTests.class, "success"), TestExecutionResultMatchers.isSuccessful() ); verify(eventRecorder).executionSkipped( TestDescriptorMatchers.isPropertyDescriptorFor(DisabledTests.class, "disabledSuccess"), eq("a reason") ); verify(eventRecorder).executionSkipped( TestDescriptorMatchers.isPropertyDescriptorFor(DisabledTests.class, "disabledFailure"), startsWith("@Disabled:") ); verify(eventRecorder).executionSkipped( TestDescriptorMatchers.isClassDescriptorFor(DisabledTests.DisabledGroup.class), startsWith("@Disabled:") ); verify(eventRecorder).executionFinished( TestDescriptorMatchers.isClassDescriptorFor(DisabledTests.class), TestExecutionResultMatchers.isSuccessful() ); verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
@Example void runTestsFromRootDir() { LauncherDiscoveryRequest discoveryRequest = request().selectors(selectClasspathRoots(JqwikReflectionSupport.getAllClasspathRootDirectories())) .filters(PackageNameFilter.includePackageNames("examples.packageWithSingleContainer")).build(); TestDescriptor engineDescriptor = runTests(discoveryRequest); verify(eventRecorder).executionStarted(engineDescriptor); verifyRunOfSimpleExampleTests(); verify(eventRecorder).executionFinished(engineDescriptor, TestExecutionResult.successful()); }
public void execute(TestDescriptor testDescriptor, EngineExecutionListener listener) { JmhSupport support = new JmhSupport(); ChainedOptionsBuilder optionsBuilder = support.options(); List<AbstractBenchmarkDescriptor> methods = collectBenchmarkMethods(testDescriptor); List<AbstractBenchmarkDescriptor> includes = getIncludes(testDescriptor); if (!support.isEnabled()) { listener.executionSkipped(testDescriptor, "No benchmarks"); return; } List<String> includePatterns = evaluateBenchmarksToRun(includes, listener); if (!shouldRun(includePatterns)) { return; } includePatterns.forEach(optionsBuilder::include); CacheFunction cache = new CacheFunction(methods); Options options = optionsBuilder.build(); NotifyingOutputFormat notifyingOutputFormat = new NotifyingOutputFormat(listener, cache, support.createOutputFormat(options)); try { listener.executionStarted(testDescriptor); support.publishResults(notifyingOutputFormat, new Runner(options, notifyingOutputFormat).run()); listener.executionFinished(testDescriptor, TestExecutionResult.successful()); } catch (RunnerException e) { listener.executionFinished(testDescriptor, TestExecutionResult.failed(e)); } }