Skip to content

Commit 5c1f6ee

Browse files
baflQAbaflQA
authored
Add DataProvider execution support to Allure TestNG setup section (#1236)
1 parent 5ad7b9e commit 5c1f6ee

18 files changed

+579
-11
lines changed

‎allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
import org.testng.IAttributes;
4040
import org.testng.IClass;
4141
import org.testng.IConfigurationListener;
42+
import org.testng.IDataProviderListener;
43+
import org.testng.IDataProviderMethod;
4244
import org.testng.IInvokedMethod;
4345
import org.testng.IInvokedMethodListener;
4446
import org.testng.IMethodInstance;
@@ -110,7 +112,8 @@ public class AllureTestNg implements
110112
ITestListener,
111113
IInvokedMethodListener,
112114
IConfigurationListener,
113-
IMethodInterceptor {
115+
IMethodInterceptor,
116+
IDataProviderListener {
114117

115118
private static final Logger LOGGER = LoggerFactory.getLogger(AllureTestNg.class);
116119

@@ -150,6 +153,10 @@ public class AllureTestNg implements
150153
* Store uuid for class test containers.
151154
*/
152155
private final Map<ITestClass, String> classContainerUuidStorage = new ConcurrentHashMap<>();
156+
/**
157+
* Store uuid for data provider containers.
158+
*/
159+
private final Map<ITestNGMethod, String> dataProviderContainerUuidStorage = new ConcurrentHashMap<>();
153160
private final ReadWriteLock lock = new ReentrantReadWriteLock();
154161
private final AllureLifecycle lifecycle;
155162
private final AllureTestNgTestFilter testFilter;
@@ -280,6 +287,14 @@ public void onAfterClass(final ITestClass testClass) {
280287
getLifecycle().stopTestContainer(uuid);
281288
getLifecycle().writeTestContainer(uuid);
282289
});
290+
dataProviderContainerUuidStorage.entrySet().removeIf(entry -> {
291+
if (entry.getKey().getTestClass().equals(testClass)) {
292+
getLifecycle().stopTestContainer(entry.getValue());
293+
getLifecycle().writeTestContainer(entry.getValue());
294+
return true;
295+
}
296+
return false;
297+
});
283298
}
284299

285300
@Override
@@ -302,6 +317,10 @@ public void onTestStart(final ITestResult testResult) {
302317
.map(ITestResult::getMethod)
303318
.map(ITestNGMethod::getTestClass)
304319
.ifPresent(clazz -> addClassContainerChild(clazz, uuid));
320+
321+
Optional.of(testResult)
322+
.map(ITestResult::getMethod)
323+
.ifPresent(method -> addDataProviderContainerChild(method, uuid));
305324
}
306325

307326
@SuppressWarnings("BooleanExpressionComplexity")
@@ -626,6 +645,65 @@ public void onConfigurationSkip(final ITestResult itr) {
626645
//do nothing
627646
}
628647

648+
@Override
649+
public void beforeDataProviderExecution(final IDataProviderMethod dataProviderMethod,
650+
final ITestNGMethod method,
651+
final ITestContext iTestContext) {
652+
currentExecutable.remove();
653+
final String containerUuid = dataProviderContainerUuidStorage.computeIfAbsent(
654+
method,
655+
key -> {
656+
final String uuid = UUID.randomUUID().toString();
657+
getLifecycle().startTestContainer(
658+
new TestResultContainer()
659+
.setUuid(uuid)
660+
.setName(method.getMethodName())
661+
);
662+
return uuid;
663+
}
664+
);
665+
666+
final String uuid = currentExecutable.get();
667+
final FixtureResult result = new FixtureResult()
668+
.setName(dataProviderMethod.getMethod().getName())
669+
.setStage(Stage.RUNNING);
670+
671+
processDescription(
672+
getClass().getClassLoader(),
673+
dataProviderMethod.getMethod(),
674+
result::setDescription,
675+
result::setDescriptionHtml
676+
);
677+
678+
getLifecycle().startPrepareFixture(containerUuid, uuid, result);
679+
}
680+
681+
@Override
682+
public void afterDataProviderExecution(final IDataProviderMethod dataProviderMethod,
683+
final ITestNGMethod method,
684+
final ITestContext iTestContext) {
685+
final String uuid = currentExecutable.get();
686+
getLifecycle().updateFixture(uuid, result -> {
687+
if (result.getStatus() == null) {
688+
result.setStatus(Status.PASSED);
689+
}
690+
});
691+
getLifecycle().stopFixture(uuid);
692+
currentExecutable.remove();
693+
}
694+
695+
@Override
696+
public void onDataProviderFailure(final ITestNGMethod method,
697+
final ITestContext ctx,
698+
final RuntimeException t) {
699+
final String uuid = currentExecutable.get();
700+
getLifecycle().updateFixture(uuid, result -> result
701+
.setStatus(getStatus(t))
702+
.setStatusDetails(getStatusDetails(t).orElse(null)));
703+
getLifecycle().stopFixture(uuid);
704+
currentExecutable.remove();
705+
}
706+
629707
protected String getHistoryId(final ITestNGMethod method, final List<Parameter> parameters) {
630708
final MessageDigest digest = getMd5Digest();
631709
final String testClassName = method.getTestClass().getName();
@@ -827,6 +905,10 @@ private Consumer<TestResult> setStatus(final Status status, final StatusDetails
827905
};
828906
}
829907

908+
private void addDataProviderContainerChild(final ITestNGMethod method, final String childUuid) {
909+
this.addChildToContainer(dataProviderContainerUuidStorage.get(method), childUuid);
910+
}
911+
830912
private void addClassContainerChild(final ITestClass clazz, final String childUuid) {
831913
this.addChildToContainer(classContainerUuidStorage.get(clazz), childUuid);
832914
}

‎allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java‎

Lines changed: 135 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import io.qameta.allure.testng.samples.TestsWithIdForFilter;
4040
import org.assertj.core.api.Condition;
4141
import org.assertj.core.groups.Tuple;
42-
import org.testng.ITestNGListener;
4342
import org.testng.TestNG;
4443
import org.testng.annotations.DataProvider;
4544
import org.testng.annotations.Test;
@@ -130,7 +129,7 @@ public void shouldNotDisplayConfigurationFailsAsTests() {
130129
public void shouldSetConfigurationProperty() {
131130
AllureTestNgConfig allureTestNgConfig = AllureTestNgConfig.loadConfigProperties();
132131
allureTestNgConfig.setHideDisabledTests(true);
133-
assertThat(allureTestNgConfig.isHideDisabledTests()).isEqualTo(true);
132+
assertThat(allureTestNgConfig.isHideDisabledTests()).isTrue();
134133
}
135134

136135
@AllureFeatures.Parallel
@@ -507,7 +506,7 @@ public void multipleSuites() {
507506
.hasSize(3);
508507
List<String> uids = testResults.stream().map(TestResult::getUuid).collect(Collectors.toList());
509508
assertThat(testContainers).as("Unexpected quantity of testng containers has been written")
510-
.hasSize(8).extracting(TestResultContainer::getName)
509+
.hasSize(9).extracting(TestResultContainer::getName)
511510
.contains(beforeMethodName, beforeMethodName, firstTagName, firstSuiteName, secondTagName,
512511
secondSuiteName);
513512

@@ -553,7 +552,7 @@ public void parallelMethods() {
553552
assertThat(testResults).as("Unexpected quantity of testng case results has been written")
554553
.hasSize(2001);
555554
assertThat(testContainers).as("Unexpected quantity of testng containers has been written")
556-
.hasSize(6006);
555+
.hasSize(6007);
557556

558557
assertContainersPerMethod(before1, testContainers, uids);
559558
assertContainersPerMethod(before2, testContainers, uids);
@@ -1218,7 +1217,7 @@ public void shouldAddBeforeFixtureToFakeTestResult(final String suite, final Str
12181217
.findAny();
12191218
assertThat(result).as("Before failed fake test result").isNotEmpty();
12201219
final Optional<TestResultContainer> befores = results.getTestResultContainers().stream()
1221-
.filter(c -> Objects.nonNull(c.getBefores()) && c.getBefores().size() > 0)
1220+
.filter(c -> Objects.nonNull(c.getBefores()) && !c.getBefores().isEmpty())
12221221
.findAny();
12231222
assertThat(result).as("Before failed configuration container").isNotEmpty();
12241223
assertThat(befores.get().getChildren())
@@ -1252,7 +1251,6 @@ private AllureResults runTestNgSuites(AllureTestNgConfig config, final String...
12521251

12531252
private AllureResults runTestNgSuites(final Consumer<TestNG> configurer,
12541253
final String... suites) {
1255-
;
12561254
return runTestNgSuites(configurer, AllureTestNgConfig.loadConfigProperties(), suites);
12571255
}
12581256

@@ -1276,7 +1274,7 @@ private AllureResults runTestNgSuites(final Consumer<TestNG> configurer,
12761274
new AllureTestNgTestFilter(),
12771275
config);
12781276
final TestNG testNg = new TestNG(false);
1279-
testNg.addListener((ITestNGListener) adapter);
1277+
testNg.addListener(adapter);
12801278
testNg.setTestSuites(suiteFiles);
12811279

12821280
configurer.accept(testNg);
@@ -1461,7 +1459,7 @@ public void onlyId() {
14611459
@Test
14621460
@AllureFeatures.Filtration
14631461
public void idAssignToOtherTest() {
1464-
TestPlanV1_0 plan = new TestPlanV1_0().setTests(Arrays.asList(otherId));
1462+
TestPlanV1_0 plan = new TestPlanV1_0().setTests(singletonList(otherId));
14651463
List<TestResult> testResults = runTestPlan(plan, TestsWithIdForFilter.class).getTestResults();
14661464

14671465
assertThat(testResults)
@@ -1476,7 +1474,7 @@ public void idAssignToOtherTest() {
14761474
@Test
14771475
@AllureFeatures.Filtration
14781476
public void skippedTest() {
1479-
TestPlanV1_0 plan = new TestPlanV1_0().setTests(Arrays.asList(skipped));
1477+
TestPlanV1_0 plan = new TestPlanV1_0().setTests(singletonList(skipped));
14801478
List<TestResult> testResults = runTestPlan(plan, TestsWithIdForFilter.class).getTestResults();
14811479
assertThat(testResults)
14821480
.hasSize(1)
@@ -1508,13 +1506,140 @@ public AllureResults runTestPlan(final TestPlan plan, final Class<?>... testClas
15081506
return RunUtils.runTests(lifecycle -> {
15091507
final AllureTestNg adapter = new AllureTestNg(lifecycle, new AllureTestNgTestFilter(plan));
15101508
final TestNG testNG = new TestNG(false);
1511-
testNG.addListener((ITestNGListener) adapter);
1509+
testNG.addListener(adapter);
15121510
testNG.setTestClasses(testClasses);
15131511
testNG.setOutputDirectory("build/test-output");
15141512
testNG.run();
15151513
});
15161514
}
15171515

1516+
@AllureFeatures.Fixtures
1517+
@Test(description = "Should process data provider in setup")
1518+
public void shouldProcessDataProviderInSetup() {
1519+
final AllureResults results = runTestNgSuites("suites/data-provider-with-attachment.xml");
1520+
1521+
assertThat(results.getTestResultContainers())
1522+
.flatExtracting(TestResultContainer::getBefores)
1523+
.extracting(FixtureResult::getName, FixtureResult::getStatus)
1524+
.contains(Tuple.tuple("dataProvider", Status.PASSED));
1525+
1526+
assertThat(results.getTestResultContainers())
1527+
.flatExtracting(TestResultContainer::getBefores)
1528+
.filteredOn("name", "dataProvider")
1529+
.flatExtracting(FixtureResult::getAttachments)
1530+
.hasSize(1)
1531+
.extracting(Attachment::getName)
1532+
.contains("attachment");
1533+
}
1534+
1535+
@AllureFeatures.Fixtures
1536+
@Test(description = "Should process failed data provider in setup")
1537+
public void shouldProcessFailedDataProviderInSetup() {
1538+
final AllureResults results = runTestNgSuites("suites/failed-data-provider.xml");
1539+
1540+
assertThat(results.getTestResultContainers())
1541+
.flatExtracting(TestResultContainer::getBefores)
1542+
.extracting(FixtureResult::getName, FixtureResult::getStatus)
1543+
.contains(Tuple.tuple("dataProvider", Status.BROKEN));
1544+
}
1545+
1546+
@AllureFeatures.Fixtures
1547+
@Test(description = "Should process flaky data provider in setup")
1548+
public void shouldProcessFlakyDataProvider() {
1549+
final AllureResults results = runTestNgSuites("suites/flaky-data-provider.xml");
1550+
1551+
assertThat(results.getTestResultContainers())
1552+
.flatExtracting(TestResultContainer::getBefores)
1553+
.extracting(FixtureResult::getName, FixtureResult::getStatus)
1554+
.containsSubsequence(
1555+
Tuple.tuple("provide", Status.BROKEN),
1556+
Tuple.tuple("provide", Status.PASSED)
1557+
);
1558+
}
1559+
1560+
@AllureFeatures.Fixtures
1561+
@Test(description = "Should properly link data provider container to test result")
1562+
public void shouldProperlyLinkDataProviderContainerToTestResult() {
1563+
final AllureResults results = runTestNgSuites("suites/data-provider-with-attachment.xml");
1564+
1565+
final TestResult tr = findTestResultByName(results, "test");
1566+
final TestResultContainer dpContainer = results.getTestResultContainers().stream()
1567+
.filter(c -> c.getName().equals("test"))
1568+
.findFirst()
1569+
.orElseThrow(() -> new IllegalStateException("DP container not found"));
1570+
1571+
assertThat(dpContainer.getChildren())
1572+
.contains(tr.getUuid());
1573+
}
1574+
1575+
@AllureFeatures.Fixtures
1576+
@Test(description = "Should link multiple tests to data provider container")
1577+
public void shouldLinkMultipleTestsToDataProviderContainer() {
1578+
final AllureResults results = runTestNgSuites("suites/data-provider-multiple-tests.xml");
1579+
1580+
final List<TestResult> test1Results = results.getTestResults().stream()
1581+
.filter(tr -> tr.getName().equals("test1"))
1582+
.collect(Collectors.toList());
1583+
final List<TestResult> test2Results = results.getTestResults().stream()
1584+
.filter(tr -> tr.getName().equals("test2"))
1585+
.collect(Collectors.toList());
1586+
1587+
assertThat(test1Results).hasSize(2);
1588+
assertThat(test2Results).hasSize(2);
1589+
1590+
final TestResultContainer dpContainer1 = findTestContainerByName(results, "test1");
1591+
final TestResultContainer dpContainer2 = findTestContainerByName(results, "test2");
1592+
1593+
assertThat(dpContainer1.getChildren())
1594+
.containsAll(test1Results.stream().map(TestResult::getUuid).collect(Collectors.toList()));
1595+
assertThat(dpContainer2.getChildren())
1596+
.containsAll(test2Results.stream().map(TestResult::getUuid).collect(Collectors.toList()));
1597+
}
1598+
1599+
@AllureFeatures.Fixtures
1600+
@Test(description = "Should link inherited data provider")
1601+
public void shouldLinkInheritedDataProvider() {
1602+
final AllureResults results = runTestNgSuites("suites/data-provider-inheritance.xml");
1603+
1604+
final TestResult testBase = findTestResultByName(results, "testBase");
1605+
final TestResult testChild = findTestResultByName(results, "testChild");
1606+
1607+
final TestResultContainer dpContainerBase = findTestContainerByName(results, "testBase");
1608+
final TestResultContainer dpContainerChild = findTestContainerByName(results, "testChild");
1609+
1610+
assertThat(dpContainerBase.getChildren()).contains(testBase.getUuid());
1611+
assertThat(dpContainerChild.getChildren()).contains(testChild.getUuid());
1612+
}
1613+
1614+
@AllureFeatures.Fixtures
1615+
@Test(description = "Should link correct data provider in multiple classes")
1616+
public void shouldLinkCorrectDataProviderInMultipleClasses() {
1617+
final AllureResults results = runTestNgSuites("suites/data-provider-multiple-classes.xml");
1618+
1619+
final TestResult test1 = findTestResultByName(results, "test1");
1620+
final TestResult test2 = findTestResultByName(results, "test2");
1621+
1622+
final TestResultContainer dpContainer1 = findTestContainerByName(results, "test1");
1623+
final TestResultContainer dpContainer2 = findTestContainerByName(results, "test2");
1624+
1625+
assertThat(dpContainer1.getChildren())
1626+
.contains(test1.getUuid())
1627+
.doesNotContain(test2.getUuid());
1628+
assertThat(dpContainer2.getChildren())
1629+
.contains(test2.getUuid())
1630+
.doesNotContain(test1.getUuid());
1631+
}
1632+
1633+
@AllureFeatures.Fixtures
1634+
@Test(description = "Should process parallel data provider")
1635+
public void shouldProcessParallelDataProvider() {
1636+
final AllureResults results = runTestNgSuites("suites/data-provider-parallel.xml");
1637+
1638+
assertThat(results.getTestResults()).hasSize(4);
1639+
final TestResultContainer dpContainer = findTestContainerByName(results, "test");
1640+
assertThat(dpContainer.getChildren()).hasSize(4);
1641+
}
1642+
15181643
private Integer getOrderParameter(final TestResult result) {
15191644
return result.getParameters().stream()
15201645
.filter(p -> p.getName().equals("order"))

‎allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceBase.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright 2016-2024 Qameta Software Inc
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.qameta.allure.testng.samples;
17+
18+
import org.testng.annotations.DataProvider;
19+
import org.testng.annotations.Test;
20+
21+
public class DataProviderInheritanceBase {
22+
23+
@DataProvider
24+
public Object[][] dp() {
25+
return new Object[][]{{"data"}};
26+
}
27+
28+
@Test(dataProvider = "dp")
29+
public void testBase(String s) {
30+
}
31+
}

0 commit comments

Comments
 (0)