Skip to content

Commit a36cd7c

Browse files
HardNorthHardNorth
authored
Merge pull request #24 from reportportal/develop
Release
2 parents 7e20fd8 + 6afb492 commit a36cd7c

13 files changed

Lines changed: 361 additions & 5 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ hs_err_pid*
2626
.gradle/
2727
build/
2828
test-output/
29+
bin/
2930

3031
# IntelliJ Idea files
3132
.idea/

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Changelog
22
## [Unreleased]
3+
### Added
4+
- TestNG retries handling, by @HardNorth
5+
### Changed
6+
- Client version updated to [5.4.8](https://github.com/reportportal/client-java/releases/tag/5.4.8), by @HardNorth
37

48
## [5.4.3]
59
### Changed

‎build.gradle‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,14 @@ repositories {
4040
}
4141

4242
dependencies {
43-
api 'com.epam.reportportal:client-java:5.4.7'
43+
api 'com.epam.reportportal:client-java:5.4.8'
4444

4545
implementation "io.cucumber:cucumber-gherkin:${project.cucumber_version}"
4646
implementation 'org.slf4j:slf4j-api:2.0.7'
4747
implementation 'org.apache.commons:commons-lang3:3.19.0'
4848

49+
compileOnly "io.cucumber:cucumber-testng:${project.cucumber_version}"
50+
4951
testImplementation 'com.squareup.okhttp3:okhttp:4.12.0'
5052
testImplementation "io.cucumber:cucumber-java:${project.cucumber_version}"
5153
testImplementation 'com.epam.reportportal:agent-java-test-utils:0.1.0'

‎src/main/java/com/epam/reportportal/cucumber/ScenarioReporter.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.epam.reportportal.cucumber;
1818

19+
import com.epam.reportportal.cucumber.testng.TestNgRetriesListener;
1920
import com.epam.reportportal.cucumber.util.HookSuite;
2021
import com.epam.reportportal.listeners.ItemStatus;
2122
import com.epam.reportportal.listeners.ItemType;
@@ -996,6 +997,15 @@ protected void beforeScenario(@Nonnull TestCase scenario) {
996997

997998
// If it's a ScenarioOutline use Example's line number as code reference to detach one Test Item from another
998999
StartTestItemRQ startTestItemRQ = buildStartScenarioRequest(scenario);
1000+
boolean testNgRetry = TestNgRetriesListener.isRetry(
1001+
scenario.getUri() + KEY_VALUE_SEPARATOR + scenario.getLocation().getLine());
1002+
if (testNgRetry) {
1003+
String retryOf = s.getId().blockingGet("");
1004+
if (!retryOf.isEmpty()) {
1005+
startTestItemRQ.setRetry(true);
1006+
startTestItemRQ.setRetryOf(retryOf);
1007+
}
1008+
}
9991009
s.setId(startScenario(rootId, startTestItemRQ));
10001010
descriptionsMap.put(s.getId(), ofNullable(startTestItemRQ.getDescription()).orElse(StringUtils.EMPTY));
10011011
if (getLaunch().getParameters().isCallbackReportingEnabled()) {

‎src/main/java/com/epam/reportportal/cucumber/Utils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public class Utils {
4444

4545
private static final String EMPTY = "";
4646
public static final String TAG_KEY = "@";
47-
private static final String KEY_VALUE_SEPARATOR = ":";
47+
public static final String KEY_VALUE_SEPARATOR = ":";
4848

4949
private Utils() {
5050
throw new AssertionError("No instances should exist for the class!");

‎src/main/java/com/epam/reportportal/cucumber/testng/TestNgRetriesListener.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright 2026 EPAM Systems
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.epam.reportportal.cucumber.testng;
18+
19+
import com.epam.reportportal.cucumber.Utils;
20+
import org.testng.IExecutionListener;
21+
import org.testng.ITestListener;
22+
import org.testng.ITestResult;
23+
24+
import java.util.Map;
25+
import java.util.concurrent.ConcurrentHashMap;
26+
27+
import static java.util.Optional.ofNullable;
28+
29+
/**
30+
* The listener is supposed to be used with TestNG Cucumber runner and retry listeners to allow
31+
* {@link com.epam.reportportal.cucumber.ScenarioReporter} determine if a Scenario is a retry.
32+
*/
33+
public class TestNgRetriesListener implements IExecutionListener, ITestListener {
34+
private static final Map<String, Boolean> RETRIES = new ConcurrentHashMap<>();
35+
private static final Boolean ENABLED;
36+
37+
static {
38+
boolean hasTestNG = false;
39+
try {
40+
Class.forName("io.cucumber.testng.PickleWrapper");
41+
hasTestNG = true;
42+
} catch (ClassNotFoundException ignore) {
43+
}
44+
ENABLED = hasTestNG;
45+
}
46+
47+
@Override
48+
public void onExecutionFinish() {
49+
if (!ENABLED) {
50+
return;
51+
}
52+
RETRIES.clear();
53+
}
54+
55+
private static void setRetryFlag(ITestResult result) {
56+
if (!ENABLED) {
57+
return;
58+
}
59+
ofNullable(result.getParameters()).map(params -> (io.cucumber.testng.PickleWrapper) params[0])
60+
.map(io.cucumber.testng.PickleWrapper::getPickle)
61+
.map(pickle -> pickle.getUri().toString() + Utils.KEY_VALUE_SEPARATOR + pickle.getLine())
62+
.ifPresent(uniqueId -> RETRIES.put(uniqueId, result.wasRetried()));
63+
}
64+
65+
@Override
66+
public void onTestSuccess(ITestResult result) {
67+
setRetryFlag(result);
68+
}
69+
70+
@Override
71+
public void onTestFailure(ITestResult result) {
72+
setRetryFlag(result);
73+
}
74+
75+
@Override
76+
public void onTestSkipped(ITestResult result) {
77+
setRetryFlag(result);
78+
}
79+
80+
@Override
81+
public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
82+
setRetryFlag(result);
83+
}
84+
85+
public static boolean isRetry(String id) {
86+
return RETRIES.getOrDefault(id, false);
87+
}
88+
}

‎src/main/resources/META-INF/services/org.testng.ITestNGListener‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#
2+
# Copyright 2026 EPAM Systems
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
com.epam.reportportal.cucumber.testng.TestNgRetriesListener

‎src/test/java/com/epam/reportportal/cucumber/ParameterTest.java‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636

3737
import java.util.Arrays;
3838
import java.util.List;
39+
import java.util.Map;
3940
import java.util.concurrent.ExecutorService;
4041
import java.util.concurrent.Executors;
4142
import java.util.stream.Collectors;
@@ -57,6 +58,13 @@ public static class OneSimpleAndOneScenarioOutlineScenarioReporterTest extends A
5758

5859
}
5960

61+
@CucumberOptions(features = "src/test/resources/features/BasicScenarioOutlineParameters.feature", glue = {
62+
"com.epam.reportportal.cucumber.integration.feature" }, plugin = {
63+
"com.epam.reportportal.cucumber.integration.TestScenarioReporter" })
64+
public static class RunScenarioOutlineParametersTest extends AbstractTestNGCucumberTests {
65+
66+
}
67+
6068
@CucumberOptions(features = "src/test/resources/features/DocStringParameters.feature", glue = {
6169
"com.epam.reportportal.cucumber.integration.feature" }, plugin = {
6270
"com.epam.reportportal.cucumber.integration.TestScenarioReporter" })
@@ -77,7 +85,7 @@ public static class DataTableParameterTest extends AbstractTestNGCucumberTests {
7785

7886
private final String launchId = CommonUtils.namedId("launch_");
7987
private final String suiteId = CommonUtils.namedId("feature_");
80-
private final List<String> testIds = Stream.generate(() -> CommonUtils.namedId("scenario_")).limit(2).collect(Collectors.toList());
88+
private final List<String> testIds = Stream.generate(() -> CommonUtils.namedId("scenario_")).limit(3).collect(Collectors.toList());
8189
private final List<Pair<String, List<String>>> stepIds = testIds.stream()
8290
.map(id -> Pair.of(id, Stream.generate(() -> CommonUtils.namedId("scenario_")).limit(3).collect(Collectors.toList())))
8391
.collect(Collectors.toList());
@@ -120,6 +128,33 @@ public void verify_agent_creates_correct_step_names() {
120128
});
121129
}
122130

131+
List<Map<String, String>> EXPECTED_PARAMETERS = Arrays.asList(
132+
Map.of("str", "\"first\"", "parameters", "123"),
133+
Map.of("str", "\"second\"", "parameters", "12345"),
134+
Map.of("str", "\"third\"", "parameters", "12345678")
135+
);
136+
137+
@Test
138+
public void verify_agent_correctly_reports_parameters() {
139+
TestUtils.runTests(RunScenarioOutlineParametersTest.class);
140+
141+
verify(client, times(1)).startTestItem(any());
142+
ArgumentCaptor<StartTestItemRQ> testCaptor = ArgumentCaptor.forClass(StartTestItemRQ.class);
143+
verify(client, times(3)).startTestItem(same(suiteId), testCaptor.capture());
144+
verify(client, times(3)).startTestItem(same(testIds.get(0)), any());
145+
verify(client, times(3)).startTestItem(same(testIds.get(1)), any());
146+
verify(client, times(3)).startTestItem(same(testIds.get(2)), any());
147+
148+
List<StartTestItemRQ> items = testCaptor.getAllValues();
149+
IntStream.range(0, items.size()).forEach(i -> {
150+
StartTestItemRQ test = items.get(i);
151+
assertThat(
152+
test.getParameters().stream().collect(Collectors.toMap(ParameterResource::getKey, ParameterResource::getValue)),
153+
equalTo(EXPECTED_PARAMETERS.get(i))
154+
);
155+
});
156+
}
157+
123158
@Test
124159
@SuppressWarnings("unchecked")
125160
public void verify_agent_reports_docstring_parameter() {

‎src/test/java/com/epam/reportportal/cucumber/TestNgRetryDetectionTest.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* Copyright 2026 EPAM Systems
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.epam.reportportal.cucumber;
18+
19+
import com.epam.reportportal.cucumber.integration.TestScenarioReporter;
20+
import com.epam.reportportal.cucumber.integration.feature.RetrySteps;
21+
import com.epam.reportportal.cucumber.integration.testng.RetryAnalyzer;
22+
import com.epam.reportportal.cucumber.integration.util.TestUtils;
23+
import com.epam.reportportal.listeners.ListenerParameters;
24+
import com.epam.reportportal.service.ReportPortal;
25+
import com.epam.reportportal.service.ReportPortalClient;
26+
import com.epam.reportportal.util.test.CommonUtils;
27+
import com.epam.ta.reportportal.ws.model.StartTestItemRQ;
28+
import io.cucumber.testng.AbstractTestNGCucumberTests;
29+
import io.cucumber.testng.CucumberOptions;
30+
import org.apache.commons.lang3.tuple.Pair;
31+
import org.junit.jupiter.api.AfterEach;
32+
import org.junit.jupiter.api.BeforeEach;
33+
import org.junit.jupiter.api.Test;
34+
import org.mockito.ArgumentCaptor;
35+
import org.testng.IAnnotationTransformer;
36+
import org.testng.annotations.ITestAnnotation;
37+
38+
import java.lang.reflect.Constructor;
39+
import java.lang.reflect.Method;
40+
import java.util.List;
41+
import java.util.concurrent.ExecutorService;
42+
import java.util.concurrent.Executors;
43+
import java.util.stream.Collectors;
44+
import java.util.stream.IntStream;
45+
import java.util.stream.Stream;
46+
47+
import static org.hamcrest.MatcherAssert.assertThat;
48+
import static org.hamcrest.Matchers.*;
49+
import static org.mockito.ArgumentMatchers.same;
50+
import static org.mockito.Mockito.*;
51+
52+
public class TestNgRetryDetectionTest {
53+
public static class AnnotationTransformer implements IAnnotationTransformer {
54+
@Override
55+
public void transform(ITestAnnotation annotation, Class testClass, Constructor constructor, Method method) {
56+
annotation.setRetryAnalyzer(RetryAnalyzer.class);
57+
}
58+
}
59+
60+
@CucumberOptions(features = "src/test/resources/features/TestNgRetry.feature", glue = {
61+
"com.epam.reportportal.cucumber.integration.feature" }, plugin = {
62+
"com.epam.reportportal.cucumber.integration.TestScenarioReporter" })
63+
public static class RetryCucumberTest extends AbstractTestNGCucumberTests {
64+
}
65+
66+
private final String launchId = CommonUtils.namedId("launch_");
67+
private final String suiteId = CommonUtils.namedId("feature_");
68+
private final List<String> testIds = Stream.generate(() -> CommonUtils.namedId("scenario_")).limit(2).collect(Collectors.toList());
69+
private final List<String> stepIds = Stream.generate(() -> CommonUtils.namedId("step_")).limit(4).collect(Collectors.toList());
70+
private final List<Pair<String, List<String>>> steps = IntStream.range(0, testIds.size())
71+
.mapToObj(i -> Pair.of(testIds.get(i), stepIds.subList(i * 2, i * 2 + 2)))
72+
.collect(Collectors.toList());
73+
74+
private final ListenerParameters params = TestUtils.standardParameters();
75+
private final ReportPortalClient client = mock(ReportPortalClient.class);
76+
private final ExecutorService executorService = Executors.newSingleThreadExecutor();
77+
private final ReportPortal reportPortal = ReportPortal.create(client, params, executorService);
78+
79+
@BeforeEach
80+
public void setup() {
81+
TestUtils.mockLaunch(client, launchId, suiteId, steps);
82+
TestUtils.mockLogging(client);
83+
TestScenarioReporter.RP.set(reportPortal);
84+
RetrySteps.reset();
85+
}
86+
87+
@AfterEach
88+
public void tearDown() {
89+
CommonUtils.shutdownExecutorService(executorService);
90+
}
91+
92+
@Test
93+
public void verify_retry_flags_and_links() {
94+
TestUtils.runTestsWithListener(AnnotationTransformer.class, RetryCucumberTest.class);
95+
96+
ArgumentCaptor<StartTestItemRQ> startScenarioCaptor = ArgumentCaptor.forClass(StartTestItemRQ.class);
97+
verify(client, times(2)).startTestItem(same(suiteId), startScenarioCaptor.capture());
98+
99+
List<StartTestItemRQ> scenarioStarts = startScenarioCaptor.getAllValues();
100+
assertThat(scenarioStarts, hasSize(2));
101+
102+
StartTestItemRQ firstAttempt = scenarioStarts.get(0);
103+
assertThat(firstAttempt.isRetry(), nullValue());
104+
assertThat(firstAttempt.getRetryOf(), nullValue());
105+
106+
StartTestItemRQ secondAttempt = scenarioStarts.get(1);
107+
assertThat(secondAttempt.isRetry(), equalTo(true));
108+
assertThat(secondAttempt.getRetryOf(), equalTo(testIds.get(0)));
109+
}
110+
}

‎src/test/java/com/epam/reportportal/cucumber/integration/feature/RetrySteps.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2026 EPAM Systems
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.epam.reportportal.cucumber.integration.feature;
18+
19+
import io.cucumber.java.en.Given;
20+
import io.cucumber.java.en.Then;
21+
22+
import java.util.concurrent.atomic.AtomicInteger;
23+
24+
public class RetrySteps {
25+
private static final AtomicInteger ATTEMPTS = new AtomicInteger();
26+
27+
public static void reset() {
28+
ATTEMPTS.set(0);
29+
}
30+
31+
@Given("I fail {int} times")
32+
public void i_fail_times(int times) {
33+
int attempt = ATTEMPTS.incrementAndGet();
34+
if (attempt <= times) {
35+
throw new IllegalStateException("Flaky failure on attempt " + attempt);
36+
}
37+
}
38+
39+
@Then("I pass")
40+
public void i_pass() {
41+
// Step intentionally left blank.
42+
}
43+
}

0 commit comments

Comments
 (0)