diff --git a/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java b/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java index 90a9a39c9..52d8d07b9 100644 --- a/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java +++ b/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java @@ -59,6 +59,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -149,6 +150,10 @@ public class AllureJunitPlatform implements TestExecutionListener { private final ThreadLocal testPlanStorage = new InheritableThreadLocal<>(); + // unique ids of everything already reported, so a failing container only reports the tests that + // do not have a result yet + private final ThreadLocal> reportedTestsStorage = new InheritableThreadLocal<>(); + private final AllureLifecycle lifecycle; /** @@ -273,6 +278,7 @@ private static boolean isClassAvailableOnClasspath(final String clazz) { @Override public void testPlanExecutionStarted(final TestPlan testPlan) { testPlanStorage.set(testPlan); + reportedTestsStorage.set(ConcurrentHashMap.newKeySet()); } /** @@ -281,6 +287,7 @@ public void testPlanExecutionStarted(final TestPlan testPlan) { @Override public void testPlanExecutionFinished(final TestPlan testPlan) { testPlanStorage.remove(); + reportedTestsStorage.remove(); } /** @@ -319,9 +326,15 @@ public void executionFinished(final TestIdentifier testIdentifier, if (testIdentifier.isTest()) { stopTest(testIdentifier, status, statusDetails); } else if (testExecutionResult.getStatus() != TestExecutionResult.Status.SUCCESSFUL) { - // report failed containers as fake test results, linked to their own scope only - startTest(testIdentifier, Collections.singletonList(scopeKey(testIdentifier.getUniqueId()))); - stopTest(testIdentifier, status, statusDetails); + // only a container that failed before its tests ran leaves them without results. A broken + // @AfterAll blocks nothing, and a skip is never retried, so both keep the fake test result. + final boolean blockedTestsReported = testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED + && reportBlockedTests(testIdentifier, status, statusDetails); + if (!blockedTestsReported) { + // report failed containers as fake test results, linked to their own scope only + startTest(testIdentifier, Collections.singletonList(scopeKey(testIdentifier.getUniqueId()))); + stopTest(testIdentifier, status, statusDetails); + } } getLifecycle().writeScope(scopeKey(testIdentifier.getUniqueId())); } @@ -465,6 +478,50 @@ private void processParameterEvent(final Map keyValuePairs) { ); } + /** + * Reports the tests a failing container blocked, one result each. Identifying them by method rather than by + * class gives them the same history ids as a passing retry, so the retry replaces them (see issue #1155). + * + * @return true if at least one blocked test was reported + */ + private boolean reportBlockedTests(final TestIdentifier container, + final Status status, + final StatusDetails statusDetails) { + final TestPlan testPlan = testPlanStorage.get(); + final Set reportedTests = reportedTestsStorage.get(); + if (Objects.isNull(testPlan) || Objects.isNull(reportedTests)) { + return false; + } + final List blocked = getTests(testPlan, container) + .filter(test -> !reportedTests.contains(test.getUniqueId())) + .filter(test -> !shouldSkipReportingFor(test)) + .toList(); + if (blocked.isEmpty()) { + return false; + } + final List scopeKeys = Collections.singletonList(scopeKey(container.getUniqueId())); + blocked.forEach(test -> { + startTest(test, scopeKeys); + stopTest(test, status, statusDetails); + }); + return true; + } + + /** + * Collects the leaves under the given node, counting a childless container as a leaf — the same rule as + * {@link #reportNested}. A test template that never ran has no invocations, so ignoring it would drop the + * method from the report. + */ + private Stream getTests(final TestPlan testPlan, + final TestIdentifier parent) { + return testPlan.getChildren(parent).stream() + .flatMap( + child -> child.isTest() || testPlan.getChildren(child).isEmpty() + ? Stream.of(child) + : getTests(testPlan, child) + ); + } + private void reportNested(final TestPlan testPlan, final TestIdentifier testIdentifier, final Status status, @@ -554,6 +611,10 @@ private static boolean isInvocationSegment(final UniqueId.Segment segment) { private void startTest(final TestIdentifier testIdentifier, final List scopeKeys) { + final Set reportedTests = reportedTestsStorage.get(); + if (Objects.nonNull(reportedTests)) { + reportedTests.add(testIdentifier.getUniqueId()); + } final Optional testSource = testIdentifier.getSource(); final Optional testMethod = testSource .flatMap(AllureJunitPlatformUtils::getTestMethod); diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java index a030ea8c3..7ea1f75b0 100644 --- a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java @@ -33,6 +33,7 @@ import io.qameta.allure.junitplatform.features.KarateTests; import io.qameta.allure.junitplatform.features.MarkerAnnotationSupport; import io.qameta.allure.junitplatform.features.MetaAnnotationTest; +import io.qameta.allure.junitplatform.features.NestedBrokenInBeforeAllTests; import io.qameta.allure.junitplatform.features.NestedDisplayNameTests; import io.qameta.allure.junitplatform.features.NestedTests; import io.qameta.allure.junitplatform.features.OneTest; @@ -46,6 +47,7 @@ import io.qameta.allure.junitplatform.features.RepeatedTestsWithDisplayName; import io.qameta.allure.junitplatform.features.RepeatedTestsWithLongDisplayName; import io.qameta.allure.junitplatform.features.ReportEntryParameter; +import io.qameta.allure.junitplatform.features.RetryBeforeAllTests; import io.qameta.allure.junitplatform.features.RuntimeParametersTest; import io.qameta.allure.junitplatform.features.RuntimeSuiteLabelTest; import io.qameta.allure.junitplatform.features.RuntimeSystemLabelsTest; @@ -258,7 +260,10 @@ void shouldProcessBrokenInBeforeAllTests() { tr -> Optional.of(tr).map(TestResult::getStatusDetails).map(StatusDetails::getMessage).orElse(null) ) .containsExactlyInAnyOrder( - tuple("BrokenInBeforeAllTests", Status.BROKEN, "Exception in @BeforeAll") + tuple("test1()", Status.BROKEN, "Exception in @BeforeAll"), + tuple("test2()", Status.BROKEN, "Exception in @BeforeAll"), + // never ran, so it has no invocations: one result for the method, not one per value + tuple("parameterisedTest(String)", Status.BROKEN, "Exception in @BeforeAll") ); } @@ -281,7 +286,9 @@ void shouldProcessBrokenInAfterAllTests() { tuple("parameterisedTest(String) [2] value = \"b\"", Status.PASSED, null), tuple("parameterisedTest(String) [3] value = \"c\"", Status.PASSED, null), tuple("test1()", Status.PASSED, null), - tuple("test2()", Status.PASSED, null) + tuple("test2()", Status.PASSED, null), + // reported once as skipped, not a second time as broken + tuple("disabledTest()", Status.SKIPPED, "disabled on purpose") ); } @@ -439,6 +446,52 @@ void shouldUseTemplateIdAndHiddenInvocationIdForHistory() { .doesNotHaveDuplicates(); } + @Test + @AllureFeatures.History + void shouldGiveTestsBlockedByBrokenBeforeAllTheirOwnHistoryId() { + final AllureResults failedRun; + final AllureResults retriedRun; + try { + RetryBeforeAllTests.beforeAllShouldFail = true; + failedRun = runClasses(RetryBeforeAllTests.class); + RetryBeforeAllTests.beforeAllShouldFail = false; + retriedRun = runClasses(RetryBeforeAllTests.class); + } finally { + RetryBeforeAllTests.beforeAllShouldFail = false; + } + + assertThat(failedRun.getTestResults()) + .extracting(TestResult::getName, TestResult::getStatus) + .containsExactlyInAnyOrder( + tuple("test1()", Status.BROKEN), + tuple("test2()", Status.BROKEN), + tuple("test3()", Status.BROKEN) + ); + + // the ids must match, or the retry cannot replace the failures and they stay in the report forever + assertThat(failedRun.getTestResults()) + .extracting(TestResult::getHistoryId) + .containsExactlyInAnyOrderElementsOf( + retriedRun.getTestResults().stream().map(TestResult::getHistoryId).toList() + ); + } + + @Test + @AllureFeatures.BrokenTests + void shouldReportTestsBlockedByNestedContainerOnlyOnce() { + final AllureResults results = runClasses(NestedBrokenInBeforeAllTests.class); + + // the inner class reports its blocked tests first, so the outer one must not report them again + assertThat(results.getTestResults()) + .extracting(TestResult::getName, TestResult::getStatus) + .containsExactlyInAnyOrder( + tuple("outerTest()", Status.PASSED), + tuple("innerTest1()", Status.BROKEN), + tuple("innerTest2()", Status.BROKEN), + tuple("NestedBrokenInBeforeAllTests", Status.BROKEN) + ); + } + @Test @AllureFeatures.Steps void shouldAddSteps() { @@ -1520,16 +1573,17 @@ void shouldLinkTestsToTheirScopes() { @Test @AllureFeatures.Fixtures - void shouldLinkFailedContainerFakeTestToItsScope() { + void shouldLinkBlockedTestsToFailedContainerScope() { final AllureResults results = runClasses(BrokenInBeforeAllTests.class); final List testResults = results.getTestResults(); assertThat(testResults) - .hasSize(1); + .hasSize(3); - final String uuid = testResults.get(0).getUuid(); - assertThat(results.getTestResultContainers()) - .filteredOn(container -> container.getChildren().contains(uuid)) - .hasSize(1); + testResults.forEach( + testResult -> assertThat(results.getTestResultContainers()) + .filteredOn(container -> container.getChildren().contains(testResult.getUuid())) + .hasSize(1) + ); } } diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/BrokenInAfterAllTests.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/BrokenInAfterAllTests.java index 422d8e921..eef8a940a 100644 --- a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/BrokenInAfterAllTests.java +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/BrokenInAfterAllTests.java @@ -16,6 +16,7 @@ package io.qameta.allure.junitplatform.features; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -31,6 +32,12 @@ static void exception() { void test1() { } + // a skipped test already has its own result, so the broken @AfterAll must not report it again + @Disabled("disabled on purpose") + @Test + void disabledTest() { + } + @Test void test2() { } diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/NestedBrokenInBeforeAllTests.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/NestedBrokenInBeforeAllTests.java new file mode 100644 index 000000000..c5c048f4e --- /dev/null +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/NestedBrokenInBeforeAllTests.java @@ -0,0 +1,50 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junitplatform.features; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +public class NestedBrokenInBeforeAllTests { + + @AfterAll + static void outerAfterAll() { + throw new RuntimeException("Exception in outer @AfterAll"); + } + + @Test + void outerTest() { + } + + @Nested + class Inner { + + @BeforeAll + static void innerBeforeAll() { + throw new RuntimeException("Exception in inner @BeforeAll"); + } + + @Test + void innerTest1() { + } + + @Test + void innerTest2() { + } + } +} diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/RetryBeforeAllTests.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/RetryBeforeAllTests.java new file mode 100644 index 000000000..28a02e517 --- /dev/null +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/RetryBeforeAllTests.java @@ -0,0 +1,50 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junitplatform.features; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +/** + * The class from issue #1155: {@code @BeforeAll} fails on the first attempt and succeeds on the retry. + * It has to be one class with a toggle rather than two fixtures, because only the same class produces + * the same history ids. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class RetryBeforeAllTests { + + public static boolean beforeAllShouldFail = true; + + @BeforeAll + void beforeAll() { + if (beforeAllShouldFail) { + throw new RuntimeException("Simulated failure in @BeforeAll"); + } + } + + @Test + void test1() { + } + + @Test + void test2() { + } + + @Test + void test3() { + } +}