Report tests blocked by a broken @BeforeAll as separate results (fixes #1155)

allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java

@@ -59,6 +59,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -149,6 +150,10 @@ public class AllureJunitPlatform implements TestExecutionListener {
 
     private final ThreadLocal<TestPlan> testPlanStorage = new InheritableThreadLocal<>();
 
+    // unique ids of everything already reported, so a failing container only reports the tests that
+    // do not have a result yet
+    private final ThreadLocal<Set<String>> reportedTestsStorage = new InheritableThreadLocal<>();
+
     private final AllureLifecycle lifecycle;
 
     /**
@@ -273,6 +278,7 @@ private static boolean isClassAvailableOnClasspath(final String clazz) {
     @Override
     public void testPlanExecutionStarted(final TestPlan testPlan) {
         testPlanStorage.set(testPlan);
+        reportedTestsStorage.set(ConcurrentHashMap.newKeySet());
     }
 
     /**
@@ -281,6 +287,7 @@ public void testPlanExecutionStarted(final TestPlan testPlan) {
     @Override
     public void testPlanExecutionFinished(final TestPlan testPlan) {
         testPlanStorage.remove();
+        reportedTestsStorage.remove();
     }
 
     /**
@@ -319,9 +326,15 @@ public void executionFinished(final TestIdentifier testIdentifier,
         if (testIdentifier.isTest()) {
             stopTest(testIdentifier, status, statusDetails);
         } else if (testExecutionResult.getStatus() != TestExecutionResult.Status.SUCCESSFUL) {
-            // report failed containers as fake test results, linked to their own scope only
-            startTest(testIdentifier, Collections.singletonList(scopeKey(testIdentifier.getUniqueId())));
-            stopTest(testIdentifier, status, statusDetails);
+            // only a container that failed before its tests ran leaves them without results. A broken
+            // @AfterAll blocks nothing, and a skip is never retried, so both keep the fake test result.
+            final boolean blockedTestsReported = testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED
+                    && reportBlockedTests(testIdentifier, status, statusDetails);
+            if (!blockedTestsReported) {
+                // report failed containers as fake test results, linked to their own scope only
+                startTest(testIdentifier, Collections.singletonList(scopeKey(testIdentifier.getUniqueId())));
+                stopTest(testIdentifier, status, statusDetails);
+            }
         }
         getLifecycle().writeScope(scopeKey(testIdentifier.getUniqueId()));
     }
@@ -465,6 +478,50 @@ private void processParameterEvent(final Map<String, String> keyValuePairs) {
         );
     }
 
+    /**
+     * Reports the tests a failing container blocked, one result each. Identifying them by method rather than by
+     * class gives them the same history ids as a passing retry, so the retry replaces them (see issue #1155).
+     *
+     * @return true if at least one blocked test was reported
+     */
+    private boolean reportBlockedTests(final TestIdentifier container,
+                                       final Status status,
+                                       final StatusDetails statusDetails) {
+        final TestPlan testPlan = testPlanStorage.get();
+        final Set<String> reportedTests = reportedTestsStorage.get();
+        if (Objects.isNull(testPlan) || Objects.isNull(reportedTests)) {
+            return false;
+        }
+        final List<TestIdentifier> blocked = getTests(testPlan, container)
+                .filter(test -> !reportedTests.contains(test.getUniqueId()))
+                .filter(test -> !shouldSkipReportingFor(test))
+                .toList();
+        if (blocked.isEmpty()) {
+            return false;
+        }
+        final List<AllureExternalKey> scopeKeys = Collections.singletonList(scopeKey(container.getUniqueId()));
+        blocked.forEach(test -> {
+            startTest(test, scopeKeys);
+            stopTest(test, status, statusDetails);
+        });
+        return true;
+    }
+
+    /**
+     * Collects the leaves under the given node, counting a childless container as a leaf — the same rule as
+     * {@link #reportNested}. A test template that never ran has no invocations, so ignoring it would drop the
+     * method from the report.
+     */
+    private Stream<TestIdentifier> getTests(final TestPlan testPlan,
+                                            final TestIdentifier parent) {
+        return testPlan.getChildren(parent).stream()
+                .flatMap(
+                        child -> child.isTest() || testPlan.getChildren(child).isEmpty()
+                                ? Stream.of(child)
+                                : getTests(testPlan, child)
+                );
+    }
+
     private void reportNested(final TestPlan testPlan,
                               final TestIdentifier testIdentifier,
                               final Status status,
@@ -554,6 +611,10 @@ private static boolean isInvocationSegment(final UniqueId.Segment segment) {
 
     private void startTest(final TestIdentifier testIdentifier,
                            final List<AllureExternalKey> scopeKeys) {
+        final Set<String> reportedTests = reportedTestsStorage.get();
+        if (Objects.nonNull(reportedTests)) {
+            reportedTests.add(testIdentifier.getUniqueId());
+        }
         final Optional<TestSource> testSource = testIdentifier.getSource();
         final Optional<Method> testMethod = testSource
                 .flatMap(AllureJunitPlatformUtils::getTestMethod);

allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java

@@ -33,6 +33,7 @@
 import io.qameta.allure.junitplatform.features.KarateTests;
 import io.qameta.allure.junitplatform.features.MarkerAnnotationSupport;
 import io.qameta.allure.junitplatform.features.MetaAnnotationTest;
+import io.qameta.allure.junitplatform.features.NestedBrokenInBeforeAllTests;
 import io.qameta.allure.junitplatform.features.NestedDisplayNameTests;
 import io.qameta.allure.junitplatform.features.NestedTests;
 import io.qameta.allure.junitplatform.features.OneTest;
@@ -46,6 +47,7 @@
 import io.qameta.allure.junitplatform.features.RepeatedTestsWithDisplayName;
 import io.qameta.allure.junitplatform.features.RepeatedTestsWithLongDisplayName;
 import io.qameta.allure.junitplatform.features.ReportEntryParameter;
+import io.qameta.allure.junitplatform.features.RetryBeforeAllTests;
 import io.qameta.allure.junitplatform.features.RuntimeParametersTest;
 import io.qameta.allure.junitplatform.features.RuntimeSuiteLabelTest;
 import io.qameta.allure.junitplatform.features.RuntimeSystemLabelsTest;
@@ -258,7 +260,10 @@ void shouldProcessBrokenInBeforeAllTests() {
                         tr -> Optional.of(tr).map(TestResult::getStatusDetails).map(StatusDetails::getMessage).orElse(null)
                 )
                 .containsExactlyInAnyOrder(
-                        tuple("BrokenInBeforeAllTests", Status.BROKEN, "Exception in @BeforeAll")
+                        tuple("test1()", Status.BROKEN, "Exception in @BeforeAll"),
+                        tuple("test2()", Status.BROKEN, "Exception in @BeforeAll"),
+                        // never ran, so it has no invocations: one result for the method, not one per value
+                        tuple("parameterisedTest(String)", Status.BROKEN, "Exception in @BeforeAll")
                 );
     }
 
@@ -281,7 +286,9 @@ void shouldProcessBrokenInAfterAllTests() {
                         tuple("parameterisedTest(String) [2] value = \"b\"", Status.PASSED, null),
                         tuple("parameterisedTest(String) [3] value = \"c\"", Status.PASSED, null),
                         tuple("test1()", Status.PASSED, null),
-                        tuple("test2()", Status.PASSED, null)
+                        tuple("test2()", Status.PASSED, null),
+                        // reported once as skipped, not a second time as broken
+                        tuple("disabledTest()", Status.SKIPPED, "disabled on purpose")
                 );
     }
 
@@ -439,6 +446,52 @@ void shouldUseTemplateIdAndHiddenInvocationIdForHistory() {
                 .doesNotHaveDuplicates();
     }
 
+    @Test
+    @AllureFeatures.History
+    void shouldGiveTestsBlockedByBrokenBeforeAllTheirOwnHistoryId() {
+        final AllureResults failedRun;
+        final AllureResults retriedRun;
+        try {
+            RetryBeforeAllTests.beforeAllShouldFail = true;
+            failedRun = runClasses(RetryBeforeAllTests.class);
+            RetryBeforeAllTests.beforeAllShouldFail = false;
+            retriedRun = runClasses(RetryBeforeAllTests.class);
+        } finally {
+            RetryBeforeAllTests.beforeAllShouldFail = false;
+        }
+
+        assertThat(failedRun.getTestResults())
+                .extracting(TestResult::getName, TestResult::getStatus)
+                .containsExactlyInAnyOrder(
+                        tuple("test1()", Status.BROKEN),
+                        tuple("test2()", Status.BROKEN),
+                        tuple("test3()", Status.BROKEN)
+                );
+
+        // the ids must match, or the retry cannot replace the failures and they stay in the report forever
+        assertThat(failedRun.getTestResults())
+                .extracting(TestResult::getHistoryId)
+                .containsExactlyInAnyOrderElementsOf(
+                        retriedRun.getTestResults().stream().map(TestResult::getHistoryId).toList()
+                );
+    }
+
+    @Test
+    @AllureFeatures.BrokenTests
+    void shouldReportTestsBlockedByNestedContainerOnlyOnce() {
+        final AllureResults results = runClasses(NestedBrokenInBeforeAllTests.class);
+
+        // the inner class reports its blocked tests first, so the outer one must not report them again
+        assertThat(results.getTestResults())
+                .extracting(TestResult::getName, TestResult::getStatus)
+                .containsExactlyInAnyOrder(
+                        tuple("outerTest()", Status.PASSED),
+                        tuple("innerTest1()", Status.BROKEN),
+                        tuple("innerTest2()", Status.BROKEN),
+                        tuple("NestedBrokenInBeforeAllTests", Status.BROKEN)
+                );
+    }
+
     @Test
     @AllureFeatures.Steps
     void shouldAddSteps() {
@@ -1520,16 +1573,17 @@ void shouldLinkTestsToTheirScopes() {
 
     @Test
     @AllureFeatures.Fixtures
-    void shouldLinkFailedContainerFakeTestToItsScope() {
+    void shouldLinkBlockedTestsToFailedContainerScope() {
         final AllureResults results = runClasses(BrokenInBeforeAllTests.class);
 
         final List<TestResult> testResults = results.getTestResults();
         assertThat(testResults)
-                .hasSize(1);
+                .hasSize(3);
 
-        final String uuid = testResults.get(0).getUuid();
-        assertThat(results.getTestResultContainers())
-                .filteredOn(container -> container.getChildren().contains(uuid))
-                .hasSize(1);
+        testResults.forEach(
+                testResult -> assertThat(results.getTestResultContainers())
+                        .filteredOn(container -> container.getChildren().contains(testResult.getUuid()))
+                        .hasSize(1)
+        );
     }
 }

allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/BrokenInAfterAllTests.java

@@ -16,6 +16,7 @@
 package io.qameta.allure.junitplatform.features;
 
 import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -31,6 +32,12 @@ static void exception() {
     void test1() {
     }
 
+    // a skipped test already has its own result, so the broken @AfterAll must not report it again
+    @Disabled("disabled on purpose")
+    @Test
+    void disabledTest() {
+    }
+
     @Test
     void test2() {
     }

allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/NestedBrokenInBeforeAllTests.java

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2016-2026 Qameta Software Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package io.qameta.allure.junitplatform.features;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+public class NestedBrokenInBeforeAllTests {
+
+    @AfterAll
+    static void outerAfterAll() {
+        throw new RuntimeException("Exception in outer @AfterAll");
+    }
+
+    @Test
+    void outerTest() {
+    }
+
+    @Nested
+    class Inner {
+
+        @BeforeAll
+        static void innerBeforeAll() {
+            throw new RuntimeException("Exception in inner @BeforeAll");
+        }
+
+        @Test
+        void innerTest1() {
+        }
+
+        @Test
+        void innerTest2() {
+        }
+    }
+}

allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/RetryBeforeAllTests.java

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2016-2026 Qameta Software Inc
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package io.qameta.allure.junitplatform.features;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+
+/**
+ * The class from issue #1155: {@code @BeforeAll} fails on the first attempt and succeeds on the retry.
+ * It has to be one class with a toggle rather than two fixtures, because only the same class produces
+ * the same history ids.
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class RetryBeforeAllTests {
+
+    public static boolean beforeAllShouldFail = true;
+
+    @BeforeAll
+    void beforeAll() {
+        if (beforeAllShouldFail) {
+            throw new RuntimeException("Simulated failure in @BeforeAll");
+        }
+    }
+
+    @Test
+    void test1() {
+    }
+
+    @Test
+    void test2() {
+    }
+
+    @Test
+    void test3() {
+    }
+}