From 813f4b90b5e8a06865473af8bea5721900e1bfd6 Mon Sep 17 00:00:00 2001 From: andrejs Date: Fri, 18 Sep 2026 14:24:30 +0000 Subject: [PATCH] Report global errors on feature exist event --- .../io/qameta/allure/karate/AllureKarate.java | 52 ++++ .../karate/AllureKarateAdvancedTest.java | 1 + .../karate/AllureKarateGlobalErrorsTest.java | 222 ++++++++++++++++++ .../allure/karate/AllureKarateTest.java | 1 + .../after-failed-scenario.feature | 7 + .../after-feature-failure.feature | 10 + .../example-preparation-failure.feature | 7 + ...scenario-and-feature-hooks-failure.feature | 8 + .../setup-failure-target.feature | 4 + .../shared-hooks-failure.feature | 13 + .../shared-setup-failures.feature | 14 ++ .../suppressed-called-cleanup-failure.feature | 7 + .../suppressed-cleanup-failure.feature | 8 + .../suppressed-preparation-failure.feature | 8 + 14 files changed, 362 insertions(+) create mode 100644 allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java create mode 100644 allure-karate/src/test/resources/testdata/global-errors/after-failed-scenario.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/after-feature-failure.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/example-preparation-failure.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/scenario-and-feature-hooks-failure.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/setup-failure-target.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/shared-hooks-failure.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/shared-setup-failures.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/suppressed-called-cleanup-failure.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/suppressed-cleanup-failure.feature create mode 100644 allure-karate/src/test/resources/testdata/global-errors/suppressed-preparation-failure.feature diff --git a/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java b/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java index f554fc7f9..23e04c8f1 100644 --- a/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java +++ b/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java @@ -16,6 +16,8 @@ package io.qameta.allure.karate; import io.karatelabs.common.Json; +import io.karatelabs.core.FeatureRunEvent; +import io.karatelabs.core.FeatureRuntime; import io.karatelabs.core.HttpRunEvent; import io.karatelabs.core.RunEvent; import io.karatelabs.core.RunListener; @@ -67,6 +69,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static io.qameta.allure.util.ResultsUtils.createGlobalError; import static io.qameta.allure.util.ResultsUtils.createLabel; import static io.qameta.allure.util.ResultsUtils.createLink; import static io.qameta.allure.util.ResultsUtils.createParameter; @@ -94,6 +97,7 @@ public class AllureKarate implements RunListener { private final AllureLifecycle lifecycle; + private final Map> reportedScenarios = new ConcurrentHashMap<>(); private final Map scenarioContexts = new ConcurrentHashMap<>(); private final Map activeStepKeys = new ConcurrentHashMap<>(); private final Set redactedStepKeys = ConcurrentHashMap.newKeySet(); @@ -121,6 +125,9 @@ public AllureKarate(final AllureLifecycle lifecycle) { @Override public boolean onEvent(final RunEvent event) { switch (event.getType()) { + case FEATURE_EXIT: + afterFeature((FeatureRunEvent) event); + return true; case SCENARIO_ENTER: return beforeScenario(((ScenarioRunEvent) event).source()); case SCENARIO_EXIT: @@ -294,6 +301,13 @@ private void afterScenario(final ScenarioRunEvent event) { return; } + // Shared cleanup hooks run after SCENARIO_EXIT, so remember which steps are already reported. + maybeResult.ifPresent( + result -> reportedScenarios + .computeIfAbsent(sr.getFeatureRuntime(), key -> new ConcurrentHashMap<>()) + .put(result, new ReportedScenario(result.getStepResults().size(), redactFailure)) + ); + final List list = new ArrayList<>(); if (!context.reportDisabled() && event.result() != null @@ -315,6 +329,41 @@ private void afterScenario(final ScenarioRunEvent event) { lifecycle.writeTest(testKey); } + private void afterFeature(final FeatureRunEvent event) { + final Map reported = reportedScenarios.remove(event.source()); + if (event.source().isCalled() || Objects.isNull(event.result())) { + return; + } + + for (ScenarioResult result : event.result().getScenarioResults()) { + final ReportedScenario snapshot = Objects.isNull(reported) ? null : reported.get(result); + reportFeatureErrors(result, snapshot); + } + } + + private void reportFeatureErrors(final ScenarioResult result, final ReportedScenario snapshot) { + // Preparation errors may occur before any scenario emits lifecycle events. + final int reportedStepCount = Objects.isNull(snapshot) ? 0 : snapshot.stepCount(); + final boolean redact = result.isReportDisabled() + || getTagTexts(result.getScenario()).contains("report=false") + || Objects.nonNull(snapshot) && snapshot.redactFailure(); + final String context = "Karate feature " + getFeatureNameQualified(result.getScenario().getFeature()) + + " failed"; + final List steps = result.getStepResults(); + for (int i = reportedStepCount; i < steps.size(); i++) { + final StepResult step = steps.get(i); + // beforeScenario and afterScenario hooks belong to scenario results, not globals. + if (step.isFailed() && !step.isHook()) { + lifecycle.writeGlobals( + createGlobalError( + redact ? context + ": " + ScenarioResult.SUPPRESSED_FAILURE_MESSAGE : context, + redact ? null : step.getError() + ) + ); + } + } + } + private void finishCalledScenario(final ScenarioContext context, final boolean failed, final boolean redactFailure, @@ -696,6 +745,9 @@ private List getLinks(final List labels) { return allureLinks; } + private record ReportedScenario(int stepCount, boolean redactFailure) { + } + private record ScenarioContext( String testUuid, AllureExternalKey ownerKey, diff --git a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateAdvancedTest.java b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateAdvancedTest.java index 9a5edabca..f082443bc 100644 --- a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateAdvancedTest.java +++ b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateAdvancedTest.java @@ -93,6 +93,7 @@ void shouldRedactReportDisabledCalledFailureFromCaller() { .containsExactly(ScenarioResult.SUPPRESSED_FAILURE_MESSAGE, null); assertThat(call.getSteps()).isEmpty(); assertThat(results.getAttachments()).isEmpty(); + assertThat(results.getGlobals()).isEmpty(); } /** diff --git a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java new file mode 100644 index 000000000..47a9555c1 --- /dev/null +++ b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.karate; + +import io.karatelabs.core.ScenarioResult; +import io.qameta.allure.Description; +import io.qameta.allure.model.GlobalError; +import io.qameta.allure.model.TestResult; +import io.qameta.allure.test.AllureResults; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static io.qameta.allure.model.Status.BROKEN; +import static io.qameta.allure.model.Status.FAILED; +import static org.assertj.core.api.Assertions.assertThat; + +@SuppressWarnings({"MultipleStringLiterals", "PMD.AvoidDuplicateLiterals"}) +class AllureKarateGlobalErrorsTest extends TestRunner { + + /** + * A feature cleanup failure produces a global error identifying the feature, exception, and occurrence time. + */ + @Test + @Description + void shouldReportAfterFeatureFailureAsGlobalError() { + final long started = System.currentTimeMillis(); + final AllureResults results = run("classpath:testdata/global-errors/after-feature-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()).contains("after-feature-failure.feature", "feature cleanup failed"); + assertThat(error.getTrace()).contains("feature cleanup failed"); + assertThat(error.getTimestamp()).isBetween(started, System.currentTimeMillis()); + } + + /** + * A scenario failure does not prevent the feature's cleanup failure from being reported as a global error. + */ + @Test + @Description + void shouldReportFeatureCleanupAfterScenarioFailure() { + final AllureResults results = run("classpath:testdata/global-errors/after-failed-scenario.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()).contains("after-failed-scenario.feature", "cleanup after failure"); + assertThat(error.getTrace()).contains("cleanup after failure"); + } + + /** + * Concurrently executed features each report their own cleanup error once, with the correct feature identity. + */ + @Test + @Description + void shouldKeepParallelFeatureErrorsIsolated() { + final AllureResults results = run( + 4, + "classpath:testdata/global-errors/after-feature-failure.feature", + "classpath:testdata/global-errors/after-failed-scenario.feature" + ); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(2); + assertThat(errors).anySatisfy( + error -> assertThat(error.getMessage()).contains("after-feature-failure.feature", "feature cleanup failed") + ); + assertThat(errors).anySatisfy( + error -> assertThat(error.getMessage()).contains("after-failed-scenario.feature", "cleanup after failure") + ); + } + + /** + * Scenario-outline cleanup and feature cleanup failures are both reported as global errors. + */ + @Test + @Description + void shouldReportEachSharedHookFailure() { + final AllureResults results = run("classpath:testdata/global-errors/shared-hooks-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(2); + assertThat(errors).allSatisfy(error -> { + assertThat(error.getMessage()).contains("shared-hooks-failure.feature"); + assertThat(error.getTimestamp()).isPositive(); + }); + assertThat(errors).anySatisfy(error -> { + assertThat(error.getMessage()).contains("outline cleanup failed"); + assertThat(error.getTrace()).contains("outline cleanup failed"); + }); + assertThat(errors).anySatisfy(error -> { + assertThat(error.getMessage()).contains("feature cleanup failed"); + assertThat(error.getTrace()).contains("feature cleanup failed"); + }); + } + + /** + * Feature cleanup errors honor report suppression: safe feature identity remains, but exception details are hidden. + */ + @Test + @Description + void shouldRedactSuppressedFeatureCleanupFailure() { + final AllureResults results = run("classpath:testdata/global-errors/suppressed-cleanup-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()) + .contains("suppressed-cleanup-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE) + .doesNotContain("private-cleanup-value"); + assertThat(error.getTrace()).isNull(); + assertThat(error.getActual()).isNull(); + assertThat(error.getExpected()).isNull(); + assertThat(error.getTimestamp()).isPositive(); + } + + /** + * A visible caller with a suppressed callee failure keeps its global cleanup error details redacted. + */ + @Test + @Description + void shouldRedactFeatureCleanupAfterSuppressedCalledFailure() { + final AllureResults results = run("classpath:testdata/global-errors/suppressed-called-cleanup-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()) + .contains("suppressed-called-cleanup-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE) + .doesNotContain("private-caller-cleanup-value"); + assertThat(error.getTrace()).isNull(); + assertThat(error.getActual()).isNull(); + assertThat(error.getExpected()).isNull(); + } + + /** + * Example preparation failures produce a global error with the feature identity and original exception details. + */ + @Test + @Description + void shouldReportFeaturePreparationFailureAsGlobalError() { + final AllureResults results = run("classpath:testdata/global-errors/example-preparation-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()).contains("example-preparation-failure.feature", "example preparation failed"); + assertThat(error.getTrace()).contains("example preparation failed"); + assertThat(error.getTimestamp()).isPositive(); + } + + /** + * Report suppression also protects errors raised while preparing examples before scenario execution begins. + */ + @Test + @Description + void shouldRedactSuppressedFeaturePreparationFailure() { + final AllureResults results = run("classpath:testdata/global-errors/suppressed-preparation-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()) + .contains("suppressed-preparation-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE) + .doesNotContain("private-preparation-value"); + assertThat(error.getTrace()).isNull(); + assertThat(error.getActual()).isNull(); + assertThat(error.getExpected()).isNull(); + } + + /** + * Failures in callonce, callSingle, and setup scenarios remain scenario failures, not global errors. + */ + @Test + @Description + void shouldKeepSharedSetupFailuresOnScenarioResults() { + final AllureResults results = run("classpath:testdata/global-errors/shared-setup-failures.feature"); + + final TestResult callonce = results.getTestResultByName("Callonce setup failure"); + final TestResult callSingle = results.getTestResultByName("CallSingle setup failure"); + final TestResult setup = results.getTestResultByName("Prepare shared data"); + assertThat(List.of(callonce, callSingle, setup)).allSatisfy(result -> { + assertThat(result.getStatus()).isIn(FAILED, BROKEN); + assertThat(result.getStatusDetails().getMessage()).contains("shared setup failed"); + }); + assertThat(results.getGlobals()).isEmpty(); + } + + /** + * A feature cleanup error is global even when scenario cleanup also fails; per-scenario hooks are not global errors. + */ + @Test + @Description + void shouldReportOnlyFeatureHookFailuresAsGlobals() { + final AllureResults results = run("classpath:testdata/global-errors/scenario-and-feature-hooks-failure.feature"); + + final List errors = globalErrors(results); + assertThat(errors).hasSize(1); + final GlobalError error = errors.get(0); + assertThat(error.getMessage()).contains("scenario-and-feature-hooks-failure.feature", "feature cleanup failed"); + assertThat(error.getTrace()).contains("feature cleanup failed"); + } + + private List globalErrors(final AllureResults results) { + return results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream()).toList(); + } +} diff --git a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateTest.java b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateTest.java index 63f7668b2..ff7d8936a 100644 --- a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateTest.java +++ b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateTest.java @@ -134,6 +134,7 @@ void shouldCreateStatusDetailsIfTestFailed() { "java.lang.AssertionError: expected status: 200, actual: 401" ) ); + assertThat(results.getGlobals()).isEmpty(); } @Test diff --git a/allure-karate/src/test/resources/testdata/global-errors/after-failed-scenario.feature b/allure-karate/src/test/resources/testdata/global-errors/after-failed-scenario.feature new file mode 100644 index 000000000..1f4b2bd85 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/after-failed-scenario.feature @@ -0,0 +1,7 @@ +Feature: Cleanup after a failed scenario + +Background: + * configure afterFeature = function(){ throw new Error('cleanup after failure') } + +Scenario: Failing scenario + * match 1 == 2 diff --git a/allure-karate/src/test/resources/testdata/global-errors/after-feature-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/after-feature-failure.feature new file mode 100644 index 000000000..e0ef7c3a7 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/after-feature-failure.feature @@ -0,0 +1,10 @@ +Feature: Feature cleanup failure + +Background: + * configure afterFeature = function(){ throw new Error('feature cleanup failed') } + +Scenario: First scenario + * match 1 == 1 + +Scenario: Second scenario + * match 2 == 2 diff --git a/allure-karate/src/test/resources/testdata/global-errors/example-preparation-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/example-preparation-failure.feature new file mode 100644 index 000000000..71555ff60 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/example-preparation-failure.feature @@ -0,0 +1,7 @@ +Feature: Example preparation failure + +Scenario Outline: Prepared example + * match value == 1 + +Examples: + | (function(){ throw new Error('example preparation failed') })() | diff --git a/allure-karate/src/test/resources/testdata/global-errors/scenario-and-feature-hooks-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/scenario-and-feature-hooks-failure.feature new file mode 100644 index 000000000..c5f270221 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/scenario-and-feature-hooks-failure.feature @@ -0,0 +1,8 @@ +Feature: Scenario and feature cleanup failures + +Background: + * configure afterScenario = function(){ throw new Error('scenario cleanup failed') } + * configure afterFeature = function(){ throw new Error('feature cleanup failed') } + +Scenario: Scenario with both cleanup hooks + * match 1 == 1 diff --git a/allure-karate/src/test/resources/testdata/global-errors/setup-failure-target.feature b/allure-karate/src/test/resources/testdata/global-errors/setup-failure-target.feature new file mode 100644 index 000000000..d08e0abad --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/setup-failure-target.feature @@ -0,0 +1,4 @@ +Feature: Failing shared setup + +Scenario: Shared setup action + * karate.fail('shared setup failed') diff --git a/allure-karate/src/test/resources/testdata/global-errors/shared-hooks-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/shared-hooks-failure.feature new file mode 100644 index 000000000..10e0b799f --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/shared-hooks-failure.feature @@ -0,0 +1,13 @@ +Feature: Shared cleanup hooks + +Background: + * configure afterScenarioOutline = function(){ throw new Error('outline cleanup failed') } + * configure afterFeature = function(){ throw new Error('feature cleanup failed') } + +Scenario Outline: Example + * match value == '#number' + +Examples: + | value | + | 1 | + | 2 | diff --git a/allure-karate/src/test/resources/testdata/global-errors/shared-setup-failures.feature b/allure-karate/src/test/resources/testdata/global-errors/shared-setup-failures.feature new file mode 100644 index 000000000..8025de06c --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/shared-setup-failures.feature @@ -0,0 +1,14 @@ +Feature: Shared setup failures + +@setup +Scenario: Prepare shared data + * karate.fail('shared setup failed') + +Scenario: Callonce setup failure + * callonce read('classpath:testdata/global-errors/setup-failure-target.feature') + +Scenario: CallSingle setup failure + * def result = karate.callSingle('classpath:testdata/global-errors/setup-failure-target.feature') + +Scenario: Setup scenario failure + * def result = karate.setup() diff --git a/allure-karate/src/test/resources/testdata/global-errors/suppressed-called-cleanup-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/suppressed-called-cleanup-failure.feature new file mode 100644 index 000000000..cc594d7b5 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/suppressed-called-cleanup-failure.feature @@ -0,0 +1,7 @@ +Feature: Cleanup after a suppressed called failure + +Background: + * configure afterFeature = function(){ throw new Error('private-caller-cleanup-value') } + +Scenario: Caller with suppressed failure and cleanup + * call read('classpath:testdata/called-report-disabled.feature') diff --git a/allure-karate/src/test/resources/testdata/global-errors/suppressed-cleanup-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/suppressed-cleanup-failure.feature new file mode 100644 index 000000000..8c8cf0d7e --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/suppressed-cleanup-failure.feature @@ -0,0 +1,8 @@ +@report=false +Feature: Suppressed feature cleanup + +Background: + * configure afterFeature = function(){ throw new Error('private-cleanup-value') } + +Scenario: Suppressed cleanup failure + * match 1 == 1 diff --git a/allure-karate/src/test/resources/testdata/global-errors/suppressed-preparation-failure.feature b/allure-karate/src/test/resources/testdata/global-errors/suppressed-preparation-failure.feature new file mode 100644 index 000000000..502647401 --- /dev/null +++ b/allure-karate/src/test/resources/testdata/global-errors/suppressed-preparation-failure.feature @@ -0,0 +1,8 @@ +@report=false +Feature: Suppressed example preparation + +Scenario Outline: Suppressed prepared example + * match value == 1 + +Examples: + | (function(){ throw new Error('private-preparation-value') })() |