Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
package io.qameta.allure.karate;

import io.karatelabs.common.Json;
import io.karatelabs.core.FeatureRunEvent;
import io.karatelabs.core.FeatureRuntime;
import io.karatelabs.core.HttpRunEvent;
import io.karatelabs.core.RunEvent;
import io.karatelabs.core.RunListener;
Expand Down Expand Up @@ -67,6 +69,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static io.qameta.allure.util.ResultsUtils.createGlobalError;
import static io.qameta.allure.util.ResultsUtils.createLabel;
import static io.qameta.allure.util.ResultsUtils.createLink;
import static io.qameta.allure.util.ResultsUtils.createParameter;
Expand Down Expand Up @@ -94,6 +97,7 @@ public class AllureKarate implements RunListener {

private final AllureLifecycle lifecycle;

private final Map<FeatureRuntime, Map<ScenarioResult, ReportedScenario>> reportedScenarios = new ConcurrentHashMap<>();
private final Map<ScenarioRuntime, ScenarioContext> scenarioContexts = new ConcurrentHashMap<>();
private final Map<ScenarioRuntime, AllureExternalKey> activeStepKeys = new ConcurrentHashMap<>();
private final Set<AllureExternalKey> redactedStepKeys = ConcurrentHashMap.newKeySet();
Expand Down Expand Up @@ -121,6 +125,9 @@ public AllureKarate(final AllureLifecycle lifecycle) {
@Override
public boolean onEvent(final RunEvent event) {
switch (event.getType()) {
case FEATURE_EXIT:
afterFeature((FeatureRunEvent) event);
return true;
case SCENARIO_ENTER:
return beforeScenario(((ScenarioRunEvent) event).source());
case SCENARIO_EXIT:
Expand Down Expand Up @@ -294,6 +301,13 @@ private void afterScenario(final ScenarioRunEvent event) {
return;
}

// Shared cleanup hooks run after SCENARIO_EXIT, so remember which steps are already reported.
maybeResult.ifPresent(
result -> reportedScenarios
.computeIfAbsent(sr.getFeatureRuntime(), key -> new ConcurrentHashMap<>())
.put(result, new ReportedScenario(result.getStepResults().size(), redactFailure))
);

final List<Parameter> list = new ArrayList<>();
if (!context.reportDisabled()
&& event.result() != null
Expand All @@ -315,6 +329,41 @@ private void afterScenario(final ScenarioRunEvent event) {
lifecycle.writeTest(testKey);
}

private void afterFeature(final FeatureRunEvent event) {
final Map<ScenarioResult, ReportedScenario> reported = reportedScenarios.remove(event.source());
if (event.source().isCalled() || Objects.isNull(event.result())) {
return;
}

for (ScenarioResult result : event.result().getScenarioResults()) {
final ReportedScenario snapshot = Objects.isNull(reported) ? null : reported.get(result);
reportFeatureErrors(result, snapshot);
}
}

private void reportFeatureErrors(final ScenarioResult result, final ReportedScenario snapshot) {
// Preparation errors may occur before any scenario emits lifecycle events.
final int reportedStepCount = Objects.isNull(snapshot) ? 0 : snapshot.stepCount();
final boolean redact = result.isReportDisabled()
|| getTagTexts(result.getScenario()).contains("report=false")
|| Objects.nonNull(snapshot) && snapshot.redactFailure();
final String context = "Karate feature " + getFeatureNameQualified(result.getScenario().getFeature())
+ " failed";
final List<StepResult> steps = result.getStepResults();
for (int i = reportedStepCount; i < steps.size(); i++) {
final StepResult step = steps.get(i);
// beforeScenario and afterScenario hooks belong to scenario results, not globals.
if (step.isFailed() && !step.isHook()) {
lifecycle.writeGlobals(
createGlobalError(
redact ? context + ": " + ScenarioResult.SUPPRESSED_FAILURE_MESSAGE : context,
redact ? null : step.getError()
)
);
}
}
}

private void finishCalledScenario(final ScenarioContext context,
final boolean failed,
final boolean redactFailure,
Expand Down Expand Up @@ -696,6 +745,9 @@ private List<Link> getLinks(final List<String> labels) {
return allureLinks;
}

private record ReportedScenario(int stepCount, boolean redactFailure) {
}

private record ScenarioContext(
String testUuid,
AllureExternalKey ownerKey,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ void shouldRedactReportDisabledCalledFailureFromCaller() {
.containsExactly(ScenarioResult.SUPPRESSED_FAILURE_MESSAGE, null);
assertThat(call.getSteps()).isEmpty();
assertThat(results.getAttachments()).isEmpty();
assertThat(results.getGlobals()).isEmpty();
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
/*
* Copyright 2016-2026 Qameta Software Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.qameta.allure.karate;

import io.karatelabs.core.ScenarioResult;
import io.qameta.allure.Description;
import io.qameta.allure.model.GlobalError;
import io.qameta.allure.model.TestResult;
import io.qameta.allure.test.AllureResults;
import org.junit.jupiter.api.Test;

import java.util.List;

import static io.qameta.allure.model.Status.BROKEN;
import static io.qameta.allure.model.Status.FAILED;
import static org.assertj.core.api.Assertions.assertThat;

@SuppressWarnings({"MultipleStringLiterals", "PMD.AvoidDuplicateLiterals"})
class AllureKarateGlobalErrorsTest extends TestRunner {

/**
* A feature cleanup failure produces a global error identifying the feature, exception, and occurrence time.
*/
@Test
@Description
void shouldReportAfterFeatureFailureAsGlobalError() {
final long started = System.currentTimeMillis();
final AllureResults results = run("classpath:testdata/global-errors/after-feature-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage()).contains("after-feature-failure.feature", "feature cleanup failed");
assertThat(error.getTrace()).contains("feature cleanup failed");
assertThat(error.getTimestamp()).isBetween(started, System.currentTimeMillis());
}

/**
* A scenario failure does not prevent the feature's cleanup failure from being reported as a global error.
*/
@Test
@Description
void shouldReportFeatureCleanupAfterScenarioFailure() {
final AllureResults results = run("classpath:testdata/global-errors/after-failed-scenario.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage()).contains("after-failed-scenario.feature", "cleanup after failure");
assertThat(error.getTrace()).contains("cleanup after failure");
}

/**
* Concurrently executed features each report their own cleanup error once, with the correct feature identity.
*/
@Test
@Description
void shouldKeepParallelFeatureErrorsIsolated() {
final AllureResults results = run(
4,
"classpath:testdata/global-errors/after-feature-failure.feature",
"classpath:testdata/global-errors/after-failed-scenario.feature"
);

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(2);
assertThat(errors).anySatisfy(
error -> assertThat(error.getMessage()).contains("after-feature-failure.feature", "feature cleanup failed")
);
assertThat(errors).anySatisfy(
error -> assertThat(error.getMessage()).contains("after-failed-scenario.feature", "cleanup after failure")
);
}

/**
* Scenario-outline cleanup and feature cleanup failures are both reported as global errors.
*/
@Test
@Description
void shouldReportEachSharedHookFailure() {
final AllureResults results = run("classpath:testdata/global-errors/shared-hooks-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(2);
assertThat(errors).allSatisfy(error -> {
assertThat(error.getMessage()).contains("shared-hooks-failure.feature");
assertThat(error.getTimestamp()).isPositive();
});
assertThat(errors).anySatisfy(error -> {
assertThat(error.getMessage()).contains("outline cleanup failed");
assertThat(error.getTrace()).contains("outline cleanup failed");
});
assertThat(errors).anySatisfy(error -> {
assertThat(error.getMessage()).contains("feature cleanup failed");
assertThat(error.getTrace()).contains("feature cleanup failed");
});
}

/**
* Feature cleanup errors honor report suppression: safe feature identity remains, but exception details are hidden.
*/
@Test
@Description
void shouldRedactSuppressedFeatureCleanupFailure() {
final AllureResults results = run("classpath:testdata/global-errors/suppressed-cleanup-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage())
.contains("suppressed-cleanup-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE)
.doesNotContain("private-cleanup-value");
assertThat(error.getTrace()).isNull();
assertThat(error.getActual()).isNull();
assertThat(error.getExpected()).isNull();
assertThat(error.getTimestamp()).isPositive();
}

/**
* A visible caller with a suppressed callee failure keeps its global cleanup error details redacted.
*/
@Test
@Description
void shouldRedactFeatureCleanupAfterSuppressedCalledFailure() {
final AllureResults results = run("classpath:testdata/global-errors/suppressed-called-cleanup-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage())
.contains("suppressed-called-cleanup-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE)
.doesNotContain("private-caller-cleanup-value");
assertThat(error.getTrace()).isNull();
assertThat(error.getActual()).isNull();
assertThat(error.getExpected()).isNull();
}

/**
* Example preparation failures produce a global error with the feature identity and original exception details.
*/
@Test
@Description
void shouldReportFeaturePreparationFailureAsGlobalError() {
final AllureResults results = run("classpath:testdata/global-errors/example-preparation-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage()).contains("example-preparation-failure.feature", "example preparation failed");
assertThat(error.getTrace()).contains("example preparation failed");
assertThat(error.getTimestamp()).isPositive();
}

/**
* Report suppression also protects errors raised while preparing examples before scenario execution begins.
*/
@Test
@Description
void shouldRedactSuppressedFeaturePreparationFailure() {
final AllureResults results = run("classpath:testdata/global-errors/suppressed-preparation-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage())
.contains("suppressed-preparation-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE)
.doesNotContain("private-preparation-value");
assertThat(error.getTrace()).isNull();
assertThat(error.getActual()).isNull();
assertThat(error.getExpected()).isNull();
}

/**
* Failures in callonce, callSingle, and setup scenarios remain scenario failures, not global errors.
*/
@Test
@Description
void shouldKeepSharedSetupFailuresOnScenarioResults() {
final AllureResults results = run("classpath:testdata/global-errors/shared-setup-failures.feature");

final TestResult callonce = results.getTestResultByName("Callonce setup failure");
final TestResult callSingle = results.getTestResultByName("CallSingle setup failure");
final TestResult setup = results.getTestResultByName("Prepare shared data");
assertThat(List.of(callonce, callSingle, setup)).allSatisfy(result -> {
assertThat(result.getStatus()).isIn(FAILED, BROKEN);
assertThat(result.getStatusDetails().getMessage()).contains("shared setup failed");
});
assertThat(results.getGlobals()).isEmpty();
}

/**
* A feature cleanup error is global even when scenario cleanup also fails; per-scenario hooks are not global errors.
*/
@Test
@Description
void shouldReportOnlyFeatureHookFailuresAsGlobals() {
final AllureResults results = run("classpath:testdata/global-errors/scenario-and-feature-hooks-failure.feature");

final List<GlobalError> errors = globalErrors(results);
assertThat(errors).hasSize(1);
final GlobalError error = errors.get(0);
assertThat(error.getMessage()).contains("scenario-and-feature-hooks-failure.feature", "feature cleanup failed");
assertThat(error.getTrace()).contains("feature cleanup failed");
}

private List<GlobalError> globalErrors(final AllureResults results) {
return results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream()).toList();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ void shouldCreateStatusDetailsIfTestFailed() {
"java.lang.AssertionError: expected status: 200, actual: 401"
)
);
assertThat(results.getGlobals()).isEmpty();
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Feature: Cleanup after a failed scenario

Background:
* configure afterFeature = function(){ throw new Error('cleanup after failure') }

Scenario: Failing scenario
* match 1 == 2
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Feature: Feature cleanup failure

Background:
* configure afterFeature = function(){ throw new Error('feature cleanup failed') }

Scenario: First scenario
* match 1 == 1

Scenario: Second scenario
* match 2 == 2
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Feature: Example preparation failure

Scenario Outline: Prepared example
* match value == 1

Examples:
| (function(){ throw new Error('example preparation failed') })() |
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Feature: Scenario and feature cleanup failures

Background:
* configure afterScenario = function(){ throw new Error('scenario cleanup failed') }
* configure afterFeature = function(){ throw new Error('feature cleanup failed') }

Scenario: Scenario with both cleanup hooks
* match 1 == 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Feature: Failing shared setup

Scenario: Shared setup action
* karate.fail('shared setup failed')
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
Feature: Shared cleanup hooks

Background:
* configure afterScenarioOutline = function(){ throw new Error('outline cleanup failed') }
* configure afterFeature = function(){ throw new Error('feature cleanup failed') }

Scenario Outline: Example <value>
* match value == '#number'

Examples:
| value |
| 1 |
| 2 |
Loading
Loading