Skip to content

Support composed flaky, muted, and severity annotations across Java test adapters #1331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
baev merged 1 commit into from
Jul 22, 2026
Merged

Support composed flaky, muted, and severity annotations across Java test adapters #1331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 89 additions & 4 deletions allure-java-commons/src/main/java/io/qameta/allure/util/AnnotationUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import io.qameta.allure.LabelAnnotation;
import io.qameta.allure.LinkAnnotation;
import io.qameta.allure.Muted;
import io.qameta.allure.Severity;
import io.qameta.allure.SeverityLevel;
import io.qameta.allure.model.Label;
import io.qameta.allure.model.Link;
import org.slf4j.Logger;
Expand All @@ -32,6 +34,7 @@
import java.util.Collection;
import java.util.HashSet;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
Expand All @@ -46,6 +49,7 @@
* test cases via reflection.
*
*/
@SuppressWarnings("PMD.GodClass")
public final class AnnotationUtils {

private static final Logger LOGGER = LoggerFactory.getLogger(AnnotationUtils.class);
Expand All @@ -57,23 +61,77 @@ private AnnotationUtils() {
}

/**
* Returns true if {@link io.qameta.allure.Flaky} annotation is present.
* Returns true if {@link io.qameta.allure.Flaky} annotation is present. The annotation is
* also discovered when used as a meta annotation on other (custom) annotations.
*
* @param annotatedElement the element to search annotations on.
* @return true if {@link io.qameta.allure.Flaky} annotation is present, false otherwise.
*/
public static boolean isFlaky(final AnnotatedElement annotatedElement) {
return annotatedElement.isAnnotationPresent(Flaky.class);
return isFlaky(asList(annotatedElement.getAnnotations()));
}

/**
* Returns true if {@link io.qameta.allure.Muted} annotation is present.
* Returns true if {@link io.qameta.allure.Flaky} annotation is present within given annotations,
* either directly or as a meta annotation.
*
* @param annotations annotations to analyse.
* @return true if {@link io.qameta.allure.Flaky} annotation is present, false otherwise.
*/
public static boolean isFlaky(final Collection<Annotation> annotations) {
return findMetaAnnotations(Flaky.class, annotations).findAny().isPresent();
}

/**
* Returns true if {@link io.qameta.allure.Muted} annotation is present. The annotation is
* also discovered when used as a meta annotation on other (custom) annotations.
*
* @param annotatedElement the element to search annotations on.
* @return true if {@link io.qameta.allure.Muted} annotation is present, false otherwise.
*/
public static boolean isMuted(final AnnotatedElement annotatedElement) {
return annotatedElement.isAnnotationPresent(Muted.class);
return isMuted(asList(annotatedElement.getAnnotations()));
}

/**
* Returns true if {@link io.qameta.allure.Muted} annotation is present within given annotations,
* either directly or as a meta annotation.
*
* @param annotations annotations to analyse.
* @return true if {@link io.qameta.allure.Muted} annotation is present, false otherwise.
*/
public static boolean isMuted(final Collection<Annotation> annotations) {
return findMetaAnnotations(Muted.class, annotations).findAny().isPresent();
}

/**
* Returns the severity specified by the {@link io.qameta.allure.Severity} annotation. The
* annotation is also discovered when used as a meta annotation on other (custom) annotations.
*
* @param annotatedElement the element to search annotations on.
* @return the discovered severity, if any.
*/
public static Optional<SeverityLevel> getSeverity(final AnnotatedElement annotatedElement) {
return getSeverity(asList(annotatedElement.getAnnotations()));
}

/**
* Returns the severity specified by the {@link io.qameta.allure.Severity} annotation within
* given annotations. A directly declared {@link io.qameta.allure.Severity} takes precedence;
* otherwise the annotation is also discovered when used as a meta annotation on other (custom)
* annotations.
*
* @param annotations annotations to analyse.
* @return the discovered severity, if any.
*/
public static Optional<SeverityLevel> getSeverity(final Collection<Annotation> annotations) {
return annotations.stream()
.filter(Severity.class::isInstance)
.map(Severity.class::cast)
.findFirst()
.map(Optional::of)
.orElseGet(() -> findMetaAnnotations(Severity.class, annotations).findFirst())
.map(Severity::value);
}

/**
Expand Down Expand Up @@ -169,6 +227,33 @@ private static <T, U extends Annotation> Stream<T> extractMetaAnnotations(
return Stream.empty();
}

// Unlike AnnotatedElement#getAnnotationsByType, discovers annotations of the given type even
// when they are used as a meta annotation on another (custom) annotation.
private static <T extends Annotation> Stream<T> findMetaAnnotations(
final Class<T> annotationType,
final Collection<Annotation> candidates) {
final Set<Annotation> visited = new HashSet<>();
return candidates.stream()
.flatMap(AnnotationUtils::extractRepeatable)
.flatMap(candidate -> findMetaAnnotations(annotationType, candidate, visited));
}

private static <T extends Annotation> Stream<T> findMetaAnnotations(
final Class<T> annotationType,
final Annotation candidate,
final Set<Annotation> visited) {
if (isInJavaLangAnnotationPackage(candidate.annotationType()) || !visited.add(candidate)) {
return Stream.empty();
}
final Stream<T> current = annotationType.equals(candidate.annotationType())
? Stream.of(annotationType.cast(candidate))
: Stream.empty();
final Stream<T> children = Stream.of(candidate.annotationType().getAnnotations())
.flatMap(AnnotationUtils::extractRepeatable)
.flatMap(annotation -> findMetaAnnotations(annotationType, annotation, visited));
return Stream.concat(current, children);
}

private static Stream<Label> extractLabels(final LabelAnnotation m, final Annotation annotation) {
if (Objects.equals(m.value(), LabelAnnotation.DEFAULT_VALUE)) {
return callValueMethod(annotation)
Expand Down
100 changes: 100 additions & 0 deletions allure-java-commons/src/test/java/io/qameta/allure/util/AnnotationUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,17 @@
import io.qameta.allure.Epic;
import io.qameta.allure.Epics;
import io.qameta.allure.Feature;
import io.qameta.allure.Flaky;
import io.qameta.allure.Issue;
import io.qameta.allure.Issues;
import io.qameta.allure.LabelAnnotation;
import io.qameta.allure.Link;
import io.qameta.allure.LinkAnnotation;
import io.qameta.allure.Links;
import io.qameta.allure.Muted;
import io.qameta.allure.Owner;
import io.qameta.allure.Severity;
import io.qameta.allure.SeverityLevel;
import io.qameta.allure.Story;
import io.qameta.allure.TmsLink;
import io.qameta.allure.TmsLinks;
Expand All @@ -42,6 +46,9 @@
import static io.qameta.allure.Allure.step;
import static io.qameta.allure.util.AnnotationUtils.getLabels;
import static io.qameta.allure.util.AnnotationUtils.getLinks;
import static io.qameta.allure.util.AnnotationUtils.getSeverity;
import static io.qameta.allure.util.AnnotationUtils.isFlaky;
import static io.qameta.allure.util.AnnotationUtils.isMuted;
import static io.qameta.allure.util.ResultsUtils.EPIC_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.FEATURE_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.STORY_LABEL_NAME;
Expand Down Expand Up @@ -497,6 +504,99 @@ void inheritedLinkAnnotationsTest() {
);
}

@Flaky
@Muted
@Severity(SeverityLevel.MINOR)
static class WithDirectStatusAnnotations {
}

static class WithoutStatusAnnotations {
}

@Test
void shouldDetectDirectFlakyMutedSeverity() {
assertThat(isFlaky(WithDirectStatusAnnotations.class)).isTrue();
assertThat(isMuted(WithDirectStatusAnnotations.class)).isTrue();
assertThat(getSeverity(WithDirectStatusAnnotations.class)).contains(SeverityLevel.MINOR);
}

@Test
void shouldReturnDefaultsWhenStatusAnnotationsAbsent() {
assertThat(isFlaky(WithoutStatusAnnotations.class)).isFalse();
assertThat(isMuted(WithoutStatusAnnotations.class)).isFalse();
assertThat(getSeverity(WithoutStatusAnnotations.class)).isEmpty();
}

@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
@Flaky
@Muted
@Severity(SeverityLevel.CRITICAL)
@interface CriticalUnstable {
}

@CriticalUnstable
static class WithComposedStatusAnnotation {
}

@Test
void shouldDetectFlakyMutedSeverityViaMetaAnnotation() {
assertThat(isFlaky(WithComposedStatusAnnotation.class)).isTrue();
assertThat(isMuted(WithComposedStatusAnnotation.class)).isTrue();
assertThat(getSeverity(WithComposedStatusAnnotation.class)).contains(SeverityLevel.CRITICAL);
}

@CriticalUnstable
@Severity(SeverityLevel.MINOR)
static class ComposedBeforeDirectSeverity {
}

@Severity(SeverityLevel.MINOR)
@CriticalUnstable
static class DirectBeforeComposedSeverity {
}

@Test
void directSeverityShouldTakePrecedenceOverMetaAnnotation() {
// regardless of declaration order, the directly declared @Severity(MINOR) must win over
// the CRITICAL severity contributed by the @CriticalUnstable meta annotation
assertThat(getSeverity(ComposedBeforeDirectSeverity.class)).contains(SeverityLevel.MINOR);
assertThat(getSeverity(DirectBeforeComposedSeverity.class)).contains(SeverityLevel.MINOR);
}

@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
@CriticalUnstable
@interface NestedUnstable {
}

@NestedUnstable
static class WithNestedComposedStatusAnnotation {
}

@Test
void shouldDetectFlakyMutedSeverityViaNestedMetaAnnotation() {
assertThat(isFlaky(WithNestedComposedStatusAnnotation.class)).isTrue();
assertThat(isMuted(WithNestedComposedStatusAnnotation.class)).isTrue();
assertThat(getSeverity(WithNestedComposedStatusAnnotation.class)).contains(SeverityLevel.CRITICAL);
}

@Flaky
@Muted
@Severity(SeverityLevel.BLOCKER)
static class UnstableParent {
}

static class UnstableChild extends UnstableParent {
}

@Test
void shouldInheritStatusAnnotationsFromSuperclass() {
assertThat(isFlaky(UnstableChild.class)).isTrue();
assertThat(isMuted(UnstableChild.class)).isTrue();
assertThat(getSeverity(UnstableChild.class)).contains(SeverityLevel.BLOCKER);
}

private static Set<Label> getLabelsFor(final Class<?> annotatedClass) {
return step(
"Extract labels from annotated class",
Expand Down
22 changes: 12 additions & 10 deletions allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
import io.qameta.allure.AllureLifecycle;
import io.qameta.allure.AttachmentOptions;
import io.qameta.allure.Description;
import io.qameta.allure.Severity;
import io.qameta.allure.SeverityLevel;
import io.qameta.allure.model.Label;
import io.qameta.allure.model.Parameter;
import io.qameta.allure.model.Status;
Expand Down Expand Up @@ -610,12 +608,22 @@ private void startTest(final TestIdentifier testIdentifier,

result.setDescription(description);

testMethod.map(this::getSeverity)
testMethod.map(AnnotationUtils::getSeverity)
.filter(Optional::isPresent)
.orElse(testClass.flatMap(this::getSeverity))
.orElse(testClass.flatMap(AnnotationUtils::getSeverity))
.map(ResultsUtils::createSeverityLabel)
.ifPresent(result.getLabels()::add);

final boolean flaky = testMethod.map(AnnotationUtils::isFlaky).orElse(false)
|| testClass.map(AnnotationUtils::isFlaky).orElse(false);
final boolean muted = testMethod.map(AnnotationUtils::isMuted).orElse(false)
|| testClass.map(AnnotationUtils::isMuted).orElse(false);
result.setStatusDetails(
new StatusDetails()
.setFlaky(flaky)
.setMuted(muted)
);

testMethod.ifPresent(
method -> ResultsUtils.processDescription(
method.getDeclaringClass().getClassLoader(),
Expand Down Expand Up @@ -781,12 +789,6 @@ private List<TestIdentifier> getParents(final TestIdentifier testIdentifier) {
return result;
}

private Optional<SeverityLevel> getSeverity(final AnnotatedElement annotatedElement) {
return getAnnotations(annotatedElement, Severity.class)
.map(Severity::value)
.findAny();
}

private Optional<String> getDisplayName(final AnnotatedElement annotatedElement) {
return getAnnotations(annotatedElement, DisplayName.class)
.map(DisplayName::value)
Expand Down
43 changes: 43 additions & 0 deletions ...-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,10 @@
import io.qameta.allure.junitplatform.features.DisabledTests;
import io.qameta.allure.junitplatform.features.DynamicTests;
import io.qameta.allure.junitplatform.features.FailedTests;
import io.qameta.allure.junitplatform.features.FlakyMutedTest;
import io.qameta.allure.junitplatform.features.JupiterUniqueIdTest;
import io.qameta.allure.junitplatform.features.MarkerAnnotationSupport;
import io.qameta.allure.junitplatform.features.MetaAnnotationTest;
import io.qameta.allure.junitplatform.features.NestedDisplayNameTests;
import io.qameta.allure.junitplatform.features.NestedTests;
import io.qameta.allure.junitplatform.features.OneTest;
Expand Down Expand Up @@ -809,6 +811,47 @@ void shouldSetSeverity() {
.contains("trivial");
}

@AllureFeatures.MarkerAnnotations
@Test
void shouldSetFlakyAndMuted() {
final AllureResults results = runClasses(FlakyMutedTest.class);
assertThat(results.getTestResults())
.filteredOn("name", "passedFlakyMuted()")
.extracting(tr -> tr.getStatusDetails().isFlaky(), tr -> tr.getStatusDetails().isMuted())
.containsExactly(tuple(true, true));
}

@AllureFeatures.MarkerAnnotations
@Test
void shouldKeepFlakyAndMutedForFailedTest() {
final AllureResults results = runClasses(FlakyMutedTest.class);
assertThat(results.getTestResults())
.filteredOn("name", "failedFlakyMuted()")
.extracting(
TestResult::getStatus,
tr -> tr.getStatusDetails().isFlaky(),
tr -> tr.getStatusDetails().isMuted()
)
.containsExactly(tuple(Status.FAILED, true, true));
}

@AllureFeatures.MarkerAnnotations
@Test
void shouldSupportFlakyMutedSeverityAsMetaAnnotation() {
final AllureResults results = runClasses(MetaAnnotationTest.class);
final List<TestResult> testResults = results.getTestResults();

assertThat(testResults)
.extracting(tr -> tr.getStatusDetails().isFlaky(), tr -> tr.getStatusDetails().isMuted())
.containsExactly(tuple(true, true));

assertThat(testResults)
.flatExtracting(TestResult::getLabels)
.filteredOn("name", SEVERITY_LABEL_NAME)
.extracting(Label::getValue)
.containsExactly("critical");
}

@AllureFeatures.MarkerAnnotations
@Test
void shouldSetOwner() {
Expand Down
Loading
Loading