From d665da91fe3b991fbc6625c12feebefa1bd8c30d Mon Sep 17 00:00:00 2001 From: Natalie Chouinard Date: Wed, 11 Nov 2026 13:30:43 -0500 Subject: [PATCH] [SCEV] Drop stale no-wrap flags from hoisted IV increments replaceCongruentIVs could hoist an IV increment above a loop exit guard while keeping nsw/nuw flags that only held below it, letting InstCombine remove an array index mask. Drop those flags, keeping only flags shared by both increments, and only apply an increment's flags to the post-increment add rec when overflow would be UB. Partial backports of several upstream LLVM changes. --- include/llvm/Analysis/ScalarEvolution.h | 16 ++ .../llvm/Analysis/ScalarEvolutionExpander.h | 15 +- lib/Analysis/ScalarEvolution.cpp | 200 +++++++++++++- lib/Analysis/ScalarEvolutionExpander.cpp | 87 +++++- .../IndVarSimplify/guarded-latch-inc-flags.ll | 43 +++ .../IndVarSimplify/iv-hoist-poison-flags.ll | 250 ++++++++++++++++++ .../ScalarEvolution/post-inc-nowrap-flags.ll | 86 ++++++ 7 files changed, 680 insertions(+), 17 deletions(-) create mode 100644 tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll create mode 100644 tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll create mode 100644 tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index d47cab829ced..ca4f02eb7be0 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -379,6 +379,13 @@ namespace llvm { /// computeLoopDisposition - Compute a LoopDisposition value. LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L); + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + /// A cache of the predicate "does the given loop contain an instruction + /// that can throw?" + DenseMap LoopMayThrow; + // HLSL Change End - Partial backport of PR27315. + /// BlockDispositions - Memoized computeBlockDisposition results. DenseMap< const SCEV *, @@ -421,6 +428,15 @@ namespace llvm { /// SCEVs. const SCEV *createNodeForPHI(PHINode *PN); + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + /// Return true if \p I, the increment of an add recurrence on the loop + /// \p L, will never generate poison, because generating poison would be + /// undefined behavior. Only then can the no-wrap flags of \p I be applied + /// to the post-increment add recurrence. + bool isAddRecNeverPoison(const Instruction *I, const Loop *L); + // HLSL Change End - Partial backport of PR27315. + /// createNodeForGEP - Provide the special handling we need to analyze GEP /// SCEVs. const SCEV *createNodeForGEP(GEPOperator *GEP); diff --git a/include/llvm/Analysis/ScalarEvolutionExpander.h b/include/llvm/Analysis/ScalarEvolutionExpander.h index c70529d157ed..23237fab9e6e 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -132,8 +132,19 @@ namespace llvm { Instruction *getIVIncOperand(Instruction *IncV, Instruction *InsertPos, bool allowScale); - /// \brief Utility for hoisting an IV increment. - bool hoistIVInc(Instruction *IncV, Instruction *InsertPos); + // HLSL Change Begin - Partial backport of PR57187: + // https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 + // Dropped flags are not re-inferred with ScalarEvolution, which can + // re-derive them from the very flags being dropped. + /// \brief Utility for hoisting \p IncV (with all subexpressions required + /// for its computation) before \p InsertPos. If \p DropPoisonFlags is set, + /// drops all poison-generating flags from \p IncV and from the + /// instructions hoisted with it. It should be used when we are going to + /// introduce a new use in the new position that didn't exist before, and + /// may trigger new UB in case of poison. + bool hoistIVInc(Instruction *IncV, Instruction *InsertPos, + bool DropPoisonFlags = false); + // HLSL Change End - Partial backport of PR57187. /// \brief replace congruent phis with their most canonical /// representative. Return the number of phis eliminated. diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index cde7b4a3a852..f4c3ab36dda6 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -3598,6 +3598,176 @@ ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) { } } +// HLSL Change Begin - Partial backport of PR27315 and a19edc4d15b0: +// https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 +// https://github.com/llvm/llvm-project/commit/a19edc4d15b0dae0210b90615775edd76f021008 +// isAddRecNeverPoison omits upstream's isSCEVExprNeverPoison check, which DXC +// lacks, so it is more conservative. + +/// Return true if \p I yields a full-poison value whenever any of its operands +/// is full-poison. This is llvm::propagatesFullPoison from ValueTracking at the +/// revisions above, which LLVM 3.7 does not have. +static bool propagatesFullPoison(const Instruction *I) { + switch (I->getOpcode()) { + case Instruction::Add: + case Instruction::Sub: + case Instruction::Xor: + case Instruction::Trunc: + case Instruction::BitCast: + case Instruction::AddrSpaceCast: + // These operations all propagate poison unconditionally. Note that poison + // is not any particular value, so xor or subtraction of poison with + // itself still yields poison, not zero. + return true; + + case Instruction::AShr: + case Instruction::SExt: + // For these operations, one bit of the input is replicated across + // multiple output bits. A replicated poison bit is still poison. + return true; + + case Instruction::Shl: { + // Left shift *by* a poison value is poison. The number of + // positions to shift is unsigned, so no negative values are + // possible there. Left shift by zero places preserves poison. So + // it only remains to consider left shift of poison by a positive + // number of places. + // + // A left shift by a positive number of places leaves the lowest order bit + // non-poisoned. However, if such a shift has a no-wrap flag, then we can + // make the poison operand violate that flag, yielding a fresh full-poison + // value. + auto *OBO = cast(I); + return OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap(); + } + + case Instruction::Mul: { + // A multiplication by zero yields a non-poison zero result, so we need to + // rule out zero as an operand. Conservatively, multiplication by a + // non-zero constant is not multiplication by zero. + // + // Multiplication by a non-zero constant can leave some bits + // non-poisoned. For example, a multiplication by 2 leaves the lowest + // order bit unpoisoned. So we need to consider that. + // + // Multiplication by 1 preserves poison. If the multiplication has a + // no-wrap flag, then we can make the poison operand violate that flag + // when multiplied by any integer other than 0 and 1. + auto *OBO = cast(I); + if (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) { + for (Value *V : OBO->operands()) { + if (auto *CI = dyn_cast(V)) { + // A ConstantInt cannot yield poison, so we can assume that it is + // the other operand that is poison. + return !CI->isZero(); + } + } + } + return false; + } + + case Instruction::ICmp: + // Comparing poison with any value yields poison. This is why, for + // instance, x s< (x +nsw 1) can be folded to true. + return true; + + case Instruction::GetElementPtr: + // A GEP implicitly represents a sequence of additions, subtractions, + // truncations, sign extensions and multiplications. The multiplications + // are by the non-zero sizes of some set of types, so we do not have to be + // concerned with multiplication by zero. If the GEP is in-bounds, then + // these operations are implicitly no-signed-wrap so poison is propagated + // by the arguments above for Add, Sub, Trunc, SExt and Mul. + return cast(I)->isInBounds(); + + default: + return false; + } +} + +bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) { + // For an add recurrence specifically, we assume that infinite loops without + // side effects are undefined behavior, and then reason as follows: + // + // If the add recurrence is poison in any iteration, it is poison on all + // future iterations (since incrementing poison yields poison). If the result + // of the add recurrence is fed into the loop latch condition and the loop + // does not contain any throws or exiting blocks other than the latch, we now + // have the ability to "choose" whether the backedge is taken or not (by + // choosing a sufficiently evil value for the poison feeding into the branch) + // for every iteration including and after the one in which \p I first became + // poison. There are two possibilities (let's call the iteration in which \p + // I first became poison as K): + // + // 1. In the set of iterations including and after K, the loop body executes + // no side effects. In this case executing the backege an infinte number + // of times will yield undefined behavior. + // + // 2. In the set of iterations including and after K, the loop body executes + // at least one side effect. In this case, that specific instance of side + // effect is control dependent on poison, which also yields undefined + // behavior. + + auto *ExitingBB = L->getExitingBlock(); + auto *LatchBB = L->getLoopLatch(); + if (!ExitingBB || !LatchBB || ExitingBB != LatchBB) + return false; + + SmallPtrSet Pushed; + SmallVector PoisonStack; + + // We start by assuming \c I, the post-inc add recurrence, is poison. Only + // things that are known to be fully poison under that assumption go on the + // PoisonStack. + Pushed.insert(I); + PoisonStack.push_back(I); + + bool LatchControlDependentOnPoison = false; + while (!PoisonStack.empty()) { + const Instruction *Poison = PoisonStack.pop_back_val(); + + for (auto *PoisonUser : Poison->users()) { + if (propagatesFullPoison(cast(PoisonUser))) { + if (Pushed.insert(cast(PoisonUser)).second) + PoisonStack.push_back(cast(PoisonUser)); + } else if (auto *BI = dyn_cast(PoisonUser)) { + assert(BI->isConditional() && "Only possibility!"); + if (BI->getParent() == LatchBB) { + LatchControlDependentOnPoison = true; + break; + } + } + } + } + + if (!LatchControlDependentOnPoison) + return false; + + // Now check if loop is no-throw, and cache the information. In the future, + // we can consider commoning this logic with LICMSafetyInfo into a separate + // analysis pass. + // + // Upstream later replaced this with an abnormal exit check (8598412e24d7) + // that, at the time, treated every call, including DXIL intrinsics, as one. + + auto Itr = LoopMayThrow.find(L); + if (Itr == LoopMayThrow.end()) { + bool MayThrow = false; + for (auto *BB : L->getBlocks()) { + MayThrow = std::any_of(BB->begin(), BB->end(), + [](Instruction &I) { return I.mayThrow(); }); + if (MayThrow) + break; + } + auto InsertPair = LoopMayThrow.insert({L, MayThrow}); + assert(InsertPair.second && "We just checked!"); + Itr = InsertPair.first; + } + + return !Itr->second; +} +// HLSL Change End - Partial backport of PR27315 and a19edc4d15b0. + /// createNodeForPHI - PHI nodes have two cases. Either the PHI node exists in /// a loop header, making it a potential recurrence, or it doesn't. /// @@ -3666,8 +3836,11 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { cast(Accum)->getLoop() == L)) { SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; - // If the increment doesn't overflow, then neither the addrec nor - // the post-increment will overflow. + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + // If the increment doesn't overflow, then the addrec doesn't + // either. The post-increment is handled below. + // HLSL Change End - Partial backport of PR27315. if (const AddOperator *OBO = dyn_cast(BEValueV)) { if (OBO->getOperand(0) == PN) { if (OBO->hasNoUnsignedWrap()) @@ -3698,17 +3871,25 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { const SCEV *StartVal = getSCEV(StartValueV); const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); - // Since the no-wrap flags are on the increment, they apply to the - // post-incremented value as well. - if (isLoopInvariant(Accum, L)) - (void)getAddRecExpr(getAddExpr(StartVal, Accum), - Accum, L, Flags); - // Okay, for the entire analysis of this edge we assumed the PHI // to be symbolic. We now need to go back and purge all of the // entries for the scalars that use the symbolic expression. ForgetSymbolicName(PN, SymbolicName); ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; + + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + // isAddRecNeverPoison is more conservative than upstream's. + + // We can add Flags to the post-inc expression only if we + // know that it us *undefined behavior* for BEValueV to + // overflow. + if (auto *BEInst = dyn_cast(BEValueV)) + if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L)) + (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, + Flags); + // HLSL Change End - Partial backport of PR27315. + return PHISCEV; } } @@ -4722,6 +4903,8 @@ void ScalarEvolution::forgetLoop(const Loop *L) { // ValuesAtScopes map. for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) forgetLoop(*I); + + LoopMayThrow.erase(L); // HLSL Change - Partial backport of PR27315. } /// forgetValue - This method should be called by the client when it has @@ -8173,6 +8356,7 @@ void ScalarEvolution::releaseMemory() { ConstantEvolutionLoopExitValue.clear(); ValuesAtScopes.clear(); LoopDispositions.clear(); + LoopMayThrow.clear(); // HLSL Change - Partial backport of PR27315. BlockDispositions.clear(); UnsignedRanges.clear(); SignedRanges.clear(); diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp index bcc1d3ed35c9..898e32f5085d 100644 --- a/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/lib/Analysis/ScalarEvolutionExpander.cpp @@ -908,12 +908,36 @@ Instruction *SCEVExpander::getIVIncOperand(Instruction *IncV, } } +// HLSL Change Begin - Partial backport of PR57187 and PR59777: +// https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 +// https://github.com/llvm/llvm-project/commit/90c184662925b8f17bc9cf05cd60a9e98201fff6 +// Dropped flags are not re-inferred with ScalarEvolution, which can re-derive +// them from the very flags being dropped. + +/// Drop the poison-generating flags (nuw, nsw, exact and inbounds) of \p I. +/// LLVM 3.7 lacks Instruction::dropPoisonGeneratingFlags. +static void dropPoisonGeneratingFlags(Instruction *I) { + if (isa(I)) { + cast(I)->setHasNoUnsignedWrap(false); + cast(I)->setHasNoSignedWrap(false); + } else if (isa(I)) { + cast(I)->setIsExact(false); + } else if (auto *GEP = dyn_cast(I)) { + GEP->setIsInBounds(false); + } +} + /// hoistStep - Attempt to hoist a simple IV increment above InsertPos to make /// it available to other uses in this loop. Recursively hoist any operands, /// until we reach a value that dominates InsertPos. -bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos) { - if (SE.DT->dominates(IncV, InsertPos)) - return true; +bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos, + bool DropPoisonFlags) { + if (SE.DT->dominates(IncV, InsertPos)) { + if (DropPoisonFlags) + dropPoisonGeneratingFlags(IncV); + return true; + } + // HLSL Change End - Partial backport of PR57187 and PR59777. // InsertPos must itself dominate IncV so that IncV's new position satisfies // its existing users. @@ -936,6 +960,11 @@ bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos) { for (SmallVectorImpl::reverse_iterator I = IVIncs.rbegin(), E = IVIncs.rend(); I != E; ++I) { (*I)->moveBefore(InsertPos); + // HLSL Change Begin - Partial backport of PR57187: + // https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 + if (DropPoisonFlags) + dropPoisonGeneratingFlags(*I); + // HLSL Change End - Partial backport of PR57187. } return true; } @@ -1683,6 +1712,27 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L, return V; } +// HLSL Change Begin - Partial backport of #79512: +// https://github.com/llvm/llvm-project/commit/da437330beca0411912ca46f2d5ef1e46403acbd +/// Return true if \p OrigInc and \p IsomorphicInc apply the same overflowing +/// operation, with the same step operand, to their phis \p OrigPhi and +/// \p IsomorphicPhi. For congruent phis, the two increments then compute the +/// same value and overflow in the same iterations. +static bool isSameIVIncOperation(const PHINode *OrigPhi, + const Instruction *OrigInc, + const PHINode *IsomorphicPhi, + const Instruction *IsomorphicInc) { + auto *OrigOBO = dyn_cast(OrigInc); + auto *IsomorphicOBO = dyn_cast(IsomorphicInc); + return OrigOBO && IsomorphicOBO && + OrigOBO->getOpcode() == IsomorphicOBO->getOpcode() && + OrigOBO->getType() == IsomorphicOBO->getType() && + OrigOBO->getOperand(0) == OrigPhi && + IsomorphicOBO->getOperand(0) == IsomorphicPhi && + OrigOBO->getOperand(1) == IsomorphicOBO->getOperand(1); +} +// HLSL Change End - Partial backport of #79512. + /// replaceCongruentIVs - Check for congruent phis in this loop header and /// replace them with their most canonical representative. Return the number of /// phis eliminated. @@ -1772,12 +1822,35 @@ SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT, // is isomorphic with the original phi. It's worth eagerly cleaning up the // common case of a single IV increment so that DeleteDeadPHIs can remove // cycles that had postinc uses. + // HLSL Change Begin - Partial backport of PR57187 and #79512: + // https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 + // https://github.com/llvm/llvm-project/commit/da437330beca0411912ca46f2d5ef1e46403acbd + // Flags are only kept when both increments apply the same operation to + // their phis, which upstream does not require. + // Replacing IsomorphicInc with OrigInc adds uses of OrigInc where its + // flags may not hold, e.g. above a loop exit they depend on, so + // hoistIVInc drops them. A nuw/nsw flag that both increments carry still + // holds there and is kept. + bool KeepNUW = false; + bool KeepNSW = false; + if (isSameIVIncOperation(OrigPhiRef, OrigInc, Phi, IsomorphicInc)) { + auto *OrigOBO = cast(OrigInc); + auto *IsomorphicOBO = cast(IsomorphicInc); + KeepNUW = + OrigOBO->hasNoUnsignedWrap() && IsomorphicOBO->hasNoUnsignedWrap(); + KeepNSW = + OrigOBO->hasNoSignedWrap() && IsomorphicOBO->hasNoSignedWrap(); + } const SCEV *TruncExpr = SE.getTruncateOrNoop(SE.getSCEV(OrigInc), IsomorphicInc->getType()); - if (OrigInc != IsomorphicInc - && TruncExpr == SE.getSCEV(IsomorphicInc) - && ((isa(OrigInc) && isa(IsomorphicInc)) - || hoistIVInc(OrigInc, IsomorphicInc))) { + if (OrigInc != IsomorphicInc && TruncExpr == SE.getSCEV(IsomorphicInc) && + ((isa(OrigInc) && isa(IsomorphicInc)) || + hoistIVInc(OrigInc, IsomorphicInc, /*DropPoisonFlags*/ true))) { + if (KeepNUW) + cast(OrigInc)->setHasNoUnsignedWrap(true); + if (KeepNSW) + cast(OrigInc)->setHasNoSignedWrap(true); + // HLSL Change End - Partial backport of PR57187 and #79512. DEBUG_WITH_TYPE(DebugType, dbgs() << "INDVARS: Eliminated congruent iv.inc: " << *IsomorphicInc << '\n'); diff --git a/tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll b/tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll new file mode 100644 index 000000000000..7dad8542c57d --- /dev/null +++ b/tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll @@ -0,0 +1,43 @@ +; RUN: %dxopt %s -indvars -S | FileCheck %s + +; ScalarEvolution::createNodeForPHI used to apply the no-wrap flags of an IV +; increment to the post-increment add recurrence unconditionally. That add +; recurrence is also the SCEV of any other instruction computing the same +; value, even one that executes on iterations where the increment does not. +; SimplifyIndvar then used those flags to mark such an instruction nsw. +; Partial backport of PR27315: +; https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +; %inc_i is nsw because it only executes below the exit guard, where +; %i < INT32_MAX. With %c true, %inc_u computes INT32_MAX + 1 in the header on +; the second iteration, so it must not become nsw. +define i32 @guarded_latch_inc_flags(i1 %c) { +; CHECK-LABEL: @guarded_latch_inc_flags( +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] +; CHECK-NEXT: %inc_u = add nuw i32 %i, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %i, 2147483646 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: %inc_i = add nuw nsw i32 %i, 1 +; CHECK-NEXT: br i1 %c, label %loop, label %exit +entry: + br label %loop + +loop: + %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] + %inc_u = add i32 %i, 1 + %cond = icmp sgt i32 %i, 2147483646 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} diff --git a/tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll b/tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll new file mode 100644 index 000000000000..087747f0bfb7 --- /dev/null +++ b/tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll @@ -0,0 +1,250 @@ +; RUN: %dxopt %s -indvars -S | FileCheck %s + +; When SCEVExpander::replaceCongruentIVs replaces an isomorphic induction +; variable increment with the canonical increment, either by hoisting the +; canonical increment above it (hoistIVInc) or by reusing a canonical increment +; that already dominates it, the canonical increment gets new uses. Its +; poison-generating flags may only have held at its original position, e.g. +; below a loop exit guard, so they are dropped. Only nuw/nsw flags that both +; increments already carry are kept. +; +; Based on: +; https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 (PR57187) +; https://github.com/llvm/llvm-project/commit/90c184662925b8f17bc9cf05cd60a9e98201fff6 (PR59777) +; https://github.com/llvm/llvm-project/commit/1f9de23e94af3ff50f67a3b16be3769919c2e146 +; https://github.com/llvm/llvm-project/commit/da437330beca0411912ca46f2d5ef1e46403acbd (#79512) + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +; Hoisting %inc_i (add nsw) from the latch above the loop exit guard to replace +; %inc_u (add without nsw) must drop nsw, because %inc_u executes on the first +; iteration before the exit guard when %start_nonneg can be INT32_MAX. +define i32 @hoist_iv_inc_across_guard_drops_nsw(i32 %start) { +; CHECK-LABEL: @hoist_iv_inc_across_guard_drops_nsw( +; CHECK-NEXT: entry: +; CHECK-NEXT: %start_nonneg = and i32 %start, 2147483647 +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ %start_nonneg, %entry ], [ %inc_i, %latch ] +; CHECK-NEXT: %inc_i = add nuw i32 %i, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %i, 9 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: br i1 true, label %loop, label %exit +; CHECK: exit: +; CHECK-NEXT: %res = phi i32 [ %inc_i, %loop ], [ %inc_i, %latch ] +; CHECK-NEXT: ret i32 %res +entry: + %start_nonneg = and i32 %start, 2147483647 + br label %loop + +loop: + %i = phi i32 [ %start_nonneg, %entry ], [ %inc_i, %latch ] + %u = phi i32 [ %start_nonneg, %entry ], [ %inc_u, %latch ] + %g = phi i32 [ -1, %entry ], [ %inc_g, %latch ] + %inc_u = add i32 %u, 1 + %cond = icmp sgt i32 %i, 9 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + %inc_g = add nsw i32 %g, 1 + %backedge_cond = icmp slt i32 %inc_g, 1000 + br i1 %backedge_cond, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} + +; Like above, but the overflow happens on the second iteration: with %c true, +; %inc_u computes INT32_MAX + 1 in the header before the exit guard is taken. +; The add that replaces it must not be nsw, and nsw must not be re-derived from +; ScalarEvolution, whose add recurrences for %i and %u carry %inc_i's flags. +define i32 @hoist_iv_inc_across_guard_one_iteration(i1 %c) { +; CHECK-LABEL: @hoist_iv_inc_across_guard_one_iteration( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] +; CHECK-NEXT: %inc_i = add nuw i32 %i, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %i, 2147483646 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: br i1 %c, label %loop, label %exit +; CHECK: exit: +; CHECK-NEXT: %res = phi i32 [ %inc_i, %loop ], [ %inc_i, %latch ] +; CHECK-NEXT: ret i32 %res +entry: + br label %loop + +loop: + %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] + %u = phi i32 [ 2147483646, %entry ], [ %inc_u, %latch ] + %inc_u = add i32 %u, 1 + %cond = icmp sgt i32 %i, 2147483646 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} + +; Opposite phi order: %inc_u already dominates %inc_i and replaces it. %inc_u +; must not become nsw because of %inc_i, which only executes below the guard. +define i32 @reuse_dominating_iv_inc_across_guard(i1 %c) { +; CHECK-LABEL: @reuse_dominating_iv_inc_across_guard( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %u = phi i32 [ 2147483646, %entry ], [ %inc_u, %latch ] +; CHECK-NEXT: %inc_u = add nuw i32 %u, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %u, 2147483646 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: br i1 %c, label %loop, label %exit +; CHECK: exit: +; CHECK-NEXT: %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] +; CHECK-NEXT: ret i32 %res +entry: + br label %loop + +loop: + %u = phi i32 [ 2147483646, %entry ], [ %inc_u, %latch ] + %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] + %inc_u = add i32 %u, 1 + %cond = icmp sgt i32 %i, 2147483646 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} + +; Both increments are nsw, so the hoisted increment keeps nsw. +define void @hoist_iv_inc_keeps_common_nsw(i32 %start, i32 %n, i32* %p) { +; CHECK-LABEL: @hoist_iv_inc_keeps_common_nsw( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ %start, %entry ], [ %inc_i, %loop ] +; CHECK-NEXT: %inc_i = add nsw i32 %i, 1 +; CHECK-NEXT: store i32 %inc_i, i32* %p +; CHECK-NEXT: %cmp = icmp slt i32 %inc_i, %n +; CHECK-NEXT: br i1 %cmp, label %loop, label %exit +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc_i, %loop ] + %u = phi i32 [ %start, %entry ], [ %inc_u, %loop ] + %inc_u = add nsw i32 %u, 1 + store i32 %inc_u, i32* %p + %inc_i = add nsw i32 %i, 1 + %cmp = icmp slt i32 %inc_i, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + +; Hoisting a pointer increment above the loop exit guard must drop inbounds. +define i32* @hoist_gep_iv_inc_drops_inbounds(i32* %base, i1 %c0, i1 %c1) { +; CHECK-LABEL: @hoist_gep_iv_inc_drops_inbounds( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %p = phi i32* [ %base, %entry ], [ %p.next, %latch ] +; CHECK-NEXT: %p.next = getelementptr i32, i32* %p, i32 1 +; CHECK-NEXT: br i1 %c0, label %exit, label %latch +; CHECK: exit: +; CHECK-NEXT: %res = phi i32* [ %p.next, %loop ], [ %p.next, %latch ] +; CHECK-NEXT: ret i32* %res +entry: + br label %loop + +loop: + %p = phi i32* [ %base, %entry ], [ %p.next, %latch ] + %q = phi i32* [ %base, %entry ], [ %q.next, %latch ] + %q.next = getelementptr i32, i32* %q, i32 1 + br i1 %c0, label %exit, label %latch + +latch: + %p.next = getelementptr inbounds i32, i32* %p, i32 1 + br i1 %c1, label %loop, label %exit + +exit: + %res = phi i32* [ %q.next, %loop ], [ %q.next, %latch ] + ret i32* %res +} + +; Adapted from llvm/test/Transforms/IndVarSimplify/iv-poison.ll (PR59777), +; widened from i2 (INT2_MAX = 1) to i32 (INT32_MAX = 2147483647). +; Even when %i already dominates %i2, replacing %i2 with %i introduces a new +; use of %i, so nsw, which only %i carries, must be dropped. Both increments +; are nuw by then, so nuw is kept. +define i32 @iv_hoist_nsw_poison(i32 %arg) { +; CHECK-LABEL: @iv_hoist_nsw_poison( +; CHECK-NEXT: bb: +; CHECK-NEXT: br label %bb1 +; CHECK: bb1: +; CHECK-NEXT: %.07 = phi i32 [ 2147483647, %bb ], [ %i, %bb1 ] +; CHECK-NEXT: %i = add nuw i32 %.07, 1 +; CHECK-NEXT: %.not.not = icmp ult i32 %.07, %arg +; CHECK-NEXT: br i1 %.not.not, label %common.ret, label %bb1 +; CHECK: common.ret: +; CHECK-NEXT: %i2.lcssa = phi i32 [ %i, %bb1 ] +; CHECK-NEXT: ret i32 %i2.lcssa +bb: + br label %bb1 + +bb1: + %.07 = phi i32 [ 2147483647, %bb ], [ %i, %bb1 ] + %.0 = phi i32 [ 2147483647, %bb ], [ %i2, %bb1 ] + %i = add nsw i32 %.07, 1 + %i2 = add i32 %.0, 1 + %.not.not = icmp ult i32 %.07, %arg + br i1 %.not.not, label %common.ret, label %bb1 + +common.ret: + ret i32 %i2 +} + +; Adapted from llvm/test/Transforms/IndVarSimplify/iv-poison.ll, widened from +; i4 to i32. The flagless %iv.0.next replaces %iv.1.next and must not become +; nsw. +define i32 @iv_hoist_nsw_poison2(i32 %end, i32 %start) { +; CHECK-LABEL: @iv_hoist_nsw_poison2( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %iv.0 = phi i32 [ %start, %entry ], [ %iv.0.next, %loop ] +; CHECK-NEXT: %iv.0.next = add i32 %iv.0, 1 +; CHECK-NEXT: %.not.not = icmp ult i32 %iv.0, %end +; CHECK-NEXT: br i1 %.not.not, label %exit, label %loop +; CHECK: exit: +; CHECK-NEXT: %iv.1.next.lcssa = phi i32 [ %iv.0.next, %loop ] +; CHECK-NEXT: ret i32 %iv.1.next.lcssa +entry: + br label %loop + +loop: + %iv.0 = phi i32 [ %start, %entry ], [ %iv.0.next, %loop ] + %iv.1 = phi i32 [ %start, %entry ], [ %iv.1.next, %loop ] + %iv.0.next = add i32 %iv.0, 1 + %iv.1.next = add nsw i32 %iv.1, 1 + %.not.not = icmp ult i32 %iv.0, %end + br i1 %.not.not, label %exit, label %loop + +exit: + ret i32 %iv.1.next +} diff --git a/tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll b/tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll new file mode 100644 index 000000000000..a4b2b8a636c9 --- /dev/null +++ b/tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll @@ -0,0 +1,86 @@ +; RUN: %dxopt %s -analyze -scalar-evolution | FileCheck %s + +; The no-wrap flags of an IV increment may only be applied to the +; post-increment add recurrence when overflowing the increment would be +; undefined behavior: the loop's only exiting block is the latch, the latch +; condition depends on the increment, and nothing in the loop may throw. +; Otherwise other instructions computing the same value, on iterations where +; the increment does not execute, would inherit its flags. +; Partial backport of PR27315: +; https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +declare void @may_throw() + +; CHECK-LABEL: Classifying expressions for: @latch_exit_uses_inc +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @latch_exit_uses_inc(i32 %start, i32 %n) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %loop ] + %inc = add nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + +; CHECK-LABEL: Classifying expressions for: @header_exit_guards_inc +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @header_exit_guards_inc(i32 %start, i1 %c) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %latch ] + %cond = icmp sgt i32 %i, 9 + br i1 %cond, label %exit, label %latch + +latch: + %inc = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + ret void +} + +; CHECK-LABEL: Classifying expressions for: @latch_exit_ignores_inc +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @latch_exit_ignores_inc(i32 %start, i1 %c) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %loop ] + %inc = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + ret void +} + +; CHECK-LABEL: Classifying expressions for: @loop_may_throw +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @loop_may_throw(i32 %start, i32 %n) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %loop ] + call void @may_throw() + %inc = add nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +}