diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index d47cab829ce..ca4f02eb7be 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -379,6 +379,13 @@ namespace llvm { /// computeLoopDisposition - Compute a LoopDisposition value. LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L); + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + /// A cache of the predicate "does the given loop contain an instruction + /// that can throw?" + DenseMap LoopMayThrow; + // HLSL Change End - Partial backport of PR27315. + /// BlockDispositions - Memoized computeBlockDisposition results. DenseMap< const SCEV *, @@ -421,6 +428,15 @@ namespace llvm { /// SCEVs. const SCEV *createNodeForPHI(PHINode *PN); + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + /// Return true if \p I, the increment of an add recurrence on the loop + /// \p L, will never generate poison, because generating poison would be + /// undefined behavior. Only then can the no-wrap flags of \p I be applied + /// to the post-increment add recurrence. + bool isAddRecNeverPoison(const Instruction *I, const Loop *L); + // HLSL Change End - Partial backport of PR27315. + /// createNodeForGEP - Provide the special handling we need to analyze GEP /// SCEVs. const SCEV *createNodeForGEP(GEPOperator *GEP); diff --git a/include/llvm/Analysis/ScalarEvolutionExpander.h b/include/llvm/Analysis/ScalarEvolutionExpander.h index c70529d157e..23237fab9e6 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -132,8 +132,19 @@ namespace llvm { Instruction *getIVIncOperand(Instruction *IncV, Instruction *InsertPos, bool allowScale); - /// \brief Utility for hoisting an IV increment. - bool hoistIVInc(Instruction *IncV, Instruction *InsertPos); + // HLSL Change Begin - Partial backport of PR57187: + // https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 + // Dropped flags are not re-inferred with ScalarEvolution, which can + // re-derive them from the very flags being dropped. + /// \brief Utility for hoisting \p IncV (with all subexpressions required + /// for its computation) before \p InsertPos. If \p DropPoisonFlags is set, + /// drops all poison-generating flags from \p IncV and from the + /// instructions hoisted with it. It should be used when we are going to + /// introduce a new use in the new position that didn't exist before, and + /// may trigger new UB in case of poison. + bool hoistIVInc(Instruction *IncV, Instruction *InsertPos, + bool DropPoisonFlags = false); + // HLSL Change End - Partial backport of PR57187. /// \brief replace congruent phis with their most canonical /// representative. Return the number of phis eliminated. diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index cde7b4a3a85..f4c3ab36dda 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -3598,6 +3598,176 @@ ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) { } } +// HLSL Change Begin - Partial backport of PR27315 and a19edc4d15b0: +// https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 +// https://github.com/llvm/llvm-project/commit/a19edc4d15b0dae0210b90615775edd76f021008 +// isAddRecNeverPoison omits upstream's isSCEVExprNeverPoison check, which DXC +// lacks, so it is more conservative. + +/// Return true if \p I yields a full-poison value whenever any of its operands +/// is full-poison. This is llvm::propagatesFullPoison from ValueTracking at the +/// revisions above, which LLVM 3.7 does not have. +static bool propagatesFullPoison(const Instruction *I) { + switch (I->getOpcode()) { + case Instruction::Add: + case Instruction::Sub: + case Instruction::Xor: + case Instruction::Trunc: + case Instruction::BitCast: + case Instruction::AddrSpaceCast: + // These operations all propagate poison unconditionally. Note that poison + // is not any particular value, so xor or subtraction of poison with + // itself still yields poison, not zero. + return true; + + case Instruction::AShr: + case Instruction::SExt: + // For these operations, one bit of the input is replicated across + // multiple output bits. A replicated poison bit is still poison. + return true; + + case Instruction::Shl: { + // Left shift *by* a poison value is poison. The number of + // positions to shift is unsigned, so no negative values are + // possible there. Left shift by zero places preserves poison. So + // it only remains to consider left shift of poison by a positive + // number of places. + // + // A left shift by a positive number of places leaves the lowest order bit + // non-poisoned. However, if such a shift has a no-wrap flag, then we can + // make the poison operand violate that flag, yielding a fresh full-poison + // value. + auto *OBO = cast(I); + return OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap(); + } + + case Instruction::Mul: { + // A multiplication by zero yields a non-poison zero result, so we need to + // rule out zero as an operand. Conservatively, multiplication by a + // non-zero constant is not multiplication by zero. + // + // Multiplication by a non-zero constant can leave some bits + // non-poisoned. For example, a multiplication by 2 leaves the lowest + // order bit unpoisoned. So we need to consider that. + // + // Multiplication by 1 preserves poison. If the multiplication has a + // no-wrap flag, then we can make the poison operand violate that flag + // when multiplied by any integer other than 0 and 1. + auto *OBO = cast(I); + if (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) { + for (Value *V : OBO->operands()) { + if (auto *CI = dyn_cast(V)) { + // A ConstantInt cannot yield poison, so we can assume that it is + // the other operand that is poison. + return !CI->isZero(); + } + } + } + return false; + } + + case Instruction::ICmp: + // Comparing poison with any value yields poison. This is why, for + // instance, x s< (x +nsw 1) can be folded to true. + return true; + + case Instruction::GetElementPtr: + // A GEP implicitly represents a sequence of additions, subtractions, + // truncations, sign extensions and multiplications. The multiplications + // are by the non-zero sizes of some set of types, so we do not have to be + // concerned with multiplication by zero. If the GEP is in-bounds, then + // these operations are implicitly no-signed-wrap so poison is propagated + // by the arguments above for Add, Sub, Trunc, SExt and Mul. + return cast(I)->isInBounds(); + + default: + return false; + } +} + +bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) { + // For an add recurrence specifically, we assume that infinite loops without + // side effects are undefined behavior, and then reason as follows: + // + // If the add recurrence is poison in any iteration, it is poison on all + // future iterations (since incrementing poison yields poison). If the result + // of the add recurrence is fed into the loop latch condition and the loop + // does not contain any throws or exiting blocks other than the latch, we now + // have the ability to "choose" whether the backedge is taken or not (by + // choosing a sufficiently evil value for the poison feeding into the branch) + // for every iteration including and after the one in which \p I first became + // poison. There are two possibilities (let's call the iteration in which \p + // I first became poison as K): + // + // 1. In the set of iterations including and after K, the loop body executes + // no side effects. In this case executing the backege an infinte number + // of times will yield undefined behavior. + // + // 2. In the set of iterations including and after K, the loop body executes + // at least one side effect. In this case, that specific instance of side + // effect is control dependent on poison, which also yields undefined + // behavior. + + auto *ExitingBB = L->getExitingBlock(); + auto *LatchBB = L->getLoopLatch(); + if (!ExitingBB || !LatchBB || ExitingBB != LatchBB) + return false; + + SmallPtrSet Pushed; + SmallVector PoisonStack; + + // We start by assuming \c I, the post-inc add recurrence, is poison. Only + // things that are known to be fully poison under that assumption go on the + // PoisonStack. + Pushed.insert(I); + PoisonStack.push_back(I); + + bool LatchControlDependentOnPoison = false; + while (!PoisonStack.empty()) { + const Instruction *Poison = PoisonStack.pop_back_val(); + + for (auto *PoisonUser : Poison->users()) { + if (propagatesFullPoison(cast(PoisonUser))) { + if (Pushed.insert(cast(PoisonUser)).second) + PoisonStack.push_back(cast(PoisonUser)); + } else if (auto *BI = dyn_cast(PoisonUser)) { + assert(BI->isConditional() && "Only possibility!"); + if (BI->getParent() == LatchBB) { + LatchControlDependentOnPoison = true; + break; + } + } + } + } + + if (!LatchControlDependentOnPoison) + return false; + + // Now check if loop is no-throw, and cache the information. In the future, + // we can consider commoning this logic with LICMSafetyInfo into a separate + // analysis pass. + // + // Upstream later replaced this with an abnormal exit check (8598412e24d7) + // that, at the time, treated every call, including DXIL intrinsics, as one. + + auto Itr = LoopMayThrow.find(L); + if (Itr == LoopMayThrow.end()) { + bool MayThrow = false; + for (auto *BB : L->getBlocks()) { + MayThrow = std::any_of(BB->begin(), BB->end(), + [](Instruction &I) { return I.mayThrow(); }); + if (MayThrow) + break; + } + auto InsertPair = LoopMayThrow.insert({L, MayThrow}); + assert(InsertPair.second && "We just checked!"); + Itr = InsertPair.first; + } + + return !Itr->second; +} +// HLSL Change End - Partial backport of PR27315 and a19edc4d15b0. + /// createNodeForPHI - PHI nodes have two cases. Either the PHI node exists in /// a loop header, making it a potential recurrence, or it doesn't. /// @@ -3666,8 +3836,11 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { cast(Accum)->getLoop() == L)) { SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; - // If the increment doesn't overflow, then neither the addrec nor - // the post-increment will overflow. + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + // If the increment doesn't overflow, then the addrec doesn't + // either. The post-increment is handled below. + // HLSL Change End - Partial backport of PR27315. if (const AddOperator *OBO = dyn_cast(BEValueV)) { if (OBO->getOperand(0) == PN) { if (OBO->hasNoUnsignedWrap()) @@ -3698,17 +3871,25 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { const SCEV *StartVal = getSCEV(StartValueV); const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); - // Since the no-wrap flags are on the increment, they apply to the - // post-incremented value as well. - if (isLoopInvariant(Accum, L)) - (void)getAddRecExpr(getAddExpr(StartVal, Accum), - Accum, L, Flags); - // Okay, for the entire analysis of this edge we assumed the PHI // to be symbolic. We now need to go back and purge all of the // entries for the scalars that use the symbolic expression. ForgetSymbolicName(PN, SymbolicName); ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; + + // HLSL Change Begin - Partial backport of PR27315: + // https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + // isAddRecNeverPoison is more conservative than upstream's. + + // We can add Flags to the post-inc expression only if we + // know that it us *undefined behavior* for BEValueV to + // overflow. + if (auto *BEInst = dyn_cast(BEValueV)) + if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L)) + (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, + Flags); + // HLSL Change End - Partial backport of PR27315. + return PHISCEV; } } @@ -4722,6 +4903,8 @@ void ScalarEvolution::forgetLoop(const Loop *L) { // ValuesAtScopes map. for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) forgetLoop(*I); + + LoopMayThrow.erase(L); // HLSL Change - Partial backport of PR27315. } /// forgetValue - This method should be called by the client when it has @@ -8173,6 +8356,7 @@ void ScalarEvolution::releaseMemory() { ConstantEvolutionLoopExitValue.clear(); ValuesAtScopes.clear(); LoopDispositions.clear(); + LoopMayThrow.clear(); // HLSL Change - Partial backport of PR27315. BlockDispositions.clear(); UnsignedRanges.clear(); SignedRanges.clear(); diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp index bcc1d3ed35c..898e32f5085 100644 --- a/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/lib/Analysis/ScalarEvolutionExpander.cpp @@ -908,12 +908,36 @@ Instruction *SCEVExpander::getIVIncOperand(Instruction *IncV, } } +// HLSL Change Begin - Partial backport of PR57187 and PR59777: +// https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 +// https://github.com/llvm/llvm-project/commit/90c184662925b8f17bc9cf05cd60a9e98201fff6 +// Dropped flags are not re-inferred with ScalarEvolution, which can re-derive +// them from the very flags being dropped. + +/// Drop the poison-generating flags (nuw, nsw, exact and inbounds) of \p I. +/// LLVM 3.7 lacks Instruction::dropPoisonGeneratingFlags. +static void dropPoisonGeneratingFlags(Instruction *I) { + if (isa(I)) { + cast(I)->setHasNoUnsignedWrap(false); + cast(I)->setHasNoSignedWrap(false); + } else if (isa(I)) { + cast(I)->setIsExact(false); + } else if (auto *GEP = dyn_cast(I)) { + GEP->setIsInBounds(false); + } +} + /// hoistStep - Attempt to hoist a simple IV increment above InsertPos to make /// it available to other uses in this loop. Recursively hoist any operands, /// until we reach a value that dominates InsertPos. -bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos) { - if (SE.DT->dominates(IncV, InsertPos)) - return true; +bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos, + bool DropPoisonFlags) { + if (SE.DT->dominates(IncV, InsertPos)) { + if (DropPoisonFlags) + dropPoisonGeneratingFlags(IncV); + return true; + } + // HLSL Change End - Partial backport of PR57187 and PR59777. // InsertPos must itself dominate IncV so that IncV's new position satisfies // its existing users. @@ -936,6 +960,11 @@ bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos) { for (SmallVectorImpl::reverse_iterator I = IVIncs.rbegin(), E = IVIncs.rend(); I != E; ++I) { (*I)->moveBefore(InsertPos); + // HLSL Change Begin - Partial backport of PR57187: + // https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 + if (DropPoisonFlags) + dropPoisonGeneratingFlags(*I); + // HLSL Change End - Partial backport of PR57187. } return true; } @@ -1683,6 +1712,27 @@ SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L, return V; } +// HLSL Change Begin - Partial backport of #79512: +// https://github.com/llvm/llvm-project/commit/da437330beca0411912ca46f2d5ef1e46403acbd +/// Return true if \p OrigInc and \p IsomorphicInc apply the same overflowing +/// operation, with the same step operand, to their phis \p OrigPhi and +/// \p IsomorphicPhi. For congruent phis, the two increments then compute the +/// same value and overflow in the same iterations. +static bool isSameIVIncOperation(const PHINode *OrigPhi, + const Instruction *OrigInc, + const PHINode *IsomorphicPhi, + const Instruction *IsomorphicInc) { + auto *OrigOBO = dyn_cast(OrigInc); + auto *IsomorphicOBO = dyn_cast(IsomorphicInc); + return OrigOBO && IsomorphicOBO && + OrigOBO->getOpcode() == IsomorphicOBO->getOpcode() && + OrigOBO->getType() == IsomorphicOBO->getType() && + OrigOBO->getOperand(0) == OrigPhi && + IsomorphicOBO->getOperand(0) == IsomorphicPhi && + OrigOBO->getOperand(1) == IsomorphicOBO->getOperand(1); +} +// HLSL Change End - Partial backport of #79512. + /// replaceCongruentIVs - Check for congruent phis in this loop header and /// replace them with their most canonical representative. Return the number of /// phis eliminated. @@ -1772,12 +1822,35 @@ SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT, // is isomorphic with the original phi. It's worth eagerly cleaning up the // common case of a single IV increment so that DeleteDeadPHIs can remove // cycles that had postinc uses. + // HLSL Change Begin - Partial backport of PR57187 and #79512: + // https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 + // https://github.com/llvm/llvm-project/commit/da437330beca0411912ca46f2d5ef1e46403acbd + // Flags are only kept when both increments apply the same operation to + // their phis, which upstream does not require. + // Replacing IsomorphicInc with OrigInc adds uses of OrigInc where its + // flags may not hold, e.g. above a loop exit they depend on, so + // hoistIVInc drops them. A nuw/nsw flag that both increments carry still + // holds there and is kept. + bool KeepNUW = false; + bool KeepNSW = false; + if (isSameIVIncOperation(OrigPhiRef, OrigInc, Phi, IsomorphicInc)) { + auto *OrigOBO = cast(OrigInc); + auto *IsomorphicOBO = cast(IsomorphicInc); + KeepNUW = + OrigOBO->hasNoUnsignedWrap() && IsomorphicOBO->hasNoUnsignedWrap(); + KeepNSW = + OrigOBO->hasNoSignedWrap() && IsomorphicOBO->hasNoSignedWrap(); + } const SCEV *TruncExpr = SE.getTruncateOrNoop(SE.getSCEV(OrigInc), IsomorphicInc->getType()); - if (OrigInc != IsomorphicInc - && TruncExpr == SE.getSCEV(IsomorphicInc) - && ((isa(OrigInc) && isa(IsomorphicInc)) - || hoistIVInc(OrigInc, IsomorphicInc))) { + if (OrigInc != IsomorphicInc && TruncExpr == SE.getSCEV(IsomorphicInc) && + ((isa(OrigInc) && isa(IsomorphicInc)) || + hoistIVInc(OrigInc, IsomorphicInc, /*DropPoisonFlags*/ true))) { + if (KeepNUW) + cast(OrigInc)->setHasNoUnsignedWrap(true); + if (KeepNSW) + cast(OrigInc)->setHasNoSignedWrap(true); + // HLSL Change End - Partial backport of PR57187 and #79512. DEBUG_WITH_TYPE(DebugType, dbgs() << "INDVARS: Eliminated congruent iv.inc: " << *IsomorphicInc << '\n'); diff --git a/tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll b/tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll new file mode 100644 index 00000000000..7dad8542c57 --- /dev/null +++ b/tools/clang/test/DXC/Passes/IndVarSimplify/guarded-latch-inc-flags.ll @@ -0,0 +1,43 @@ +; RUN: %dxopt %s -indvars -S | FileCheck %s + +; ScalarEvolution::createNodeForPHI used to apply the no-wrap flags of an IV +; increment to the post-increment add recurrence unconditionally. That add +; recurrence is also the SCEV of any other instruction computing the same +; value, even one that executes on iterations where the increment does not. +; SimplifyIndvar then used those flags to mark such an instruction nsw. +; Partial backport of PR27315: +; https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +; %inc_i is nsw because it only executes below the exit guard, where +; %i < INT32_MAX. With %c true, %inc_u computes INT32_MAX + 1 in the header on +; the second iteration, so it must not become nsw. +define i32 @guarded_latch_inc_flags(i1 %c) { +; CHECK-LABEL: @guarded_latch_inc_flags( +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] +; CHECK-NEXT: %inc_u = add nuw i32 %i, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %i, 2147483646 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: %inc_i = add nuw nsw i32 %i, 1 +; CHECK-NEXT: br i1 %c, label %loop, label %exit +entry: + br label %loop + +loop: + %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] + %inc_u = add i32 %i, 1 + %cond = icmp sgt i32 %i, 2147483646 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} diff --git a/tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll b/tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll new file mode 100644 index 00000000000..087747f0bfb --- /dev/null +++ b/tools/clang/test/DXC/Passes/IndVarSimplify/iv-hoist-poison-flags.ll @@ -0,0 +1,250 @@ +; RUN: %dxopt %s -indvars -S | FileCheck %s + +; When SCEVExpander::replaceCongruentIVs replaces an isomorphic induction +; variable increment with the canonical increment, either by hoisting the +; canonical increment above it (hoistIVInc) or by reusing a canonical increment +; that already dominates it, the canonical increment gets new uses. Its +; poison-generating flags may only have held at its original position, e.g. +; below a loop exit guard, so they are dropped. Only nuw/nsw flags that both +; increments already carry are kept. +; +; Based on: +; https://github.com/llvm/llvm-project/commit/86d5586d78d813a6921d786f7ddb1a41c6fb56e0 (PR57187) +; https://github.com/llvm/llvm-project/commit/90c184662925b8f17bc9cf05cd60a9e98201fff6 (PR59777) +; https://github.com/llvm/llvm-project/commit/1f9de23e94af3ff50f67a3b16be3769919c2e146 +; https://github.com/llvm/llvm-project/commit/da437330beca0411912ca46f2d5ef1e46403acbd (#79512) + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +; Hoisting %inc_i (add nsw) from the latch above the loop exit guard to replace +; %inc_u (add without nsw) must drop nsw, because %inc_u executes on the first +; iteration before the exit guard when %start_nonneg can be INT32_MAX. +define i32 @hoist_iv_inc_across_guard_drops_nsw(i32 %start) { +; CHECK-LABEL: @hoist_iv_inc_across_guard_drops_nsw( +; CHECK-NEXT: entry: +; CHECK-NEXT: %start_nonneg = and i32 %start, 2147483647 +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ %start_nonneg, %entry ], [ %inc_i, %latch ] +; CHECK-NEXT: %inc_i = add nuw i32 %i, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %i, 9 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: br i1 true, label %loop, label %exit +; CHECK: exit: +; CHECK-NEXT: %res = phi i32 [ %inc_i, %loop ], [ %inc_i, %latch ] +; CHECK-NEXT: ret i32 %res +entry: + %start_nonneg = and i32 %start, 2147483647 + br label %loop + +loop: + %i = phi i32 [ %start_nonneg, %entry ], [ %inc_i, %latch ] + %u = phi i32 [ %start_nonneg, %entry ], [ %inc_u, %latch ] + %g = phi i32 [ -1, %entry ], [ %inc_g, %latch ] + %inc_u = add i32 %u, 1 + %cond = icmp sgt i32 %i, 9 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + %inc_g = add nsw i32 %g, 1 + %backedge_cond = icmp slt i32 %inc_g, 1000 + br i1 %backedge_cond, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} + +; Like above, but the overflow happens on the second iteration: with %c true, +; %inc_u computes INT32_MAX + 1 in the header before the exit guard is taken. +; The add that replaces it must not be nsw, and nsw must not be re-derived from +; ScalarEvolution, whose add recurrences for %i and %u carry %inc_i's flags. +define i32 @hoist_iv_inc_across_guard_one_iteration(i1 %c) { +; CHECK-LABEL: @hoist_iv_inc_across_guard_one_iteration( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] +; CHECK-NEXT: %inc_i = add nuw i32 %i, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %i, 2147483646 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: br i1 %c, label %loop, label %exit +; CHECK: exit: +; CHECK-NEXT: %res = phi i32 [ %inc_i, %loop ], [ %inc_i, %latch ] +; CHECK-NEXT: ret i32 %res +entry: + br label %loop + +loop: + %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] + %u = phi i32 [ 2147483646, %entry ], [ %inc_u, %latch ] + %inc_u = add i32 %u, 1 + %cond = icmp sgt i32 %i, 2147483646 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} + +; Opposite phi order: %inc_u already dominates %inc_i and replaces it. %inc_u +; must not become nsw because of %inc_i, which only executes below the guard. +define i32 @reuse_dominating_iv_inc_across_guard(i1 %c) { +; CHECK-LABEL: @reuse_dominating_iv_inc_across_guard( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %u = phi i32 [ 2147483646, %entry ], [ %inc_u, %latch ] +; CHECK-NEXT: %inc_u = add nuw i32 %u, 1 +; CHECK-NEXT: %cond = icmp sgt i32 %u, 2147483646 +; CHECK-NEXT: br i1 %cond, label %exit, label %latch +; CHECK: latch: +; CHECK-NEXT: br i1 %c, label %loop, label %exit +; CHECK: exit: +; CHECK-NEXT: %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] +; CHECK-NEXT: ret i32 %res +entry: + br label %loop + +loop: + %u = phi i32 [ 2147483646, %entry ], [ %inc_u, %latch ] + %i = phi i32 [ 2147483646, %entry ], [ %inc_i, %latch ] + %inc_u = add i32 %u, 1 + %cond = icmp sgt i32 %i, 2147483646 + br i1 %cond, label %exit, label %latch + +latch: + %inc_i = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + %res = phi i32 [ %inc_u, %loop ], [ %inc_u, %latch ] + ret i32 %res +} + +; Both increments are nsw, so the hoisted increment keeps nsw. +define void @hoist_iv_inc_keeps_common_nsw(i32 %start, i32 %n, i32* %p) { +; CHECK-LABEL: @hoist_iv_inc_keeps_common_nsw( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %i = phi i32 [ %start, %entry ], [ %inc_i, %loop ] +; CHECK-NEXT: %inc_i = add nsw i32 %i, 1 +; CHECK-NEXT: store i32 %inc_i, i32* %p +; CHECK-NEXT: %cmp = icmp slt i32 %inc_i, %n +; CHECK-NEXT: br i1 %cmp, label %loop, label %exit +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc_i, %loop ] + %u = phi i32 [ %start, %entry ], [ %inc_u, %loop ] + %inc_u = add nsw i32 %u, 1 + store i32 %inc_u, i32* %p + %inc_i = add nsw i32 %i, 1 + %cmp = icmp slt i32 %inc_i, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + +; Hoisting a pointer increment above the loop exit guard must drop inbounds. +define i32* @hoist_gep_iv_inc_drops_inbounds(i32* %base, i1 %c0, i1 %c1) { +; CHECK-LABEL: @hoist_gep_iv_inc_drops_inbounds( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %p = phi i32* [ %base, %entry ], [ %p.next, %latch ] +; CHECK-NEXT: %p.next = getelementptr i32, i32* %p, i32 1 +; CHECK-NEXT: br i1 %c0, label %exit, label %latch +; CHECK: exit: +; CHECK-NEXT: %res = phi i32* [ %p.next, %loop ], [ %p.next, %latch ] +; CHECK-NEXT: ret i32* %res +entry: + br label %loop + +loop: + %p = phi i32* [ %base, %entry ], [ %p.next, %latch ] + %q = phi i32* [ %base, %entry ], [ %q.next, %latch ] + %q.next = getelementptr i32, i32* %q, i32 1 + br i1 %c0, label %exit, label %latch + +latch: + %p.next = getelementptr inbounds i32, i32* %p, i32 1 + br i1 %c1, label %loop, label %exit + +exit: + %res = phi i32* [ %q.next, %loop ], [ %q.next, %latch ] + ret i32* %res +} + +; Adapted from llvm/test/Transforms/IndVarSimplify/iv-poison.ll (PR59777), +; widened from i2 (INT2_MAX = 1) to i32 (INT32_MAX = 2147483647). +; Even when %i already dominates %i2, replacing %i2 with %i introduces a new +; use of %i, so nsw, which only %i carries, must be dropped. Both increments +; are nuw by then, so nuw is kept. +define i32 @iv_hoist_nsw_poison(i32 %arg) { +; CHECK-LABEL: @iv_hoist_nsw_poison( +; CHECK-NEXT: bb: +; CHECK-NEXT: br label %bb1 +; CHECK: bb1: +; CHECK-NEXT: %.07 = phi i32 [ 2147483647, %bb ], [ %i, %bb1 ] +; CHECK-NEXT: %i = add nuw i32 %.07, 1 +; CHECK-NEXT: %.not.not = icmp ult i32 %.07, %arg +; CHECK-NEXT: br i1 %.not.not, label %common.ret, label %bb1 +; CHECK: common.ret: +; CHECK-NEXT: %i2.lcssa = phi i32 [ %i, %bb1 ] +; CHECK-NEXT: ret i32 %i2.lcssa +bb: + br label %bb1 + +bb1: + %.07 = phi i32 [ 2147483647, %bb ], [ %i, %bb1 ] + %.0 = phi i32 [ 2147483647, %bb ], [ %i2, %bb1 ] + %i = add nsw i32 %.07, 1 + %i2 = add i32 %.0, 1 + %.not.not = icmp ult i32 %.07, %arg + br i1 %.not.not, label %common.ret, label %bb1 + +common.ret: + ret i32 %i2 +} + +; Adapted from llvm/test/Transforms/IndVarSimplify/iv-poison.ll, widened from +; i4 to i32. The flagless %iv.0.next replaces %iv.1.next and must not become +; nsw. +define i32 @iv_hoist_nsw_poison2(i32 %end, i32 %start) { +; CHECK-LABEL: @iv_hoist_nsw_poison2( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %iv.0 = phi i32 [ %start, %entry ], [ %iv.0.next, %loop ] +; CHECK-NEXT: %iv.0.next = add i32 %iv.0, 1 +; CHECK-NEXT: %.not.not = icmp ult i32 %iv.0, %end +; CHECK-NEXT: br i1 %.not.not, label %exit, label %loop +; CHECK: exit: +; CHECK-NEXT: %iv.1.next.lcssa = phi i32 [ %iv.0.next, %loop ] +; CHECK-NEXT: ret i32 %iv.1.next.lcssa +entry: + br label %loop + +loop: + %iv.0 = phi i32 [ %start, %entry ], [ %iv.0.next, %loop ] + %iv.1 = phi i32 [ %start, %entry ], [ %iv.1.next, %loop ] + %iv.0.next = add i32 %iv.0, 1 + %iv.1.next = add nsw i32 %iv.1, 1 + %.not.not = icmp ult i32 %iv.0, %end + br i1 %.not.not, label %exit, label %loop + +exit: + ret i32 %iv.1.next +} diff --git a/tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll b/tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll new file mode 100644 index 00000000000..a4b2b8a636c --- /dev/null +++ b/tools/clang/test/DXC/Passes/ScalarEvolution/post-inc-nowrap-flags.ll @@ -0,0 +1,86 @@ +; RUN: %dxopt %s -analyze -scalar-evolution | FileCheck %s + +; The no-wrap flags of an IV increment may only be applied to the +; post-increment add recurrence when overflowing the increment would be +; undefined behavior: the loop's only exiting block is the latch, the latch +; condition depends on the increment, and nothing in the loop may throw. +; Otherwise other instructions computing the same value, on iterations where +; the increment does not execute, would inherit its flags. +; Partial backport of PR27315: +; https://github.com/llvm/llvm-project/commit/7e4a64167d4d2e7b0b680fae1706182223047af1 + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +declare void @may_throw() + +; CHECK-LABEL: Classifying expressions for: @latch_exit_uses_inc +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @latch_exit_uses_inc(i32 %start, i32 %n) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %loop ] + %inc = add nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + +; CHECK-LABEL: Classifying expressions for: @header_exit_guards_inc +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @header_exit_guards_inc(i32 %start, i1 %c) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %latch ] + %cond = icmp sgt i32 %i, 9 + br i1 %cond, label %exit, label %latch + +latch: + %inc = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + ret void +} + +; CHECK-LABEL: Classifying expressions for: @latch_exit_ignores_inc +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @latch_exit_ignores_inc(i32 %start, i1 %c) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %loop ] + %inc = add nsw i32 %i, 1 + br i1 %c, label %loop, label %exit + +exit: + ret void +} + +; CHECK-LABEL: Classifying expressions for: @loop_may_throw +; CHECK: %inc = add nsw i32 %i, 1 +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> +define void @loop_may_throw(i32 %start, i32 %n) { +entry: + br label %loop + +loop: + %i = phi i32 [ %start, %entry ], [ %inc, %loop ] + call void @may_throw() + %inc = add nsw i32 %i, 1 + %cmp = icmp slt i32 %inc, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +}