Final sync release branch to main

.github/workflows/clang-format-checker.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install clang-format
         uses: aminya/setup-cpp@8170d66c458f4a045220b7b0966c10940bb2a15d # v1.8.1
         with:
-          clangformat: 17.0.1
+          clangformat: 17.0.6
 
       - name: Check code formatting
         env:

docs/ReleaseNotes.md

@@ -19,6 +19,14 @@ The included licenses apply to the following files:
 
 ### Upcoming Release
 
+#### SPIR-V
+
+- Added `-fvk-resource-heap-stride` and `-fvk-sampler-heap-stride`, which
+  override the computed `ArrayStrideIdEXT` of the resource and sampler
+  descriptor heaps with a literal `ArrayStride`. The value must be a power of
+  two in `[8, 256]`. Both flags require `-spirv` and
+  `-fspv-use-descriptor-heap` and are rejected otherwise.
+
 Place release notes for the upcoming release below this line and remove this
 line upon naming the release. Refer to previous for appropriate section names.
 

docs/SPIR-V.rst

@@ -2102,9 +2102,11 @@ objects as untyped variables in ``UniformConstant`` storage class:
 The concrete descriptor type is selected at each heap access. For image,
 sampler, and texel buffer resources, DXC forms a runtime array of that
 descriptor type, decorates the array with a byte stride, and uses
-``OpUntypedAccessChainKHR`` followed by ``OpLoad``. The stride is an
-``ArrayStrideIdEXT`` decoration referencing a specialization constant rather
-than a literal ``ArrayStride`` (see `Descriptor heap array stride`_ below):
+``OpUntypedAccessChainKHR`` followed by ``OpLoad``. By default the stride is an
+``ArrayStrideIdEXT`` decoration referencing a constant built from
+``OpConstantSizeOfEXT``; a literal ``ArrayStride`` is emitted only when the
+stride is overridden on the command line (see `Descriptor heap array stride`_
+below):
 
 .. code:: spirv
 
@@ -2224,10 +2226,10 @@ Three further restrictions on heap accesses produce diagnostics:
 Descriptor heap array stride
 ++++++++++++++++++++++++++++
 
-All resource heap runtime arrays share a single ``ArrayStrideIdEXT`` decoration
-rather than a literal ``ArrayStride``, because descriptor sizes are not known
-until pipeline creation. The shared value is built from ``OpConstantSizeOfEXT``
-and ``OpSpecConstantOp`` and evaluates to
+By default, all resource heap runtime arrays share a single ``ArrayStrideIdEXT``
+decoration rather than a literal ``ArrayStride``, because descriptor sizes are
+not known until pipeline creation. The shared value is built from
+``OpConstantSizeOfEXT`` and ``OpSpecConstantOp`` and evaluates to
 ``max(sizeof(image_descriptor), sizeof(buffer_descriptor))``. The sampler heap
 carries its own ``ArrayStrideIdEXT`` equal to ``sizeof(sampler_descriptor)``.
 
@@ -2238,9 +2240,10 @@ types the shader actually uses; a module will normally contain both the
 placeholder type and the distinct image types its heap accesses lower to.
 
 The stride is a property of the heap layout, not of the shader's loads. DXC
-decides it before translating any function, from the shader stage and the
-``-fspv-extension`` list. When acceleration structure descriptors may be on the
-resource heap, the formula expands to a three-way max
+decides it before translating any function, either from explicit user input
+(``-fvk-resource-heap-stride`` and ``-fvk-sampler-heap-stride``, see below) or
+by calculating it from the shader stage and the ``-fspv-extension`` list. When
+acceleration structure descriptors may be on the resource heap, the formula expands to a three-way max
 ``max(max(sizeof(image_descriptor), sizeof(buffer_descriptor)), sizeof(acceleration_structure))``.
 A ray-tracing shader that only heap-loads a texture still gets the three-way
 max. DXC adds the acceleration structure term when either
@@ -2256,8 +2259,29 @@ of the listed extension (``RayQueryKHR``, ``RayTracingKHR`` or
 ``RayTracingNV``); if several are listed, ``SPV_KHR_ray_query`` is preferred,
 then ``SPV_KHR_ray_tracing``, then ``SPV_NV_ray_tracing``. A shader that is not
 a ray-tracing stage and heap-loads a ``RaytracingAccelerationStructure``
-without such an extension is rejected, because its stride would not cover the
-acceleration structure descriptor.
+without such an extension or ``-fvk-resource-heap-stride`` is rejected, because
+its stride would not cover the acceleration structure descriptor.
+
+The computed stride can be replaced with a literal using
+``-fvk-resource-heap-stride <N>`` and ``-fvk-sampler-heap-stride <N>``. They
+emit ``OpDecorate <array> ArrayStride N`` on every runtime array of the resource
+heap and of the sampler heap respectively. ``N`` must be a power of two in the
+inclusive range [8, 256]. Both flags require ``-spirv`` and
+``-fspv-use-descriptor-heap``, and are rejected otherwise because they would
+have no effect.
+
+For a heap with an override, no ``ArrayStrideIdEXT`` is emitted and no
+``OpConstantSizeOfEXT`` is built. The flags are independent: the other heap
+keeps its computed stride.
+
+The literal is not validated against the descriptor sizes of the target
+implementation. It must match the stride at which the application lays out
+descriptors in the heap, and be at least the size of the largest descriptor
+that may appear in it; otherwise heap indices select the wrong descriptors or
+read out of bounds.
+``-fvk-resource-heap-stride`` also disables the acceleration structure check
+above, so the stride must cover acceleration structure descriptors when the
+shader loads them from the heap.
 
 HLSL Expressions
 ================

include/dxc/Support/HLSLOptions.td

@@ -454,6 +454,10 @@ def fvk_bind_sampler_heap : MultiArg<["-"], "fvk-bind-sampler-heap", 2>, MetaVar
   HelpText<"Specify Vulkan binding number and set number for the sampler heap.">;
 def fvk_bind_counter_heap : MultiArg<["-"], "fvk-bind-counter-heap", 2>, MetaVarName<"<binding> <set>">, Group<spirv_Group>, Flags<[CoreOption, DriverOption]>,
   HelpText<"Specify Vulkan binding number and set number for the counter heap.">;
+def fvk_resource_heap_stride : Separate<["-"], "fvk-resource-heap-stride">, MetaVarName<"<stride>">, Group<spirv_Group>, Flags<[CoreOption, DriverOption]>,
+  HelpText<"Override the resource descriptor heap array stride with a literal ArrayStride of <stride> bytes. Requires -spirv and -fspv-use-descriptor-heap. Must be a power of 2 in [8, 256].">;
+def fvk_sampler_heap_stride : Separate<["-"], "fvk-sampler-heap-stride">, MetaVarName<"<stride>">, Group<spirv_Group>, Flags<[CoreOption, DriverOption]>,
+  HelpText<"Override the sampler descriptor heap array stride with a literal ArrayStride of <stride> bytes. Requires -spirv and -fspv-use-descriptor-heap. Must be a power of 2 in [8, 256].">;
 // SPIRV Change Ends
 
 //////////////////////////////////////////////////////////////////////////////

include/dxc/Support/SPIRVOptions.h

@@ -110,6 +110,12 @@ struct SpirvCodeGenOptions {
   std::optional<BindingInfo> samplerHeapBinding;
   std::optional<BindingInfo> counterHeapBinding;
 
+  // -fvk-resource-heap-stride / -fvk-sampler-heap-stride
+  // When set, every runtime array of that heap gets a literal ArrayStride of
+  // this many bytes (a power of 2 in [8, 256]) instead of ArrayStrideIdEXT.
+  std::optional<uint32_t> resourceHeapStride;
+  std::optional<uint32_t> samplerHeapStride;
+
   bool signaturePacking =
       false; ///< Whether signature packing is enabled or not
 

lib/Analysis/ScalarEvolution.cpp

@@ -2941,11 +2941,14 @@ ScalarEvolution::getGEPExpr(Type *PointeeType, const SCEV *BaseExpr,
   // getSCEV(Base)->getType() has the same address space as Base->getType()
   // because SCEV::getType() preserves the address space.
   Type *IntPtrTy = getEffectiveSCEVType(BaseExpr->getType());
-  // FIXME(PR23527): Don't blindly transfer the inbounds flag from the GEP
-  // instruction to its SCEV, because the Instruction may be guarded by control
-  // flow and the no-overflow bits may not be valid for the expression in any
-  // context.
-  SCEV::NoWrapFlags Wrap = InBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
+  // HLSL Change Begin - Partial backport PR23527 (D109789):
+  // https://github.com/llvm/llvm-project/commit/2ca8a3f2132ec7d46151b4553ee166c4e635fd70
+  // Don't blindly transfer the inbounds flag from the GEP instruction to its
+  // SCEV, because the instruction may be guarded by control flow and the
+  // no-overflow bits may not be valid for the expression in any context.
+  // Match createSCEV's handling of Add and Mul by not propagating inbounds.
+  SCEV::NoWrapFlags Wrap = SCEV::FlagAnyWrap;
+  // HLSL Change End - Partial backport PR23527 (D109789)
 
   const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
   // The address space is unimportant. The first thing we do on CurTy is getting

lib/DxcSupport/HLSLOptions.cpp

@@ -357,6 +357,52 @@ handleFixedBinding(const InputArgList &args, OptSpecifier id,
   return true;
 }
 
+// Parses the single-integer descriptor-heap stride flag |id| in |args|. If
+// present, requires -spirv and -fspv-use-descriptor-heap, validates that the
+// value is a power of 2 in [8, 256] and stores it in |stride|. Returns true on
+// success (including when the flag is absent). Returns false and writes to
+// |errors| if a requirement is not met or the value is malformed or invalid,
+// using |name| as the pretty flag name.
+static bool handleHeapStride(const InputArgList &args, OptSpecifier id,
+                             std::optional<uint32_t> *stride,
+                             llvm::StringRef name, llvm::raw_ostream &errors) {
+  Arg *arg = args.getLastArg(id);
+  if (!arg) {
+    *stride = std::nullopt;
+    return true;
+  }
+
+  if (!args.hasArg(OPT_spirv)) {
+    errors << name << " requires -spirv\n";
+    return false;
+  }
+
+  // The stride is only consumed by the native heap lowering. Without it the
+  // override would be accepted and silently have no effect.
+  if (!args.hasFlag(OPT_fspv_use_descriptor_heap, OPT_fspv_use_emulated_heap,
+                    false)) {
+    errors << name << " requires -fspv-use-descriptor-heap\n";
+    return false;
+  }
+
+  llvm::StringRef value = arg->getValue();
+  uint32_t number = 0;
+  if (value.getAsInteger(10, number)) {
+    errors << "invalid " << name << " argument: '" << value << "'\n";
+    return false;
+  }
+  // Power of 2 in [8, 256] inclusive.
+  if (number < 8 || number > 256 || (number & (number - 1)) != 0) {
+    errors << name
+           << " must be a power of 2 between 8 and 256 (inclusive); got "
+           << value << "\n";
+    return false;
+  }
+
+  *stride = number;
+  return true;
+}
+
 // Check if any options that are unsupported with SPIR-V are used.
 static bool hasUnsupportedSpirvOption(const InputArgList &args,
                                       llvm::raw_ostream &errors) {
@@ -1175,6 +1221,16 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
     return 1;
   }
 
+  bool strideOk = true;
+  strideOk &= handleHeapStride(Args, OPT_fvk_resource_heap_stride,
+                               &opts.SpirvOptions.resourceHeapStride,
+                               "-fvk-resource-heap-stride", errors);
+  strideOk &= handleHeapStride(Args, OPT_fvk_sampler_heap_stride,
+                               &opts.SpirvOptions.samplerHeapStride,
+                               "-fvk-sampler-heap-stride", errors);
+  if (!strideOk)
+    return 1;
+
   for (const Arg *A : Args.filtered(OPT_fspv_extension_EQ)) {
     opts.SpirvOptions.allowedExtensions.push_back(A->getValue());
   }
@@ -1316,7 +1372,9 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
       !Args.getLastArgValue(OPT_fvk_u_shift).empty() ||
       !Args.getLastArgValue(OPT_fvk_bind_resource_heap).empty() ||
       !Args.getLastArgValue(OPT_fvk_bind_sampler_heap).empty() ||
-      !Args.getLastArgValue(OPT_fvk_bind_counter_heap).empty()) {
+      !Args.getLastArgValue(OPT_fvk_bind_counter_heap).empty() ||
+      !Args.getLastArgValue(OPT_fvk_resource_heap_stride).empty() ||
+      !Args.getLastArgValue(OPT_fvk_sampler_heap_stride).empty()) {
     errors << "SPIR-V CodeGen not available. "
               "Please recompile with -DENABLE_SPIRV_CODEGEN=ON.";
     return 1;

lib/DxilContainer/DxilContainerAssembler.cpp

@@ -36,6 +36,7 @@
 #include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Support/MD5.h"
@@ -875,9 +876,48 @@ class LinAlgRuntimeInfoCollector {
     return GetScalarComponentType(Ty, IsSigned);
   }
 
+  static bool CollectMatVecStoredValues(AllocaInst *Allocation,
+                                        Type *MatrixType,
+                                        SmallVectorImpl<Value *> &Matrices) {
+    SmallVector<Value *, 8> Worklist(1, Allocation);
+    SmallPtrSet<Value *, 8> Visited;
+    bool HasStore = false;
+    while (!Worklist.empty()) {
+      Value *Pointer = Worklist.pop_back_val();
+      if (!Visited.insert(Pointer).second)
+        continue;
+
+      for (User *U : Pointer->users()) {
+        if (isa<GetElementPtrInst>(U) || isa<BitCastInst>(U)) {
+          Worklist.push_back(U);
+          continue;
+        }
+        if (isa<LoadInst>(U))
+          continue;
+        if (StoreInst *Store = dyn_cast<StoreInst>(U)) {
+          if (Store->getPointerOperand() != Pointer ||
+              Store->getValueOperand()->getType() != MatrixType)
+            return false;
+          Matrices.push_back(Store->getValueOperand());
+          HasStore = true;
+          continue;
+        }
+        if (IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(U)) {
+          if (Intrinsic->getIntrinsicID() == Intrinsic::lifetime_start ||
+              Intrinsic->getIntrinsicID() == Intrinsic::lifetime_end ||
+              isa<DbgInfoIntrinsic>(Intrinsic))
+            continue;
+        }
+        return false;
+      }
+    }
+    return HasStore;
+  }
+
   uint8_t GetMatVecLayoutFlags(Value *Matrix) const {
     SmallVector<Value *, 8> Worklist(1, Matrix);
     SmallPtrSet<Value *, 8> Visited;
+    std::set<std::pair<AllocaInst *, Type *>> VisitedAllocations;
     uint8_t Flags = 0;
     // Default to a conservative flagset when it's not possible to determine
     // the true flags
@@ -900,6 +940,29 @@ class LinAlgRuntimeInfoCollector {
         continue;
       }
 
+      if (LoadInst *Load = dyn_cast<LoadInst>(V)) {
+        Value *Pointer = Load->getPointerOperand();
+        while (true) {
+          if (GEPOperator *GEP = dyn_cast<GEPOperator>(Pointer))
+            Pointer = GEP->getPointerOperand();
+          else if (BitCastOperator *Cast = dyn_cast<BitCastOperator>(Pointer))
+            Pointer = Cast->getOperand(0);
+          else
+            break;
+        }
+        AllocaInst *Allocation = dyn_cast<AllocaInst>(Pointer);
+        if (!Allocation)
+          return UnknownLayoutFlags;
+        // Add all stores into the pointer to the matrix tracing worklist
+        // regardless of if they are overwritten or not directly accessed.
+        // Since we can't statically resolve a dynamic index we must assume
+        // all indexes are used.
+        if (VisitedAllocations.insert({Allocation, Load->getType()}).second &&
+            !CollectMatVecStoredValues(Allocation, Load->getType(), Worklist))
+          return UnknownLayoutFlags;
+        continue;
+      }
+
       if (CallInst *CI = dyn_cast<CallInst>(V)) {
         DXIL::OpCode OpCode = OP::getOpCode(CI);
         if (OpCode == DXIL::OpCode::LinAlgCopyConvertMatrix) {

lib/Transforms/InstCombine/InstCombineCompares.cpp

@@ -3360,11 +3360,21 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
             APInt AP2Abs = C2->getValue().abs();
             if (AP1Abs.uge(AP2Abs)) {
               ConstantInt *C3 = Builder->getInt(AP1 - AP2);
-              Value *NewAdd = Builder->CreateNSWAdd(A, C3);
+              // HLSL Change Begin - Backport (D100095)
+              // https://github.com/llvm/llvm-project/commit/29f189f90d44
+              bool HasNUW = BO0->hasNoUnsignedWrap() && C3->getValue().ule(AP1);
+              bool HasNSW = BO0->hasNoSignedWrap();
+              Value *NewAdd = Builder->CreateAdd(A, C3, "", HasNUW, HasNSW);
+              // HLSL Change End
               return new ICmpInst(Pred, NewAdd, C);
             } else {
               ConstantInt *C3 = Builder->getInt(AP2 - AP1);
-              Value *NewAdd = Builder->CreateNSWAdd(C, C3);
+              // HLSL Change Begin - Backport (D100095)
+              // https://github.com/llvm/llvm-project/commit/29f189f90d44
+              bool HasNUW = BO1->hasNoUnsignedWrap() && C3->getValue().ule(AP2);
+              bool HasNSW = BO1->hasNoSignedWrap();
+              Value *NewAdd = Builder->CreateAdd(C, C3, "", HasNUW, HasNSW);
+              // HLSL Change End
               return new ICmpInst(Pred, A, NewAdd);
             }
           }

lib/Transforms/Scalar/GVN.cpp

@@ -2665,9 +2665,15 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
       PHINode::Create(CurInst->getType(), predMap.size(),
                       CurInst->getName() + ".pre-phi", CurrentBlock->begin());
   for (unsigned i = 0, e = predMap.size(); i != e; ++i) {
-    if (Value *V = predMap[i].first)
+    if (Value *V = predMap[i].first) {
+      // HLSL Change Begin - Backport
+      // https://github.com/llvm/llvm-project/commit/722339e40562 (D39637)
+      // If we use an existing value in this phi, we have to patch the original
+      // value because the phi will be used to replace a later value.
+      patchReplacementInstruction(CurInst, V);
+      // HLSL Change End
       Phi->addIncoming(V, predMap[i].second);
-    else
+    } else
       Phi->addIncoming(PREInstr, PREPred);
   }
 

lib/Transforms/Scalar/ScalarReplAggregatesHLSL.cpp

@@ -705,6 +705,10 @@ void SplitCpy(Type *Ty, Value *Dest, Value *Src,
               const DxilFieldAnnotation *fieldAnnotation,
               const bool bEltMemCpy = true) {
   if (PointerType *PT = dyn_cast<PointerType>(Ty)) {
+    if (!idxList.empty()) {
+      SimpleCopy(Dest, Src, idxList, Builder);
+      return;
+    }
     Constant *idx = Constant::getIntegerValue(
         IntegerType::get(Ty->getContext(), 32), APInt(32, 0));
     idxList.emplace_back(idx);

lib/Transforms/Utils/SimplifyIndVar.cpp

@@ -168,12 +168,18 @@ void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) {
 
   // If the condition is always true or always false, replace it with
   // a constant value.
-  if (SE->isKnownPredicate(Pred, S, X))
+  // HLSL Change Begin - Forget cached SCEVs of ICmp users before RAUW. Backport
+  // https://github.com/llvm/llvm-project/commit/fd236772f5665ce9e55f85ac992be6c22a918d3e
+  if (SE->isKnownPredicate(Pred, S, X)) {
+    SE->forgetValue(ICmp);
     ICmp->replaceAllUsesWith(ConstantInt::getTrue(ICmp->getContext()));
-  else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X))
+  } else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X)) {
+    SE->forgetValue(ICmp);
     ICmp->replaceAllUsesWith(ConstantInt::getFalse(ICmp->getContext()));
-  else
+  } else {
     return;
+  }
+  // HLSL Change End
 
   DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n');
   ++NumElimCmp;