diff --git a/.github/workflows/clang-format-checker.yml b/.github/workflows/clang-format-checker.yml index 3a2aea849a5..e29745569b0 100644 --- a/.github/workflows/clang-format-checker.yml +++ b/.github/workflows/clang-format-checker.yml @@ -32,7 +32,7 @@ jobs: - name: Install clang-format uses: aminya/setup-cpp@8170d66c458f4a045220b7b0966c10940bb2a15d # v1.8.1 with: - clangformat: 17.0.1 + clangformat: 17.0.6 - name: Check code formatting env: diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index a84fefc05af..082a606687b 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -19,6 +19,14 @@ The included licenses apply to the following files: ### Upcoming Release +#### SPIR-V + +- Added `-fvk-resource-heap-stride` and `-fvk-sampler-heap-stride`, which + override the computed `ArrayStrideIdEXT` of the resource and sampler + descriptor heaps with a literal `ArrayStride`. The value must be a power of + two in `[8, 256]`. Both flags require `-spirv` and + `-fspv-use-descriptor-heap` and are rejected otherwise. + Place release notes for the upcoming release below this line and remove this line upon naming the release. Refer to previous for appropriate section names. diff --git a/docs/SPIR-V.rst b/docs/SPIR-V.rst index 45021716027..85807b201cc 100644 --- a/docs/SPIR-V.rst +++ b/docs/SPIR-V.rst @@ -2102,9 +2102,11 @@ objects as untyped variables in ``UniformConstant`` storage class: The concrete descriptor type is selected at each heap access. For image, sampler, and texel buffer resources, DXC forms a runtime array of that descriptor type, decorates the array with a byte stride, and uses -``OpUntypedAccessChainKHR`` followed by ``OpLoad``. The stride is an -``ArrayStrideIdEXT`` decoration referencing a specialization constant rather -than a literal ``ArrayStride`` (see `Descriptor heap array stride`_ below): +``OpUntypedAccessChainKHR`` followed by ``OpLoad``. By default the stride is an +``ArrayStrideIdEXT`` decoration referencing a constant built from +``OpConstantSizeOfEXT``; a literal ``ArrayStride`` is emitted only when the +stride is overridden on the command line (see `Descriptor heap array stride`_ +below): .. code:: spirv @@ -2224,10 +2226,10 @@ Three further restrictions on heap accesses produce diagnostics: Descriptor heap array stride ++++++++++++++++++++++++++++ -All resource heap runtime arrays share a single ``ArrayStrideIdEXT`` decoration -rather than a literal ``ArrayStride``, because descriptor sizes are not known -until pipeline creation. The shared value is built from ``OpConstantSizeOfEXT`` -and ``OpSpecConstantOp`` and evaluates to +By default, all resource heap runtime arrays share a single ``ArrayStrideIdEXT`` +decoration rather than a literal ``ArrayStride``, because descriptor sizes are +not known until pipeline creation. The shared value is built from +``OpConstantSizeOfEXT`` and ``OpSpecConstantOp`` and evaluates to ``max(sizeof(image_descriptor), sizeof(buffer_descriptor))``. The sampler heap carries its own ``ArrayStrideIdEXT`` equal to ``sizeof(sampler_descriptor)``. @@ -2238,9 +2240,10 @@ types the shader actually uses; a module will normally contain both the placeholder type and the distinct image types its heap accesses lower to. The stride is a property of the heap layout, not of the shader's loads. DXC -decides it before translating any function, from the shader stage and the -``-fspv-extension`` list. When acceleration structure descriptors may be on the -resource heap, the formula expands to a three-way max +decides it before translating any function, either from explicit user input +(``-fvk-resource-heap-stride`` and ``-fvk-sampler-heap-stride``, see below) or +by calculating it from the shader stage and the ``-fspv-extension`` list. When +acceleration structure descriptors may be on the resource heap, the formula expands to a three-way max ``max(max(sizeof(image_descriptor), sizeof(buffer_descriptor)), sizeof(acceleration_structure))``. A ray-tracing shader that only heap-loads a texture still gets the three-way max. DXC adds the acceleration structure term when either @@ -2256,8 +2259,29 @@ of the listed extension (``RayQueryKHR``, ``RayTracingKHR`` or ``RayTracingNV``); if several are listed, ``SPV_KHR_ray_query`` is preferred, then ``SPV_KHR_ray_tracing``, then ``SPV_NV_ray_tracing``. A shader that is not a ray-tracing stage and heap-loads a ``RaytracingAccelerationStructure`` -without such an extension is rejected, because its stride would not cover the -acceleration structure descriptor. +without such an extension or ``-fvk-resource-heap-stride`` is rejected, because +its stride would not cover the acceleration structure descriptor. + +The computed stride can be replaced with a literal using +``-fvk-resource-heap-stride `` and ``-fvk-sampler-heap-stride ``. They +emit ``OpDecorate ArrayStride N`` on every runtime array of the resource +heap and of the sampler heap respectively. ``N`` must be a power of two in the +inclusive range [8, 256]. Both flags require ``-spirv`` and +``-fspv-use-descriptor-heap``, and are rejected otherwise because they would +have no effect. + +For a heap with an override, no ``ArrayStrideIdEXT`` is emitted and no +``OpConstantSizeOfEXT`` is built. The flags are independent: the other heap +keeps its computed stride. + +The literal is not validated against the descriptor sizes of the target +implementation. It must match the stride at which the application lays out +descriptors in the heap, and be at least the size of the largest descriptor +that may appear in it; otherwise heap indices select the wrong descriptors or +read out of bounds. +``-fvk-resource-heap-stride`` also disables the acceleration structure check +above, so the stride must cover acceleration structure descriptors when the +shader loads them from the heap. HLSL Expressions ================ diff --git a/include/dxc/Support/HLSLOptions.td b/include/dxc/Support/HLSLOptions.td index bfd986ad3ea..0bde31edc6a 100644 --- a/include/dxc/Support/HLSLOptions.td +++ b/include/dxc/Support/HLSLOptions.td @@ -454,6 +454,10 @@ def fvk_bind_sampler_heap : MultiArg<["-"], "fvk-bind-sampler-heap", 2>, MetaVar HelpText<"Specify Vulkan binding number and set number for the sampler heap.">; def fvk_bind_counter_heap : MultiArg<["-"], "fvk-bind-counter-heap", 2>, MetaVarName<" ">, Group, Flags<[CoreOption, DriverOption]>, HelpText<"Specify Vulkan binding number and set number for the counter heap.">; +def fvk_resource_heap_stride : Separate<["-"], "fvk-resource-heap-stride">, MetaVarName<"">, Group, Flags<[CoreOption, DriverOption]>, + HelpText<"Override the resource descriptor heap array stride with a literal ArrayStride of bytes. Requires -spirv and -fspv-use-descriptor-heap. Must be a power of 2 in [8, 256].">; +def fvk_sampler_heap_stride : Separate<["-"], "fvk-sampler-heap-stride">, MetaVarName<"">, Group, Flags<[CoreOption, DriverOption]>, + HelpText<"Override the sampler descriptor heap array stride with a literal ArrayStride of bytes. Requires -spirv and -fspv-use-descriptor-heap. Must be a power of 2 in [8, 256].">; // SPIRV Change Ends ////////////////////////////////////////////////////////////////////////////// diff --git a/include/dxc/Support/SPIRVOptions.h b/include/dxc/Support/SPIRVOptions.h index 0253caba63c..c12e6415736 100644 --- a/include/dxc/Support/SPIRVOptions.h +++ b/include/dxc/Support/SPIRVOptions.h @@ -110,6 +110,12 @@ struct SpirvCodeGenOptions { std::optional samplerHeapBinding; std::optional counterHeapBinding; + // -fvk-resource-heap-stride / -fvk-sampler-heap-stride + // When set, every runtime array of that heap gets a literal ArrayStride of + // this many bytes (a power of 2 in [8, 256]) instead of ArrayStrideIdEXT. + std::optional resourceHeapStride; + std::optional samplerHeapStride; + bool signaturePacking = false; ///< Whether signature packing is enabled or not diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index 72f5a24010e..cde7b4a3a85 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -2941,11 +2941,14 @@ ScalarEvolution::getGEPExpr(Type *PointeeType, const SCEV *BaseExpr, // getSCEV(Base)->getType() has the same address space as Base->getType() // because SCEV::getType() preserves the address space. Type *IntPtrTy = getEffectiveSCEVType(BaseExpr->getType()); - // FIXME(PR23527): Don't blindly transfer the inbounds flag from the GEP - // instruction to its SCEV, because the Instruction may be guarded by control - // flow and the no-overflow bits may not be valid for the expression in any - // context. - SCEV::NoWrapFlags Wrap = InBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap; + // HLSL Change Begin - Partial backport PR23527 (D109789): + // https://github.com/llvm/llvm-project/commit/2ca8a3f2132ec7d46151b4553ee166c4e635fd70 + // Don't blindly transfer the inbounds flag from the GEP instruction to its + // SCEV, because the instruction may be guarded by control flow and the + // no-overflow bits may not be valid for the expression in any context. + // Match createSCEV's handling of Add and Mul by not propagating inbounds. + SCEV::NoWrapFlags Wrap = SCEV::FlagAnyWrap; + // HLSL Change End - Partial backport PR23527 (D109789) const SCEV *TotalOffset = getConstant(IntPtrTy, 0); // The address space is unimportant. The first thing we do on CurTy is getting diff --git a/lib/DxcSupport/HLSLOptions.cpp b/lib/DxcSupport/HLSLOptions.cpp index 1630243aab6..62cdf3ca11a 100644 --- a/lib/DxcSupport/HLSLOptions.cpp +++ b/lib/DxcSupport/HLSLOptions.cpp @@ -357,6 +357,52 @@ handleFixedBinding(const InputArgList &args, OptSpecifier id, return true; } +// Parses the single-integer descriptor-heap stride flag |id| in |args|. If +// present, requires -spirv and -fspv-use-descriptor-heap, validates that the +// value is a power of 2 in [8, 256] and stores it in |stride|. Returns true on +// success (including when the flag is absent). Returns false and writes to +// |errors| if a requirement is not met or the value is malformed or invalid, +// using |name| as the pretty flag name. +static bool handleHeapStride(const InputArgList &args, OptSpecifier id, + std::optional *stride, + llvm::StringRef name, llvm::raw_ostream &errors) { + Arg *arg = args.getLastArg(id); + if (!arg) { + *stride = std::nullopt; + return true; + } + + if (!args.hasArg(OPT_spirv)) { + errors << name << " requires -spirv\n"; + return false; + } + + // The stride is only consumed by the native heap lowering. Without it the + // override would be accepted and silently have no effect. + if (!args.hasFlag(OPT_fspv_use_descriptor_heap, OPT_fspv_use_emulated_heap, + false)) { + errors << name << " requires -fspv-use-descriptor-heap\n"; + return false; + } + + llvm::StringRef value = arg->getValue(); + uint32_t number = 0; + if (value.getAsInteger(10, number)) { + errors << "invalid " << name << " argument: '" << value << "'\n"; + return false; + } + // Power of 2 in [8, 256] inclusive. + if (number < 8 || number > 256 || (number & (number - 1)) != 0) { + errors << name + << " must be a power of 2 between 8 and 256 (inclusive); got " + << value << "\n"; + return false; + } + + *stride = number; + return true; +} + // Check if any options that are unsupported with SPIR-V are used. static bool hasUnsupportedSpirvOption(const InputArgList &args, llvm::raw_ostream &errors) { @@ -1175,6 +1221,16 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude, return 1; } + bool strideOk = true; + strideOk &= handleHeapStride(Args, OPT_fvk_resource_heap_stride, + &opts.SpirvOptions.resourceHeapStride, + "-fvk-resource-heap-stride", errors); + strideOk &= handleHeapStride(Args, OPT_fvk_sampler_heap_stride, + &opts.SpirvOptions.samplerHeapStride, + "-fvk-sampler-heap-stride", errors); + if (!strideOk) + return 1; + for (const Arg *A : Args.filtered(OPT_fspv_extension_EQ)) { opts.SpirvOptions.allowedExtensions.push_back(A->getValue()); } @@ -1316,7 +1372,9 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude, !Args.getLastArgValue(OPT_fvk_u_shift).empty() || !Args.getLastArgValue(OPT_fvk_bind_resource_heap).empty() || !Args.getLastArgValue(OPT_fvk_bind_sampler_heap).empty() || - !Args.getLastArgValue(OPT_fvk_bind_counter_heap).empty()) { + !Args.getLastArgValue(OPT_fvk_bind_counter_heap).empty() || + !Args.getLastArgValue(OPT_fvk_resource_heap_stride).empty() || + !Args.getLastArgValue(OPT_fvk_sampler_heap_stride).empty()) { errors << "SPIR-V CodeGen not available. " "Please recompile with -DENABLE_SPIRV_CODEGEN=ON."; return 1; diff --git a/lib/DxilContainer/DxilContainerAssembler.cpp b/lib/DxilContainer/DxilContainerAssembler.cpp index db6fdc0044d..94569a5ceb8 100644 --- a/lib/DxilContainer/DxilContainerAssembler.cpp +++ b/lib/DxilContainer/DxilContainerAssembler.cpp @@ -36,6 +36,7 @@ #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/Support/MD5.h" @@ -875,9 +876,48 @@ class LinAlgRuntimeInfoCollector { return GetScalarComponentType(Ty, IsSigned); } + static bool CollectMatVecStoredValues(AllocaInst *Allocation, + Type *MatrixType, + SmallVectorImpl &Matrices) { + SmallVector Worklist(1, Allocation); + SmallPtrSet Visited; + bool HasStore = false; + while (!Worklist.empty()) { + Value *Pointer = Worklist.pop_back_val(); + if (!Visited.insert(Pointer).second) + continue; + + for (User *U : Pointer->users()) { + if (isa(U) || isa(U)) { + Worklist.push_back(U); + continue; + } + if (isa(U)) + continue; + if (StoreInst *Store = dyn_cast(U)) { + if (Store->getPointerOperand() != Pointer || + Store->getValueOperand()->getType() != MatrixType) + return false; + Matrices.push_back(Store->getValueOperand()); + HasStore = true; + continue; + } + if (IntrinsicInst *Intrinsic = dyn_cast(U)) { + if (Intrinsic->getIntrinsicID() == Intrinsic::lifetime_start || + Intrinsic->getIntrinsicID() == Intrinsic::lifetime_end || + isa(Intrinsic)) + continue; + } + return false; + } + } + return HasStore; + } + uint8_t GetMatVecLayoutFlags(Value *Matrix) const { SmallVector Worklist(1, Matrix); SmallPtrSet Visited; + std::set> VisitedAllocations; uint8_t Flags = 0; // Default to a conservative flagset when it's not possible to determine // the true flags @@ -900,6 +940,29 @@ class LinAlgRuntimeInfoCollector { continue; } + if (LoadInst *Load = dyn_cast(V)) { + Value *Pointer = Load->getPointerOperand(); + while (true) { + if (GEPOperator *GEP = dyn_cast(Pointer)) + Pointer = GEP->getPointerOperand(); + else if (BitCastOperator *Cast = dyn_cast(Pointer)) + Pointer = Cast->getOperand(0); + else + break; + } + AllocaInst *Allocation = dyn_cast(Pointer); + if (!Allocation) + return UnknownLayoutFlags; + // Add all stores into the pointer to the matrix tracing worklist + // regardless of if they are overwritten or not directly accessed. + // Since we can't statically resolve a dynamic index we must assume + // all indexes are used. + if (VisitedAllocations.insert({Allocation, Load->getType()}).second && + !CollectMatVecStoredValues(Allocation, Load->getType(), Worklist)) + return UnknownLayoutFlags; + continue; + } + if (CallInst *CI = dyn_cast(V)) { DXIL::OpCode OpCode = OP::getOpCode(CI); if (OpCode == DXIL::OpCode::LinAlgCopyConvertMatrix) { diff --git a/lib/Transforms/InstCombine/InstCombineCompares.cpp b/lib/Transforms/InstCombine/InstCombineCompares.cpp index bc296ab5e4e..a220dbbd9a9 100644 --- a/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -3360,11 +3360,21 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { APInt AP2Abs = C2->getValue().abs(); if (AP1Abs.uge(AP2Abs)) { ConstantInt *C3 = Builder->getInt(AP1 - AP2); - Value *NewAdd = Builder->CreateNSWAdd(A, C3); + // HLSL Change Begin - Backport (D100095) + // https://github.com/llvm/llvm-project/commit/29f189f90d44 + bool HasNUW = BO0->hasNoUnsignedWrap() && C3->getValue().ule(AP1); + bool HasNSW = BO0->hasNoSignedWrap(); + Value *NewAdd = Builder->CreateAdd(A, C3, "", HasNUW, HasNSW); + // HLSL Change End return new ICmpInst(Pred, NewAdd, C); } else { ConstantInt *C3 = Builder->getInt(AP2 - AP1); - Value *NewAdd = Builder->CreateNSWAdd(C, C3); + // HLSL Change Begin - Backport (D100095) + // https://github.com/llvm/llvm-project/commit/29f189f90d44 + bool HasNUW = BO1->hasNoUnsignedWrap() && C3->getValue().ule(AP2); + bool HasNSW = BO1->hasNoSignedWrap(); + Value *NewAdd = Builder->CreateAdd(C, C3, "", HasNUW, HasNSW); + // HLSL Change End return new ICmpInst(Pred, A, NewAdd); } } diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp index 12e848fb583..bc8efd78d68 100644 --- a/lib/Transforms/Scalar/GVN.cpp +++ b/lib/Transforms/Scalar/GVN.cpp @@ -2665,9 +2665,15 @@ bool GVN::performScalarPRE(Instruction *CurInst) { PHINode::Create(CurInst->getType(), predMap.size(), CurInst->getName() + ".pre-phi", CurrentBlock->begin()); for (unsigned i = 0, e = predMap.size(); i != e; ++i) { - if (Value *V = predMap[i].first) + if (Value *V = predMap[i].first) { + // HLSL Change Begin - Backport + // https://github.com/llvm/llvm-project/commit/722339e40562 (D39637) + // If we use an existing value in this phi, we have to patch the original + // value because the phi will be used to replace a later value. + patchReplacementInstruction(CurInst, V); + // HLSL Change End Phi->addIncoming(V, predMap[i].second); - else + } else Phi->addIncoming(PREInstr, PREPred); } diff --git a/lib/Transforms/Scalar/ScalarReplAggregatesHLSL.cpp b/lib/Transforms/Scalar/ScalarReplAggregatesHLSL.cpp index 11421f50676..0b69eeec89a 100644 --- a/lib/Transforms/Scalar/ScalarReplAggregatesHLSL.cpp +++ b/lib/Transforms/Scalar/ScalarReplAggregatesHLSL.cpp @@ -705,6 +705,10 @@ void SplitCpy(Type *Ty, Value *Dest, Value *Src, const DxilFieldAnnotation *fieldAnnotation, const bool bEltMemCpy = true) { if (PointerType *PT = dyn_cast(Ty)) { + if (!idxList.empty()) { + SimpleCopy(Dest, Src, idxList, Builder); + return; + } Constant *idx = Constant::getIntegerValue( IntegerType::get(Ty->getContext(), 32), APInt(32, 0)); idxList.emplace_back(idx); diff --git a/lib/Transforms/Utils/SimplifyIndVar.cpp b/lib/Transforms/Utils/SimplifyIndVar.cpp index 0d17526c981..aebfd23eccf 100644 --- a/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -168,12 +168,18 @@ void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) { // If the condition is always true or always false, replace it with // a constant value. - if (SE->isKnownPredicate(Pred, S, X)) + // HLSL Change Begin - Forget cached SCEVs of ICmp users before RAUW. Backport + // https://github.com/llvm/llvm-project/commit/fd236772f5665ce9e55f85ac992be6c22a918d3e + if (SE->isKnownPredicate(Pred, S, X)) { + SE->forgetValue(ICmp); ICmp->replaceAllUsesWith(ConstantInt::getTrue(ICmp->getContext())); - else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X)) + } else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X)) { + SE->forgetValue(ICmp); ICmp->replaceAllUsesWith(ConstantInt::getFalse(ICmp->getContext())); - else + } else { return; + } + // HLSL Change End DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n'); ++NumElimCmp; diff --git a/tools/clang/lib/SPIRV/SpirvEmitter.cpp b/tools/clang/lib/SPIRV/SpirvEmitter.cpp index 983735439ef..650f5982b87 100644 --- a/tools/clang/lib/SPIRV/SpirvEmitter.cpp +++ b/tools/clang/lib/SPIRV/SpirvEmitter.cpp @@ -5657,8 +5657,9 @@ SpirvInstruction *SpirvEmitter::emitDescriptorHeapBufferAccess( const BufferEXTType *bufferDescriptorType = spvContext.getBufferEXTType(bufferSC); - const SpirvType *arrayType = - getDescriptorHeapRuntimeArrayType(bufferDescriptorType); + // Buffer descriptors are always on the resource heap. + const SpirvType *arrayType = getDescriptorHeapRuntimeArrayType( + bufferDescriptorType, /*onSamplerHeap=*/false); SpirvUntypedAccessChainKHR *untypedAccessChainPtr = spvBuilder.createUntypedAccessChainKHR(untypedUniformConstantType, arrayType, heapVar, index, @@ -7357,13 +7358,18 @@ SpirvEmitter::doCXXOperatorCallExpr(const CXXOperatorCallExpr *expr, // The stride was decided without an acceleration structure term (not a // ray-tracing stage, no ray extension listed). Reject the load: the // heap would be indexed with a stride that may be too narrow. + // Exception: with -fvk-resource-heap-stride the stride is a literal + // chosen by the user, so there is no computed stride to widen. if (isRaytracingAccelerationStructure(resourceType) && - !spvBuilder.resourceHeapStrideIncludesAccelStruct()) { + !spvBuilder.resourceHeapStrideIncludesAccelStruct() && + !spirvOptions.resourceHeapStride.has_value()) { emitError("acceleration structure loaded from ResourceDescriptorHeap " "requires the resource heap stride to account for " "acceleration structure descriptors; compile with " - "-fspv-extension=SPV_KHR_ray_tracing or " - "-fspv-extension=SPV_KHR_ray_query", + "-fspv-extension=SPV_KHR_ray_tracing, " + "-fspv-extension=SPV_NV_ray_tracing, " + "-fspv-extension=SPV_KHR_ray_query, or " + "-fvk-resource-heap-stride", expr->getExprLoc()); return nullptr; } @@ -7381,8 +7387,9 @@ SpirvEmitter::doCXXOperatorCallExpr(const CXXOperatorCallExpr *expr, const SpirvType *handleType = lowerTypeVisitor.lowerType(resourceType, SpirvLayoutRule::Void, llvm::None, baseExpr->getExprLoc()); - const SpirvType *arrayType = - getDescriptorHeapRuntimeArrayType(handleType); + // The stride belongs to the heap being indexed. + const SpirvType *arrayType = getDescriptorHeapRuntimeArrayType( + handleType, isSamplerDescriptorHeap(decl)); SpirvUntypedAccessChainKHR *untypedAccessChainPtr = spvBuilder.createUntypedAccessChainKHR(untypedUniformConstantType, arrayType, var, index, @@ -9829,14 +9836,24 @@ void SpirvEmitter::createSpecConstant(const VarDecl *varDecl) { } const SpirvType * -SpirvEmitter::getDescriptorHeapRuntimeArrayType(const SpirvType *elemType) { +SpirvEmitter::getDescriptorHeapRuntimeArrayType(const SpirvType *elemType, + bool onSamplerHeap) { + // -fvk-{resource,sampler}-heap-stride replaces the computed stride: the array + // carries a literal ArrayStride and no ArrayStrideIdEXT. + const std::optional &cliStride = + onSamplerHeap ? spirvOptions.samplerHeapStride + : spirvOptions.resourceHeapStride; + if (cliStride.has_value()) + return spvContext.getRuntimeArrayType(elemType, *cliStride); + // Descriptor sizes are defined by the client API, so the stride is an // ArrayStrideIdEXT operand rather than a literal. The sampler heap holds one // descriptor type. The resource heap is a flat array in which any resource // descriptor may sit at any slot, so all its runtime arrays share one stride // (see SpirvBuilder::getResourceHeapArrayStride) instead of the stride of the - // accessed type. - SpirvInstruction *strideId = isa(elemType) + // accessed type. As with the literal override above, the stride follows the + // heap that is accessed, not the element type. + SpirvInstruction *strideId = onSamplerHeap ? spvBuilder.getSamplerHeapArrayStride() : spvBuilder.getResourceHeapArrayStride(); return spvContext.getRuntimeArrayType(elemType, llvm::None, strideId); diff --git a/tools/clang/lib/SPIRV/SpirvEmitter.h b/tools/clang/lib/SPIRV/SpirvEmitter.h index 56effc43308..f11fcfc8154 100644 --- a/tools/clang/lib/SPIRV/SpirvEmitter.h +++ b/tools/clang/lib/SPIRV/SpirvEmitter.h @@ -439,10 +439,13 @@ class SpirvEmitter : public ASTConsumer { /// Translates the given varDecl into a spec constant. void createSpecConstant(const VarDecl *varDecl); - /// Returns the OpTypeRuntimeArray for a descriptor heap array of elemType, - /// decorated with ArrayStrideIdEXT referencing an OpConstantSizeOfEXT of the - /// element (descriptor) type. - const SpirvType *getDescriptorHeapRuntimeArrayType(const SpirvType *elemType); + /// Returns the OpTypeRuntimeArray for a descriptor heap array of elemType. + /// onSamplerHeap selects which heap the descriptor is loaded from. If that + /// heap has a -fvk-{resource,sampler}-heap-stride override, the array carries + /// that literal ArrayStride; otherwise it is decorated with ArrayStrideIdEXT + /// referencing the shared stride for the heap. + const SpirvType *getDescriptorHeapRuntimeArrayType(const SpirvType *elemType, + bool onSamplerHeap); /// Emits the descriptor-heap buffer access for a buffer-like resource /// (StructuredBuffer/ByteAddressBuffer/ConstantBuffer/TextureBuffer + RW diff --git a/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.acceleration-structure.stride.error.hlsl b/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.acceleration-structure.stride.error.hlsl index 84c2cd40dbc..a2c1d2ddecd 100644 --- a/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.acceleration-structure.stride.error.hlsl +++ b/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.acceleration-structure.stride.error.hlsl @@ -8,6 +8,21 @@ // RUN: -fspv-extension=SPV_KHR_ray_query \ // RUN: -spirv %s | FileCheck %s --check-prefix=OK +// Verify -fvk-resource-heap-stride suppresses the diagnostic: the stride is a +// user-supplied literal, so there is no computed stride to widen. The +// acceleration structure runtime array must carry a literal ArrayStride (not +// ArrayStrideIdEXT) and no OpConstantSizeOfEXT may be emitted for the +// acceleration structure type. +// There is deliberately no -fspv-extension list: SPV_KHR_ray_query would make +// the pre-scan widen the stride by itself, so compiling would not depend on the +// override. The extensions this shader needs are enabled by default. +// RUN: %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap \ +// RUN: -fspv-target-env=vulkan1.3 \ +// RUN: -fvk-resource-heap-stride 256 \ +// RUN: -spirv %s > %t +// RUN: FileCheck %s --check-prefix=CLI < %t +// RUN: FileCheck %s --check-prefix=ABSENT < %t + // Verifies: an acceleration structure loaded from ResourceDescriptorHeap is // rejected when the resource-heap stride was not widened to include the // acceleration structure descriptor size. @@ -22,7 +37,7 @@ // shader also needs for RayQuery (SPV_KHR_ray_tracing alone does not provide // the RayQueryKHR capability), and must compile. -// CHECK: error: acceleration structure loaded from ResourceDescriptorHeap requires the resource heap stride to account for acceleration structure descriptors; compile with -fspv-extension=SPV_KHR_ray_tracing or -fspv-extension=SPV_KHR_ray_query +// CHECK: error: acceleration structure loaded from ResourceDescriptorHeap requires the resource heap stride to account for acceleration structure descriptors; compile with -fspv-extension=SPV_KHR_ray_tracing, -fspv-extension=SPV_NV_ray_tracing, -fspv-extension=SPV_KHR_ray_query, or -fvk-resource-heap-stride // The variable initialized from the rejected load must not cascade into a // fatal error that hides the diagnostic above. @@ -32,6 +47,17 @@ // OK-DAG: %[[Accel:[a-zA-Z0-9_]+]] = OpTypeAccelerationStructureKHR // OK-DAG: OpConstantSizeOfEXT %uint %[[Accel]] +// The acceleration structure runtime array carries the literal stride. +// CLI-DAG: %[[Accel:[a-zA-Z0-9_]+]] = OpTypeAccelerationStructureKHR +// CLI-DAG: %[[AccelArr:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[Accel]] +// CLI-DAG: OpDecorate %[[AccelArr]] ArrayStride 256 + +// No computed stride may appear anywhere in the module. This is a separate +// check, because a NOT after the last DAG match only covers the rest of the +// output, and the decorations come before the types. +// ABSENT-NOT: ArrayStrideIdEXT +// ABSENT-NOT: OpConstantSizeOfEXT + RWBuffer output : register(u0); [numthreads(1, 1, 1)] diff --git a/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.stride-cli-permute.hlsl b/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.stride-cli-permute.hlsl new file mode 100644 index 00000000000..551d10a0cc4 --- /dev/null +++ b/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.stride-cli-permute.hlsl @@ -0,0 +1,96 @@ +// Verifies: every legal CLI stride value reaches the literal ArrayStride of its +// own heap (full 6x6 cross product; the RS != SS cases show the heaps do not +// affect each other), and invalid values or missing prerequisite flags are +// rejected at option-parse time. +// +// [[RS]] = resource-heap stride, [[SS]] = sampler-heap stride (per-RUN via -D). + +// ---- Full 6x6 cross product of {8,16,32,64,128,256} x {8,16,32,64,128,256} ---- +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 8 -fvk-sampler-heap-stride 8 -spirv %s | FileCheck %s -DRS=8 -DSS=8 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 8 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s -DRS=8 -DSS=16 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 8 -fvk-sampler-heap-stride 32 -spirv %s | FileCheck %s -DRS=8 -DSS=32 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 8 -fvk-sampler-heap-stride 64 -spirv %s | FileCheck %s -DRS=8 -DSS=64 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 8 -fvk-sampler-heap-stride 128 -spirv %s | FileCheck %s -DRS=8 -DSS=128 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 8 -fvk-sampler-heap-stride 256 -spirv %s | FileCheck %s -DRS=8 -DSS=256 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 16 -fvk-sampler-heap-stride 8 -spirv %s | FileCheck %s -DRS=16 -DSS=8 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 16 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s -DRS=16 -DSS=16 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 16 -fvk-sampler-heap-stride 32 -spirv %s | FileCheck %s -DRS=16 -DSS=32 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 16 -fvk-sampler-heap-stride 64 -spirv %s | FileCheck %s -DRS=16 -DSS=64 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 16 -fvk-sampler-heap-stride 128 -spirv %s | FileCheck %s -DRS=16 -DSS=128 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 16 -fvk-sampler-heap-stride 256 -spirv %s | FileCheck %s -DRS=16 -DSS=256 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 32 -fvk-sampler-heap-stride 8 -spirv %s | FileCheck %s -DRS=32 -DSS=8 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 32 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s -DRS=32 -DSS=16 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 32 -fvk-sampler-heap-stride 32 -spirv %s | FileCheck %s -DRS=32 -DSS=32 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 32 -fvk-sampler-heap-stride 64 -spirv %s | FileCheck %s -DRS=32 -DSS=64 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 32 -fvk-sampler-heap-stride 128 -spirv %s | FileCheck %s -DRS=32 -DSS=128 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 32 -fvk-sampler-heap-stride 256 -spirv %s | FileCheck %s -DRS=32 -DSS=256 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 64 -fvk-sampler-heap-stride 8 -spirv %s | FileCheck %s -DRS=64 -DSS=8 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 64 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s -DRS=64 -DSS=16 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 64 -fvk-sampler-heap-stride 32 -spirv %s | FileCheck %s -DRS=64 -DSS=32 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 64 -fvk-sampler-heap-stride 64 -spirv %s | FileCheck %s -DRS=64 -DSS=64 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 64 -fvk-sampler-heap-stride 128 -spirv %s | FileCheck %s -DRS=64 -DSS=128 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 64 -fvk-sampler-heap-stride 256 -spirv %s | FileCheck %s -DRS=64 -DSS=256 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 8 -spirv %s | FileCheck %s -DRS=128 -DSS=8 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s -DRS=128 -DSS=16 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 32 -spirv %s | FileCheck %s -DRS=128 -DSS=32 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 64 -spirv %s | FileCheck %s -DRS=128 -DSS=64 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 128 -spirv %s | FileCheck %s -DRS=128 -DSS=128 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 256 -spirv %s | FileCheck %s -DRS=128 -DSS=256 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 256 -fvk-sampler-heap-stride 8 -spirv %s | FileCheck %s -DRS=256 -DSS=8 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 256 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s -DRS=256 -DSS=16 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 256 -fvk-sampler-heap-stride 32 -spirv %s | FileCheck %s -DRS=256 -DSS=32 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 256 -fvk-sampler-heap-stride 64 -spirv %s | FileCheck %s -DRS=256 -DSS=64 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 256 -fvk-sampler-heap-stride 128 -spirv %s | FileCheck %s -DRS=256 -DSS=128 +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 256 -fvk-sampler-heap-stride 256 -spirv %s | FileCheck %s -DRS=256 -DSS=256 + +// ---- Invalid values are rejected at option-parsing time (power-of-two in [8,256]) ---- +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 48 -spirv %s 2>&1 | FileCheck %s --check-prefix=BADRS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 4 -spirv %s 2>&1 | FileCheck %s --check-prefix=BADRS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 512 -spirv %s 2>&1 | FileCheck %s --check-prefix=BADRS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 0 -spirv %s 2>&1 | FileCheck %s --check-prefix=BADRS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-sampler-heap-stride 24 -spirv %s 2>&1 | FileCheck %s --check-prefix=BADSS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride abc -spirv %s 2>&1 | FileCheck %s --check-prefix=BADNUM + +// ---- Both flags invalid: both errors must be reported (no short-circuit) ---- +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 48 -fvk-sampler-heap-stride 24 -spirv %s 2>&1 | FileCheck %s --check-prefix=BOTH + +// ---- Flag without -spirv: "requires -spirv" diagnostic ---- +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 %s 2>&1 | FileCheck %s --check-prefix=NOSPIRVRS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-sampler-heap-stride 16 %s 2>&1 | FileCheck %s --check-prefix=NOSPIRVSS + +// ---- Flag without the native heap (plain -spirv or the emulated heap): rejected, since it would have no effect ---- +// RUN: not %dxc -T cs_6_6 -E main -fvk-resource-heap-stride 128 -spirv %s 2>&1 | FileCheck %s --check-prefix=NOHEAPRS +// RUN: not %dxc -T cs_6_6 -E main -fvk-sampler-heap-stride 16 -spirv %s 2>&1 | FileCheck %s --check-prefix=NOHEAPSS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-emulated-heap -fvk-resource-heap-stride 128 -spirv %s 2>&1 | FileCheck %s --check-prefix=NOHEAPRS +// RUN: not %dxc -T cs_6_6 -E main -fspv-use-emulated-heap -fvk-sampler-heap-stride 16 -spirv %s 2>&1 | FileCheck %s --check-prefix=NOHEAPSS + +// Capture each heap's runtime array by its element type so the two strides are +// checked independently. The Texture2D array (resource heap) must carry [[RS]]; +// the sampler array (sampler heap) must carry [[SS]]. +// CHECK-DAG: %[[TexType:[a-zA-Z0-9_]+]] = OpTypeImage %float 2D 2 0 0 1 Unknown +// CHECK-DAG: %[[SampType:[a-zA-Z0-9_]+]] = OpTypeSampler +// CHECK-DAG: %[[TexArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[TexType]]{{$}} +// CHECK-DAG: %[[SampArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[SampType]]{{$}} +// CHECK-DAG: OpDecorate %[[TexArray]] ArrayStride [[RS]] +// CHECK-DAG: OpDecorate %[[SampArray]] ArrayStride [[SS]] + +// BADRS: -fvk-resource-heap-stride must be a power of 2 between 8 and 256 (inclusive) +// BADSS: -fvk-sampler-heap-stride must be a power of 2 between 8 and 256 (inclusive) +// BADNUM: invalid -fvk-resource-heap-stride argument: 'abc' +// Both flags invalid: resource error appears first, sampler error follows on the next line. +// BOTH: -fvk-resource-heap-stride must be a power of 2 between 8 and 256 (inclusive) +// BOTH-NEXT: -fvk-sampler-heap-stride must be a power of 2 between 8 and 256 (inclusive) +// NOSPIRVRS: -fvk-resource-heap-stride requires -spirv +// NOSPIRVSS: -fvk-sampler-heap-stride requires -spirv +// NOHEAPRS: -fvk-resource-heap-stride requires -fspv-use-descriptor-heap +// NOHEAPSS: -fvk-sampler-heap-stride requires -fspv-use-descriptor-heap + +RWByteAddressBuffer outputBytes : register(u0); + +[numthreads(1, 1, 1)] +void main(uint3 tid : SV_DispatchThreadID) { + Texture2D tex = ResourceDescriptorHeap[0]; + SamplerState samp = SamplerDescriptorHeap[0]; + float4 c = tex.SampleLevel(samp, float2(0, 0), 0); + outputBytes.Store(0, asuint(c.x)); +} diff --git a/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.stride-cli.hlsl b/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.stride-cli.hlsl new file mode 100644 index 00000000000..40c987e5ebf --- /dev/null +++ b/tools/clang/test/CodeGenSPIRV/sm6_6.descriptorheap.ext.stride-cli.hlsl @@ -0,0 +1,61 @@ +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -fvk-sampler-heap-stride 16 -spirv %s > %t +// RUN: FileCheck %s < %t +// RUN: FileCheck %s --check-prefix=ABSENT < %t +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-resource-heap-stride 128 -spirv %s | FileCheck %s --check-prefix=RS-ONLY +// RUN: %dxc -T cs_6_6 -E main -Od -fspv-use-descriptor-heap -fspv-target-env=vulkan1.3 -fvk-sampler-heap-stride 16 -spirv %s | FileCheck %s --check-prefix=SS-ONLY + +// Verifies: -fvk-resource-heap-stride / -fvk-sampler-heap-stride emit a literal +// ArrayStride for their heap instead of ArrayStrideIdEXT. The shader loads a +// buffer and an image from the resource heap and a sampler from the sampler +// heap, and each runtime array is checked by name. +// The single-flag cases (RS-ONLY / SS-ONLY) verify that the other heap keeps its +// computed ArrayStrideIdEXT instead of taking the literal stride. + +// Both flags: all three arrays carry their heap's literal stride. +// CHECK-DAG: %[[BufType:[a-zA-Z0-9_]+]] = OpTypeBufferEXT StorageBuffer{{$}} +// CHECK-DAG: %[[ImgType:[a-zA-Z0-9_]+]] = OpTypeImage %float 2D 2 0 0 1 Unknown +// CHECK-DAG: %[[SampType:[a-zA-Z0-9_]+]] = OpTypeSampler +// CHECK-DAG: %[[BufArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[BufType]]{{$}} +// CHECK-DAG: %[[ImgArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[ImgType]]{{$}} +// CHECK-DAG: %[[SampArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[SampType]]{{$}} +// CHECK-DAG: OpDecorate %[[BufArray]] ArrayStride 128 +// CHECK-DAG: OpDecorate %[[ImgArray]] ArrayStride 128 +// CHECK-DAG: OpDecorate %[[SampArray]] ArrayStride 16 + +// No computed stride may appear anywhere in the module. This is a separate +// check, because a NOT after the last DAG match only covers the rest of the +// output, and the decorations come before the types. +// ABSENT-NOT: ArrayStrideIdEXT + +// Resource heap only: the sampler heap keeps its computed stride. +// RS-ONLY-DAG: %[[BufType:[a-zA-Z0-9_]+]] = OpTypeBufferEXT StorageBuffer{{$}} +// RS-ONLY-DAG: %[[ImgType:[a-zA-Z0-9_]+]] = OpTypeImage %float 2D 2 0 0 1 Unknown +// RS-ONLY-DAG: %[[SampType:[a-zA-Z0-9_]+]] = OpTypeSampler +// RS-ONLY-DAG: %[[BufArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[BufType]]{{$}} +// RS-ONLY-DAG: %[[ImgArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[ImgType]]{{$}} +// RS-ONLY-DAG: %[[SampArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[SampType]]{{$}} +// RS-ONLY-DAG: OpDecorate %[[BufArray]] ArrayStride 128 +// RS-ONLY-DAG: OpDecorate %[[ImgArray]] ArrayStride 128 +// RS-ONLY-DAG: OpDecorateId %[[SampArray]] ArrayStrideIdEXT + +// Sampler heap only: the resource heap keeps its computed stride. +// SS-ONLY-DAG: %[[BufType:[a-zA-Z0-9_]+]] = OpTypeBufferEXT StorageBuffer{{$}} +// SS-ONLY-DAG: %[[ImgType:[a-zA-Z0-9_]+]] = OpTypeImage %float 2D 2 0 0 1 Unknown +// SS-ONLY-DAG: %[[SampType:[a-zA-Z0-9_]+]] = OpTypeSampler +// SS-ONLY-DAG: %[[BufArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[BufType]]{{$}} +// SS-ONLY-DAG: %[[ImgArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[ImgType]]{{$}} +// SS-ONLY-DAG: %[[SampArray:[a-zA-Z0-9_]+]] = OpTypeRuntimeArray %[[SampType]]{{$}} +// SS-ONLY-DAG: OpDecorateId %[[BufArray]] ArrayStrideIdEXT +// SS-ONLY-DAG: OpDecorateId %[[ImgArray]] ArrayStrideIdEXT +// SS-ONLY-DAG: OpDecorate %[[SampArray]] ArrayStride 16 + +RWByteAddressBuffer outputBytes : register(u0); + +[numthreads(1, 1, 1)] +void main(uint3 tid : SV_DispatchThreadID) { + StructuredBuffer sb = ResourceDescriptorHeap[0]; + SamplerState samp = SamplerDescriptorHeap[0]; + Texture2D tex = ResourceDescriptorHeap[1]; + outputBytes.Store(0, sb.Load(tid.x)); + outputBytes.Store(4, (uint)tex.SampleLevel(samp, float2(0, 0), 0).r); +} diff --git a/tools/clang/test/DXC/Passes/GVN/pre-poison-add.ll b/tools/clang/test/DXC/Passes/GVN/pre-poison-add.ll new file mode 100644 index 00000000000..4189fa0ca3f --- /dev/null +++ b/tools/clang/test/DXC/Passes/GVN/pre-poison-add.ll @@ -0,0 +1,80 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -gvn -S | FileCheck %s + +; Check that GVN scalar PRE intersects poison-generating IR flags on existing +; predecessor values wired into the PRE PHI, while preserving flags on newly +; inserted PRE instructions cloned from the instruction being replaced. + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +@H = common global i32 0 +@G = common global i32 0 + +define i32 @test1(i1 %cond, i32 %v) nounwind { +; CHECK-LABEL: define i32 @test1( +; CHECK-SAME: i1 [[COND:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[COND]], label %[[BB:.*]], label %[[BB1:.*]] +; CHECK: [[BB]]: +; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[V]], 42 +; CHECK-NEXT: store i32 [[ADD_1]], i32* @G, align 4 +; CHECK-NEXT: br label %[[RETURN:.*]] +; CHECK: [[BB1]]: +; CHECK-NEXT: [[DOTPRE:%.*]] = add i32 [[V]], 42 +; CHECK-NEXT: br label %[[RETURN]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[ADD_2_PRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], %[[BB1]] ], [ [[ADD_1]], %[[BB]] ] +; CHECK-NEXT: store i32 [[ADD_2_PRE_PHI]], i32* @H, align 4 +; CHECK-NEXT: ret i32 0 +; +entry: + br i1 %cond, label %bb, label %bb1 + +bb: + %add.1 = add nuw nsw i32 %v, 42 + store i32 %add.1, i32* @G, align 4 + br label %return + +bb1: + br label %return + +return: + %add.2 = add i32 %v, 42 + store i32 %add.2, i32* @H, align 4 + ret i32 0 +} + +define i32 @test2(i1 %cond, i32 %v) nounwind { +; CHECK-LABEL: define i32 @test2( +; CHECK-SAME: i1 [[COND:%.*]], i32 [[V:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[COND]], label %[[BB:.*]], label %[[BB1:.*]] +; CHECK: [[BB]]: +; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[V]], 42 +; CHECK-NEXT: store i32 [[ADD_1]], i32* @G, align 4 +; CHECK-NEXT: br label %[[RETURN:.*]] +; CHECK: [[BB1]]: +; CHECK-NEXT: [[DOTPRE:%.*]] = add nuw nsw i32 [[V]], 42 +; CHECK-NEXT: br label %[[RETURN]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[ADD_2_PRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], %[[BB1]] ], [ [[ADD_1]], %[[BB]] ] +; CHECK-NEXT: store i32 [[ADD_2_PRE_PHI]], i32* @H, align 4 +; CHECK-NEXT: ret i32 0 +; +entry: + br i1 %cond, label %bb, label %bb1 + +bb: + %add.1 = add i32 %v, 42 + store i32 %add.1, i32* @G, align 4 + br label %return + +bb1: + br label %return + +return: + %add.2 = add nuw nsw i32 %v, 42 + store i32 %add.2, i32* @H, align 4 + ret i32 0 +} diff --git a/tools/clang/test/DXC/Passes/IndVarSimplify/forget-eliminated-icmp.ll b/tools/clang/test/DXC/Passes/IndVarSimplify/forget-eliminated-icmp.ll new file mode 100644 index 00000000000..902a9454ad4 --- /dev/null +++ b/tools/clang/test/DXC/Passes/IndVarSimplify/forget-eliminated-icmp.ll @@ -0,0 +1,142 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -indvars -loop-deletion -S | FileCheck %s + +; Check that SimplifyIndVar forgets cached SCEV expressions for users of an +; eliminated IV comparison before replacing it with a constant, so subsequent +; loop passes do not read stale inner-loop recurrences after LoopDeletion hoists +; those users and deletes the inner loop. + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +; %cmp (j1 >s j2) is folded to true by SimplifyIndVar. +define i32 @forget_icmp_true(i32 %n) { +; CHECK-LABEL: define i32 @forget_icmp_true( +; CHECK-SAME: i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] +; CHECK: [[OUTER_HEADER]]: +; CHECK-NEXT: [[OUTER:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ] +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RES_NEXT:%.*]], %[[OUTER_LATCH]] ] +; CHECK-NEXT: [[LA:%.*]] = add i32 [[N]], 398 +; CHECK-NEXT: [[SEL:%.*]] = select i1 true, i32 [[LA]], i32 [[N]] +; CHECK-NEXT: br label %[[INNER_EXIT:.*]] +; CHECK: [[INNER_EXIT]]: +; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[OUTER_HEADER]] ] +; CHECK-NEXT: [[OUTER_EARLY:%.*]] = icmp ult i32 [[OUTER]], [[SEL_LCSSA]] +; CHECK-NEXT: br i1 [[OUTER_EARLY]], label %[[OUTER_LATCH]], label %[[EXIT:.*]] +; CHECK: [[OUTER_LATCH]]: +; CHECK-NEXT: [[RES_NEXT]] = add i32 [[SEL_LCSSA]], [[RES]] +; CHECK-NEXT: [[OUTER_NEXT]] = add nuw nsw i32 [[OUTER]], 1 +; CHECK-NEXT: [[OUTER_DONE:%.*]] = icmp ult i32 [[OUTER_NEXT]], 1000 +; CHECK-NEXT: br i1 [[OUTER_DONE]], label %[[OUTER_HEADER]], label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], %[[INNER_EXIT]] ], [ [[RES_NEXT]], %[[OUTER_LATCH]] ] +; CHECK-NEXT: ret i32 [[RES_LCSSA]] +; +entry: + br label %outer.header + +outer.header: + %outer = phi i32 [ 0, %entry ], [ %outer.next, %outer.latch ] + %res = phi i32 [ 0, %entry ], [ %res.next, %outer.latch ] + %la = add i32 %n, 398 + br label %inner.header + +inner.header: + %j2 = phi i32 [ 2, %outer.header ], [ %j2.next, %inner.latch ] + %j1 = phi i32 [ 400, %outer.header ], [ %j1.next, %inner.latch ] + %k = phi i32 [ 5, %outer.header ], [ %k.next, %inner.latch ] + %cmp = icmp sgt i32 %j1, %j2 + %sel = select i1 %cmp, i32 %la, i32 %n + %t = add i32 %sel, %k + %early = icmp eq i32 %t, %n + br i1 %early, label %inner.exit, label %inner.latch + +inner.latch: + %j1.next = add i32 %j1, 2 + %j2.next = add i32 %j2, 2 + %k.next = add i32 %k, 1 + %inner.done = icmp ult i32 %k.next, 105 + br i1 %inner.done, label %inner.header, label %inner.exit + +inner.exit: + %outer.early = icmp ult i32 %outer, %sel + br i1 %outer.early, label %outer.latch, label %exit + +outer.latch: + %res.next = add i32 %sel, %res + %outer.next = add i32 %outer, 1 + %outer.done = icmp ult i32 %outer.next, 1000 + br i1 %outer.done, label %outer.header, label %exit + +exit: + %res.lcssa = phi i32 [ %res, %inner.exit ], [ %res.next, %outer.latch ] + ret i32 %res.lcssa +} + +; %cmp (j1 +using namespace dx::linalg; + +#ifndef SECOND_LAYOUT +#define SECOND_LAYOUT RowMajor +#endif + +using HalfA = + Matrix; + +ByteAddressBuffer Input; +RWStructuredBuffer > Output; + +[shader("compute")] +[numthreads(4, 1, 1)] +void main(uint I : SV_GroupIndex) { + const uint SecondStride = + MatrixLayout::SECOND_LAYOUT == MatrixLayout::RowMajor ? 16 : 0; + HalfA M[2] = { + HalfA::Load(Input, 0, 16), + HalfA::Load(Input, 64, SecondStride)}; + Output[I] = Multiply(M[I & 1], (vector)1.0h); +} + +// ROW: ThreadMatrixVectorMultiplyCount: 1 +// ROW: ThreadMatrixVectorMultiply[0]: ResultType=8, MatrixType=8, VectorInputType=8, Flags=2 +// MIXED: ThreadMatrixVectorMultiply[0]: ResultType=8, MatrixType=8, VectorInputType=8, Flags=3 +// RDAT: LinAlgThreadMatrixVectorMultiplyTable[1] +// RDAT: Flags: (MatrixNonMulOptimalLayout) diff --git a/tools/clang/test/DXC/dumpPSV_LinAlgMatVecEscapingArray.ll b/tools/clang/test/DXC/dumpPSV_LinAlgMatVecEscapingArray.ll new file mode 100644 index 00000000000..d0e7be6fc47 --- /dev/null +++ b/tools/clang/test/DXC/dumpPSV_LinAlgMatVecEscapingArray.ll @@ -0,0 +1,50 @@ +; REQUIRES: dxil-1-10 +; RUN: %dxa %s -o %t.dxil | FileCheck %s --check-prefix=ASM +; RUN: %dxa %t.dxil -dumppsv | FileCheck %s + +; A known RowMajor store is not sufficient when a pointer escapes to a helper +; that can overwrite the matrix with a transposed one. +; ASM: Assembly succeeded. +; CHECK: ThreadMatrixVectorMultiplyCount: 1 +; CHECK: ThreadMatrixVectorMultiply[0]: ResultType=9, MatrixType=9, VectorInputType=9, Flags=3 + +target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-ms-dx" + +%dx.types.Handle = type { i8* } +%dx.types.LinAlgMatrixC9M4N4U0S0 = type { i8* } + +define void @main() { + %storage = alloca [2 x %dx.types.LinAlgMatrixC9M4N4U0S0] + %element = getelementptr [2 x %dx.types.LinAlgMatrixC9M4N4U0S0], [2 x %dx.types.LinAlgMatrixC9M4N4U0S0]* %storage, i32 0, i32 0 + %matrix = call %dx.types.LinAlgMatrixC9M4N4U0S0 @dx.op.linAlgMatrixLoadFromDescriptor.mC9M4N4U0S0(i32 325, %dx.types.Handle undef, i32 0, i32 16, i32 0, i32 128) + store %dx.types.LinAlgMatrixC9M4N4U0S0 %matrix, %dx.types.LinAlgMatrixC9M4N4U0S0* %element + %alias = bitcast %dx.types.LinAlgMatrixC9M4N4U0S0* %element to i8* + call void @OverwriteMatrix(i8* %alias) + %loaded = load %dx.types.LinAlgMatrixC9M4N4U0S0, %dx.types.LinAlgMatrixC9M4N4U0S0* %element + %result = call <4 x float> @dx.op.linAlgMatVecMul.v4f32.mC9M4N4U0S0.v4f32(i32 336, %dx.types.LinAlgMatrixC9M4N4U0S0 %loaded, i1 true, <4 x float> zeroinitializer, i32 9) + ret void +} + +define internal void @OverwriteMatrix(i8* %pointer) { + %matrix = call %dx.types.LinAlgMatrixC9M4N4U0S0 @dx.op.linAlgMatrixLoadFromDescriptor.mC9M4N4U0S0(i32 325, %dx.types.Handle undef, i32 0, i32 0, i32 3, i32 128) + %destination = bitcast i8* %pointer to %dx.types.LinAlgMatrixC9M4N4U0S0* + store %dx.types.LinAlgMatrixC9M4N4U0S0 %matrix, %dx.types.LinAlgMatrixC9M4N4U0S0* %destination + ret void +} + +declare %dx.types.LinAlgMatrixC9M4N4U0S0 @dx.op.linAlgMatrixLoadFromDescriptor.mC9M4N4U0S0(i32, %dx.types.Handle, i32, i32, i32, i32) +declare <4 x float> @dx.op.linAlgMatVecMul.v4f32.mC9M4N4U0S0.v4f32(i32, %dx.types.LinAlgMatrixC9M4N4U0S0, i1, <4 x float>, i32) + +!dx.targetTypes = !{!0} +!dx.version = !{!1} +!dx.valver = !{!1} +!dx.shaderModel = !{!2} +!dx.entryPoints = !{!3} + +!0 = !{%dx.types.LinAlgMatrixC9M4N4U0S0 undef, i32 9, i32 4, i32 4, i32 0, i32 0} +!1 = !{i32 1, i32 10} +!2 = !{!"cs", i32 6, i32 10} +!3 = !{void ()* @main, !"main", null, null, !4} +!4 = !{i32 0, i64 2199023255552, i32 4, !5} +!5 = !{i32 1, i32 1, i32 1} diff --git a/tools/clang/test/DXC/dumpPSV_LinAlgUnknownLayout.ll b/tools/clang/test/DXC/dumpPSV_LinAlgUnknownLayout.ll index cf987bf8f42..a235b6a891d 100644 --- a/tools/clang/test/DXC/dumpPSV_LinAlgUnknownLayout.ll +++ b/tools/clang/test/DXC/dumpPSV_LinAlgUnknownLayout.ll @@ -29,7 +29,10 @@ define void @main() { } define internal <4 x float> @ConsumeMatrix(%dx.types.LinAlgMatrixC9M4N4U0S0 %matrix) { - %result = call <4 x float> @dx.op.linAlgMatVecMul.v4f32.mC9M4N4U0S0.v4f32(i32 336, %dx.types.LinAlgMatrixC9M4N4U0S0 %matrix, i1 true, <4 x float> zeroinitializer, i32 9) + %storage = alloca %dx.types.LinAlgMatrixC9M4N4U0S0 + store %dx.types.LinAlgMatrixC9M4N4U0S0 %matrix, %dx.types.LinAlgMatrixC9M4N4U0S0* %storage + %loaded = load %dx.types.LinAlgMatrixC9M4N4U0S0, %dx.types.LinAlgMatrixC9M4N4U0S0* %storage + %result = call <4 x float> @dx.op.linAlgMatVecMul.v4f32.mC9M4N4U0S0.v4f32(i32 336, %dx.types.LinAlgMatrixC9M4N4U0S0 %loaded, i1 true, <4 x float> zeroinitializer, i32 9) ret <4 x float> %result } diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index a5e9145c17c..2ee6f2f2959 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -622,7 +622,7 @@ enum class ComparisonMode { // Every mode compares encoded component bits exactly. Implementation freedom // is expressed by enumerating the permitted results rather than by an Epsilon // or Ulp tolerance, so a conforming result must match a candidate bit for - // bit. + // bit. Float matrix multiply results are the exception; see roundedMatch. Exact, PermittedResults, Excluded, @@ -1165,6 +1165,35 @@ zeroElementsOutsideView(const TypedMatrix &Source, } } +template +static bool roundedMatch(const T &Actual, const T &Expected, MatrixDim = 0) { + return ComponentTraits::exactMatch(Actual, Expected); +} + +// Float matrix multiply results need not be correctly rounded. Accept K + 2 +// ULPs of the expected value for F32, since error grows with the K-term dot +// product, and 4 ULPs for F16. ULPs are of max(|expected|, 1.0) so results near +// zero are not held to a tighter bound. Every bound stays below 1, the smallest +// step between the integer-valued test results. Inf and NaN must match. +static bool roundedMatch(double Actual, double Expected, int Precision, + double Ulps) { + if (!std::isfinite(Actual) || !std::isfinite(Expected)) + return Actual == Expected || (std::isnan(Actual) && std::isnan(Expected)); + int Exponent; + std::frexp(std::max(std::fabs(Expected), 1.0), &Exponent); + return std::fabs(Actual - Expected) <= std::ldexp(Ulps, Exponent - Precision); +} + +static bool roundedMatch(const float &Actual, const float &Expected, + MatrixDim K) { + return roundedMatch(Actual, Expected, 24, K + 2.0); +} + +static bool roundedMatch(const HLSLHalf_t &Actual, const HLSLHalf_t &Expected, + MatrixDim = 0) { + return roundedMatch(float(Actual), float(Expected), 11, 4.0); +} + template static bool exactMatrixMatch(const TypedMatrix &Actual, const TypedMatrix &Expected, @@ -1695,6 +1724,28 @@ static bool verifyHalfBuffer(const HLSLHalf_t *Actual, return Success; } +// Packed results compared with roundedMatch; only float types get a tolerance. +template +static bool verifyRoundedValues(const void *Actual, const Variant &Expected, + MatrixDim K) { + return std::visit( + [Actual, K](const auto &Values) { + using T = typename std::decay::type::value_type; + const T *ActualValues = static_cast(Actual); + for (size_t I = 0; I < Values.size(); ++I) { + if (cpu_oracle::roundedMatch(ActualValues[I], Values[I], K)) + continue; + hlsl_test::LogErrorFmt( + L"Mismatch at index %zu: actual=%s, expected=%s", I, + cpu_oracle::ComponentTraits::format(ActualValues[I]).c_str(), + cpu_oracle::ComponentTraits::format(Values[I]).c_str()); + return false; + } + return true; + }, + Expected); +} + static bool verifyComponentBuffer(ComponentType CompType, const void *Actual, VariantCompType Expected, size_t NumElements, bool Verbose) { @@ -6845,7 +6896,6 @@ static void runMatMatMul(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, const MatrixParams &Params, bool Verbose, MatrixDim K, float AFill, float BFill, UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); std::stringstream ExtraDefs; @@ -6873,8 +6923,7 @@ static void runMatMatMul(ID3D12Device *Device, MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + VERIFY_IS_TRUE(verifyRoundedValues(OutData.data(), Expected, K)); } void DxilConf_SM610_LinAlg_MatrixArithmetic::MatMatMul_Wave_16x16x16_F16() { @@ -6940,7 +6989,6 @@ static void runMatMatMulAccum(ID3D12Device *Device, const MatrixParams &Params, bool Verbose, MatrixDim K, float AFill, float BFill, float CFill, UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); std::stringstream ExtraDefs; @@ -6970,8 +7018,7 @@ static void runMatMatMulAccum(ID3D12Device *Device, MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + VERIFY_IS_TRUE(verifyRoundedValues(OutData.data(), Expected, K)); } void DxilConf_SM610_LinAlg_MatrixArithmetic:: @@ -7229,6 +7276,57 @@ void LinAlgCPUOracleTests::MatrixProductOracle() { /*MatrixA=*/{1, 2, 3, 4, 5, 6}, /*MatrixB=*/{7, 8, 9, 10, 11, 12}, &InitialAccumulator); VERIFY_IS_TRUE(Accumulated == std::vector({59, 63, 141, 152})); + + // Encoded bits: -3.0 at 3, K + 2 and K + 3 ULPs, a dropped term (-2.0), the + // near-zero floor and one ULP past it, Inf/NaN, and a K = 32 F32 result 12 + // ULPs below -1. F16 stays at 4 ULPs whatever K is. + struct RoundedCase { + bool Half; + MatrixDim K; + uint32_t Actual; + uint32_t Expected; + bool Match; + }; + const RoundedCase Cases[] = { + {false, 2, 0xC03FFFFD, 0xC0400000, true}, + {false, 2, 0xC03FFFFC, 0xC0400000, true}, + {false, 2, 0xC03FFFFB, 0xC0400000, false}, + {false, 2, 0xC0000000, 0xC0400000, false}, + {false, 2, 0x35000000, 0x00000000, true}, + {false, 2, 0x35000001, 0x00000000, false}, + {false, 2, 0x7F800000, 0x7F800000, true}, + {false, 2, 0xFF800000, 0x7F800000, false}, + {false, 2, 0x7FC00000, 0x3F800000, false}, + {false, 2, 0x7FC00000, 0x7FC00000, true}, + {false, 32, 0xBF7FFFE8, 0xBF800000, true}, + {false, 32, 0xC03FFFDE, 0xC0400000, true}, + {false, 32, 0xC03FFFDD, 0xC0400000, false}, + {false, 32, 0xC0000000, 0xC0400000, false}, + {true, 32, 0xC1FD, 0xC200, true}, + {true, 32, 0xC1FC, 0xC200, true}, + {true, 32, 0xC1FB, 0xC200, false}, + {true, 32, 0xC000, 0xC200, false}, + {true, 32, 0x1C00, 0x0000, true}, + {true, 32, 0x1C01, 0x0000, false}, + {true, 32, 0x7C00, 0x7C00, true}, + {true, 32, 0xFC00, 0x7C00, false}, + {true, 32, 0x7E00, 0x3C00, false}, + {true, 32, 0x7E00, 0x7E00, true}, + }; + for (const RoundedCase &Case : Cases) { + float A, E; + std::memcpy(&A, &Case.Actual, sizeof(A)); + std::memcpy(&E, &Case.Expected, sizeof(E)); + const bool Match = + Case.Half ? roundedMatch(HLSLHalf_t::FromHALF(uint16_t(Case.Actual)), + HLSLHalf_t::FromHALF(uint16_t(Case.Expected)), + Case.K) + : roundedMatch(A, E, Case.K); + if (Match != Case.Match) + hlsl_test::LogErrorFmt(L"roundedMatch(0x%x, 0x%x) returned %d", + Case.Actual, Case.Expected, Match); + VERIFY_ARE_EQUAL(Case.Match, Match); + } } static bool matrixArithmeticNeeds16BitTypes(ComponentType CompType) { @@ -7828,11 +7926,10 @@ buildMatrixMultiplyCompilerArgs(const MatrixMultiplyCase &Case, return SS.str(); } -static bool -verifyMatrixArithmeticMatrix(const void *Actual, size_t ActualSize, - const MatrixParams &Params, - const std::vector &ExpectedValues, - const std::wstring &PublicRule, bool Verbose) { +static bool verifyMatrixArithmeticMatrix( + const void *Actual, size_t ActualSize, const MatrixParams &Params, + const std::vector &ExpectedValues, const std::wstring &PublicRule, + bool Verbose, MatrixDim RoundedK = 0) { const std::optional> ExpectedBuffer = cpu_oracle::encodeLogicalMatrixBuffer(Params, ExpectedValues); VERIFY_IS_TRUE(ExpectedBuffer.has_value()); @@ -7852,6 +7949,11 @@ verifyMatrixArithmeticMatrix(const void *Actual, size_t ActualSize, if (!Expected) return false; + // The multiply results are packed row-major, so they compare in place. + if (RoundedK != 0 && ActualSize == Params.totalBytes()) + return verifyRoundedValues(Actual, Expected->Values, RoundedK); + VERIFY_ARE_EQUAL(RoundedK, 0u, "Rounded comparison needs a packed result"); + const cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::exactResult(*Expected, PublicRule); return cpu_oracle::verifyMatrixBuffer(Actual, ActualSize, Layout, Oracle, @@ -7947,9 +8049,10 @@ static void runMatrixMultiplyCase(ID3D12Device *Device, MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - VERIFY_IS_TRUE(verifyMatrixArithmeticMatrix(OutData.data(), OutData.size(), - Accumulator, Expected, - Case.PublicRule, Verbose)); + VERIFY_IS_TRUE(verifyMatrixArithmeticMatrix( + OutData.data(), OutData.size(), Accumulator, Expected, Case.PublicRule, + Verbose, + /*RoundedK=*/Case.AccumulatorType != ComponentType::I32 ? Case.K : 0)); } static void runWaveMultiplyCase(ID3D12Device *Device, @@ -7993,12 +8096,12 @@ makeRectangularF16WaveMultiplyCase(MatrixDim M, ComponentType AccumulatorType, Case.AccumulatorValues = makeMatrixArithmeticPattern(Case.M, Case.N, 2, 1, 5, 2); Case.PublicRule = - L"Exact non-uniform F16 product plus an independent F32 accumulator"; + L"Non-uniform F16 product plus an independent F32 accumulator"; } else if (AccumulatorType == ComponentType::F32) { Case.PublicRule = - L"Exact non-uniform F16 matrix product stored in an F32 accumulator"; + L"Non-uniform F16 matrix product stored in an F32 accumulator"; } else { - Case.PublicRule = L"Exact non-uniform rectangular F16 matrix product"; + Case.PublicRule = L"Non-uniform rectangular F16 matrix product"; } return Case; } @@ -8112,7 +8215,7 @@ makeF16MultiplyCase(MatrixDim M, MatrixDim K, MatrixDim N, Case.MatrixBValues = makeMatrixArithmeticPattern(K, N, 1, 3, 7, 3); if (Case.accumulates()) Case.AccumulatorValues = makeMatrixArithmeticPattern(M, N, 2, 1, 5, 2); - Case.PublicRule = L"Exact non-uniform F16 matrix product"; + Case.PublicRule = L"Non-uniform F16 matrix product"; return Case; } @@ -8387,15 +8490,15 @@ makeRectangularF16ThreadGroupMultiplyPlan(ComponentType AccumulatorType, Plan.KExtentMultiple = 2; if (Operation == MatrixMultiplyOperation::MultiplyAccumulate) { Plan.PublicRule = - L"Exact non-uniform ThreadGroup F16 product plus an independent F32 " + L"Non-uniform ThreadGroup F16 product plus an independent F32 " L"accumulator"; } else if (AccumulatorType == ComponentType::F32) { Plan.PublicRule = - L"Exact non-uniform ThreadGroup F16 matrix product stored in an F32 " + L"Non-uniform ThreadGroup F16 matrix product stored in an F32 " L"accumulator"; } else { Plan.PublicRule = - L"Exact non-uniform ThreadGroup F16 product with rectangular inputs"; + L"Non-uniform ThreadGroup F16 product with rectangular inputs"; } return Plan; }