From 680fa3436392f0126ec73ac760dd62baebe509e8 Mon Sep 17 00:00:00 2001 From: "Werner, Stefan" Date: Mon, 21 Sep 2026 17:17:54 +0200 Subject: [PATCH] Validate buffer type and slot in rtcInterpolate/rtcInterpolateN The bufferType and bufferSlot members of RTCInterpolateArguments and RTCInterpolateNArguments are passed from the public API straight into the per geometry interpolate_impl() routines, where the slot is used to index the fixed size vertices, tangents and vertexAttribs arrays of buffer descriptors. The only guard was an assert that is removed in release builds, and that was additionally off by one for vertex attributes as it used bufferSlot <= vertexAttribs.size() instead of <. An out of range slot therefore read a buffer descriptor past the end of the descriptor array. The resulting pointer and stride were then used to load vertex data, and the loaded bytes were written into the caller visible interpolation output arrays. This affects all interpolatable geometry types, both the vertex and the vertex attribute buffer branch, and both rtcInterpolate and rtcInterpolateN. A buffer type other than RTC_BUFFER_TYPE_VERTEX or RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE was silently treated as a vertex buffer instead of being rejected. Add Geometry::checkInterpolateBuffer(), which rejects unsupported buffer types, vertex slots that are not smaller than numTimeSteps, and vertex attribute slots that are not smaller than the vertex attribute count. These bounds also cover the vertex_buffer_tags and vertex_attrib_buffer_tags arrays of subdivision meshes, which are sized by the same two counts. Invalid requests now raise RTC_ERROR_INVALID_ARGUMENT and interpolate no data. Use the helper at all interpolation sites, replacing the asserts, and document the requirement. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 8 ++++++++ doc/src/api/rtcInterpolate.md | 8 ++++++++ kernels/common/geometry.h | 17 +++++++++++++++++ kernels/common/scene_curves.h | 7 +++---- kernels/common/scene_grid_mesh.h | 3 +-- kernels/common/scene_line_segments.h | 3 +-- kernels/common/scene_quad_mesh.h | 3 +-- kernels/common/scene_subdiv_mesh.cpp | 10 ++-------- kernels/common/scene_triangle_mesh.h | 3 +-- 9 files changed, 42 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a6f2c6667c..fac9b1e8e5 100644 --- a/README.md +++ b/README.md @@ -7188,6 +7188,14 @@ interpolation buffer, one can specify vertex buffers (`RTC_BUFFER_TYPE_VERTEX`) and vertex attribute buffers (`RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE`) as well. +The `bufferType` has to be either `RTC_BUFFER_TYPE_VERTEX` or +`RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE`, and the `bufferSlot` has to be a +valid slot of the specified buffer type, thus smaller than the number of +time steps for vertex buffers and smaller than the vertex attribute +count set using `rtcSetGeometryVertexAttributeCount` for vertex +attribute buffers. Otherwise an `RTC_ERROR_INVALID_ARGUMENT` error is +raised and no data is interpolated. + The `rtcInterpolate` call stores `valueCount` number of interpolated floating point values to the memory location pointed to by `P`. One can avoid storing the interpolated value by setting `P` to `NULL`. diff --git a/doc/src/api/rtcInterpolate.md b/doc/src/api/rtcInterpolate.md index 5b581f5c69..e2fee6ebd9 100644 --- a/doc/src/api/rtcInterpolate.md +++ b/doc/src/api/rtcInterpolate.md @@ -51,6 +51,14 @@ interpolation buffer, one can specify vertex buffers (`RTC_BUFFER_TYPE_VERTEX`) and vertex attribute buffers (`RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE`) as well. +The `bufferType` has to be either `RTC_BUFFER_TYPE_VERTEX` or +`RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE`, and the `bufferSlot` has to be a +valid slot of the specified buffer type, thus smaller than the number +of time steps for vertex buffers and smaller than the vertex attribute +count set using `rtcSetGeometryVertexAttributeCount` for vertex +attribute buffers. Otherwise an `RTC_ERROR_INVALID_ARGUMENT` error is +raised and no data is interpolated. + The `rtcInterpolate` call stores `valueCount` number of interpolated floating point values to the memory location pointed to by `P`. One can avoid storing the interpolated value by setting `P` to `NULL`. diff --git a/kernels/common/geometry.h b/kernels/common/geometry.h index 3c7ce99564..9d165b9fa5 100644 --- a/kernels/common/geometry.h +++ b/kernels/common/geometry.h @@ -401,6 +401,23 @@ namespace embree throw_RTCError(RTC_ERROR_INVALID_OPERATION,"operation not supported for this geometry"); } + /*! validates the buffer type and slot of an interpolation request. The buffer slot + * originates from the public rtcInterpolate/rtcInterpolateN API and is used to index + * fixed size arrays of buffer descriptors, thus it has to be range checked at runtime. */ + __forceinline void checkInterpolateBuffer(RTCBufferType bufferType, unsigned int bufferSlot, size_t numVertexAttributes) const + { + if (bufferType == RTC_BUFFER_TYPE_VERTEX) { + if (unlikely(bufferSlot >= numTimeSteps)) + throw_RTCError(RTC_ERROR_INVALID_ARGUMENT,"invalid vertex buffer slot specified for interpolation"); + } + else if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { + if (unlikely(bufferSlot >= numVertexAttributes)) + throw_RTCError(RTC_ERROR_INVALID_ARGUMENT,"invalid vertex attribute buffer slot specified for interpolation"); + } + else + throw_RTCError(RTC_ERROR_INVALID_ARGUMENT,"invalid buffer type specified for interpolation"); + } + /*! interpolates user data to the specified u/v locations */ virtual void interpolateN(const RTCInterpolateNArguments* const args); diff --git a/kernels/common/scene_curves.h b/kernels/common/scene_curves.h index 7350a20ecd..c488215164 100644 --- a/kernels/common/scene_curves.h +++ b/kernels/common/scene_curves.h @@ -545,8 +545,7 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < numTimeSteps) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot <= vertexAttribs.size())); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { @@ -708,10 +707,11 @@ namespace embree float* ddPdudu = args->ddPdudu; unsigned int valueCount = args->valueCount; + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); + /* we interpolate vertex attributes linearly for hermite basis */ if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { - assert(bufferSlot <= vertexAttribs.size()); const char* vsrc = vertexAttribs[bufferSlot].getPtr(); const size_t vstride = vertexAttribs[bufferSlot].getStride(); @@ -732,7 +732,6 @@ namespace embree /* interpolation for vertex buffers */ else { - assert(bufferSlot < numTimeSteps); const char* vsrc = vertices[bufferSlot].getPtr(); const char* tsrc = tangents[bufferSlot].getPtr(); const size_t vstride = vertices[bufferSlot].getStride(); diff --git a/kernels/common/scene_grid_mesh.h b/kernels/common/scene_grid_mesh.h index cd374912f5..e99fa3088f 100644 --- a/kernels/common/scene_grid_mesh.h +++ b/kernels/common/scene_grid_mesh.h @@ -86,8 +86,7 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < numTimeSteps) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot <= vertexAttribs.size())); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { diff --git a/kernels/common/scene_line_segments.h b/kernels/common/scene_line_segments.h index a672abd8d2..b87f78c5b7 100644 --- a/kernels/common/scene_line_segments.h +++ b/kernels/common/scene_line_segments.h @@ -49,8 +49,7 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < numTimeSteps) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot <= vertexAttribs.size())); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { diff --git a/kernels/common/scene_quad_mesh.h b/kernels/common/scene_quad_mesh.h index 646b08c1ab..cbef6fb9cd 100644 --- a/kernels/common/scene_quad_mesh.h +++ b/kernels/common/scene_quad_mesh.h @@ -68,8 +68,7 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < numTimeSteps) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot <= vertexAttribs.size())); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { diff --git a/kernels/common/scene_subdiv_mesh.cpp b/kernels/common/scene_subdiv_mesh.cpp index 4dc2080d36..be94bd9f65 100644 --- a/kernels/common/scene_subdiv_mesh.cpp +++ b/kernels/common/scene_subdiv_mesh.cpp @@ -854,21 +854,18 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < RTC_MAX_TIME_STEP_COUNT) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot < RTC_MAX_USER_VERTEX_BUFFERS)); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; std::vector* baseEntry = nullptr; Topology* topo = nullptr; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { - assert(bufferSlot < vertexAttribs.size()); src = vertexAttribs[bufferSlot].getPtr(); stride = vertexAttribs[bufferSlot].getStride(); baseEntry = &vertex_attrib_buffer_tags[bufferSlot]; int topologyID = vertexAttribs[bufferSlot].userData; topo = &topology[topologyID]; } else { - assert(bufferSlot < numTimeSteps); src = vertices[bufferSlot].getPtr(); stride = vertices[bufferSlot].getStride(); baseEntry = &vertex_buffer_tags[bufferSlot]; @@ -931,21 +928,18 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < RTC_MAX_TIME_STEP_COUNT) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot < RTC_MAX_USER_VERTEX_BUFFERS)); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; std::vector* baseEntry = nullptr; Topology* topo = nullptr; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) { - assert(bufferSlot < vertexAttribs.size()); src = vertexAttribs[bufferSlot].getPtr(); stride = vertexAttribs[bufferSlot].getStride(); baseEntry = &vertex_attrib_buffer_tags[bufferSlot]; int topologyID = vertexAttribs[bufferSlot].userData; topo = &topology[topologyID]; } else { - assert(bufferSlot < numTimeSteps); src = vertices[bufferSlot].getPtr(); stride = vertices[bufferSlot].getStride(); baseEntry = &vertex_buffer_tags[bufferSlot]; diff --git a/kernels/common/scene_triangle_mesh.h b/kernels/common/scene_triangle_mesh.h index 3f014d85a2..a1e712839c 100644 --- a/kernels/common/scene_triangle_mesh.h +++ b/kernels/common/scene_triangle_mesh.h @@ -62,8 +62,7 @@ namespace embree unsigned int valueCount = args->valueCount; /* calculate base pointer and stride */ - assert((bufferType == RTC_BUFFER_TYPE_VERTEX && bufferSlot < numTimeSteps) || - (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE && bufferSlot <= vertexAttribs.size())); + checkInterpolateBuffer(bufferType,bufferSlot,vertexAttribs.size()); const char* src = nullptr; size_t stride = 0; if (bufferType == RTC_BUFFER_TYPE_VERTEX_ATTRIBUTE) {