From 1029e9a303b1b2a039b4f746565096df2088531b Mon Sep 17 00:00:00 2001 From: "Werner, Stefan" Date: Mon, 21 Sep 2026 16:23:17 +0200 Subject: [PATCH] Validate face buffer of subdivision meshes The half edge of a face is looked up through the prefix sum of the face valences stored in the face buffer, but this buffer was never validated. A face with zero vertices produced a start offset that pointed behind the half edge array, thus dereferencing the half edge of such a face during commit read out of bounds. A face buffer whose valences sum up to more than the size of an index buffer caused the half edge array, which is sized through the index buffer, to get written out of bounds. The sum was further accumulated in 32 bit, thus it could also wrap around. The face buffer now gets validated before the half edge structures are built, and inconsistent face and index buffers raise an RTC_ERROR_INVALID_OPERATION error. The check is performed before any state is updated, thus a rejected geometry is left unmodified. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 6 +++ doc/src/api/RTC_GEOMETRY_TYPE_SUBDIVISION.md | 6 +++ kernels/common/scene_subdiv_mesh.cpp | 46 ++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/README.md b/README.md index a6f2c6667c..d9379b2e8b 100644 --- a/README.md +++ b/README.md @@ -3627,6 +3627,12 @@ topologies, which means that the `n`-th primitive always has the same number of vertices (e.g. being a triangle or a quad) for each topology. However, the indices of the topologies themselves may be different. +The face buffer has to be consistent with the index buffers, thus each +face must have at least 3 vertices and the sum of all face vertex +counts must not exceed the size of any index buffer. Committing a +geometry that violates this raises an `RTC_ERROR_INVALID_OPERATION` +error. + #### EXIT STATUS {#exit-status} On failure `NULL` is returned and an error code is set that can be diff --git a/doc/src/api/RTC_GEOMETRY_TYPE_SUBDIVISION.md b/doc/src/api/RTC_GEOMETRY_TYPE_SUBDIVISION.md index ec9e2fc3a5..c1bd7d7caa 100644 --- a/doc/src/api/RTC_GEOMETRY_TYPE_SUBDIVISION.md +++ b/doc/src/api/RTC_GEOMETRY_TYPE_SUBDIVISION.md @@ -163,6 +163,12 @@ topologies, which means that the `n`-th primitive always has the same number of vertices (e.g. being a triangle or a quad) for each topology. However, the indices of the topologies themselves may be different. +The face buffer has to be consistent with the index buffers, thus each +face must have at least 3 vertices and the sum of all face vertex +counts must not exceed the size of any index buffer. Committing a +geometry that violates this raises an `RTC_ERROR_INVALID_OPERATION` +error. + #### EXIT STATUS On failure `NULL` is returned and an error code is set that can be diff --git a/kernels/common/scene_subdiv_mesh.cpp b/kernels/common/scene_subdiv_mesh.cpp index 4dc2080d36..2349409447 100644 --- a/kernels/common/scene_subdiv_mesh.cpp +++ b/kernels/common/scene_subdiv_mesh.cpp @@ -11,6 +11,7 @@ #include "../../common/algorithms/parallel_sort.h" #include "../../common/algorithms/parallel_prefix_sum.h" #include "../../common/algorithms/parallel_for.h" +#include "../../common/algorithms/parallel_reduce.h" /*! maximum number of user vertex buffers for subdivision surfaces */ #define RTC_MAX_USER_VERTEX_BUFFERS 65536 @@ -633,6 +634,10 @@ namespace embree /* allocate half edge array */ halfEdges.resize(mesh->numEdges()); + /* the index buffer of each topology has to be large enough for all half edges */ + if (vertexIndices.size() < mesh->numHalfEdges) + throw_RTCError(RTC_ERROR_INVALID_OPERATION,"index buffer of subdivision mesh too small for face buffer"); + /* check if we have to recalculate the half edges */ bool recalculate = false; recalculate |= vertexIndices.isLocalModified(); @@ -688,10 +693,51 @@ namespace embree << std::endl; } + /*! statistics gathered over the face buffer to validate it */ + struct FaceBufferStats + { + uint64_t numHalfEdges = 0; + unsigned int minValence = unsigned(-1); + }; + void SubdivMesh::initializeHalfEdgeStructures () { double t0 = getSeconds(); + /* The half edge of a face is looked up through the prefix sum of the face + valences. We thus have to validate the face buffer before anything else, + as otherwise a face with zero vertices or a face buffer that does not + match the index buffer would index the half edge array out of bounds. */ + if (faceVertices.isLocalModified()) + { + const FaceBufferStats stats = parallel_reduce + (size_t(0), numFaces(), size_t(1024), FaceBufferStats(), + [&](const range& r) -> FaceBufferStats + { + FaceBufferStats stats; + for (size_t f=r.begin(); f FaceBufferStats { + FaceBufferStats stats; + stats.numHalfEdges = a.numHalfEdges + b.numHalfEdges; + stats.minValence = min(a.minValence,b.minValence); + return stats; + }); + + if (numFaces() && stats.minValence == 0) + throw_RTCError(RTC_ERROR_INVALID_OPERATION,"subdivision face with zero vertices"); + + if (stats.numHalfEdges > (uint64_t)numEdges()) + throw_RTCError(RTC_ERROR_INVALID_OPERATION,"index buffer of subdivision mesh too small for face buffer"); + + if (stats.numHalfEdges > (uint64_t)0xFFFFFFFF) + throw_RTCError(RTC_ERROR_INVALID_OPERATION,"too many edges in subdivision mesh"); + } + invalid_face.resize(numFaces()*numTimeSteps); /* calculate start edge of each face */