Fix heap buffer overflow for high valence subdivision vertices

kernels/common/scene_subdiv_mesh.cpp

@@ -561,8 +561,11 @@ namespace embree
 
         /* we have to calculate patch_type last! */
         HalfEdge::PatchType patch_type = edge->patchType();
-        for (size_t i=0; i<mesh->faceVertices[f]; i++) 
+        const bool valid = edge->validPatchTopology();
+        for (size_t i=0; i<mesh->faceVertices[f]; i++) {
           edge[i].patch_type = patch_type;
+          edge[i].valid_patch = valid;
+        }
       }
     });
   }
@@ -878,12 +881,25 @@ namespace embree
       bool has_P = P;
       bool has_dP = dPdu;     assert(!has_dP  || dPdv);
       bool has_ddP = ddPdudu; assert(!has_ddP || (ddPdvdv && ddPdudu));
+
+      const HalfEdge* halfEdge = topo->getHalfEdge(primID);
+
+      /* invalid patches cannot get evaluated */
+      if (unlikely(!this->valid(primID) || !topo->valid(primID)))
+      {
+        for (unsigned int j=0; j<valueCount; j++) {
+          if (has_P) P[j] = 0.0f;
+          if (has_dP) { dPdu[j] = 0.0f; dPdv[j] = 0.0f; }
+          if (has_ddP) { ddPdudu[j] = 0.0f; ddPdvdv[j] = 0.0f; ddPdudv[j] = 0.0f; }
+        }
+        return;
+      }
 
       for (unsigned int i=0; i<valueCount; i+=4)
       {
         vfloat4 Pt, dPdut, dPdvt, ddPdudut, ddPdvdvt, ddPdudvt;
         isa::PatchEval<vfloat4,vfloat4>(baseEntry->at(interpolationSlot(primID,i/4,stride)),commitCounter,
-                                        topo->getHalfEdge(primID),src+i*sizeof(float),stride,u,v,
+                                        halfEdge,src+i*sizeof(float),stride,u,v,
                                         has_P ? &Pt : nullptr, 
                                         has_dP ? &dPdut : nullptr, 
                                         has_dP ? &dPdvt : nullptr,
@@ -966,11 +982,27 @@ namespace embree
 
         foreach_unique(valid1,primID,[&](const vbool4& valid1, const unsigned int primID)
                        {
+                         const HalfEdge* halfEdge = topo->getHalfEdge(primID);
+
+                         /* invalid patches cannot get evaluated */
+                         if (unlikely(!this->valid(primID) || !topo->valid(primID)))
+                         {
+                           for (unsigned int j=0; j<valueCount; j++) {
+                             if (P) vfloat4::storeu(valid1,P+j*N+i,vfloat4(0.0f));
+                             if (dPdu) vfloat4::storeu(valid1,dPdu+j*N+i,vfloat4(0.0f));
+                             if (dPdv) vfloat4::storeu(valid1,dPdv+j*N+i,vfloat4(0.0f));
+                             if (ddPdudu) vfloat4::storeu(valid1,ddPdudu+j*N+i,vfloat4(0.0f));
+                             if (ddPdvdv) vfloat4::storeu(valid1,ddPdvdv+j*N+i,vfloat4(0.0f));
+                             if (ddPdudv) vfloat4::storeu(valid1,ddPdudv+j*N+i,vfloat4(0.0f));
+                           }
+                           return;
+                         }
+
                          for (unsigned int j=0; j<valueCount; j+=4) 
                          {
                            const size_t M = min(4u,valueCount-j);
                            isa::PatchEvalSimd<vbool4,vint4,vfloat4,vfloat4>(baseEntry->at(interpolationSlot(primID,j/4,stride)),commitCounter,
-                                                                            topo->getHalfEdge(primID),src+j*sizeof(float),stride,valid1,uu,vv,
+                                                                            halfEdge,src+j*sizeof(float),stride,valid1,uu,vv,
                                                                             P ? P+j*N+i : nullptr,
                                                                             dPdu ? dPdu+j*N+i : nullptr,
                                                                             dPdv ? dPdv+j*N+i : nullptr,

kernels/common/scene_subdiv_mesh.h

@@ -169,6 +169,8 @@ namespace embree
       /*! check if the i'th primitive is valid in this topology */
       __forceinline bool valid(size_t i) const 
       {
+        if (unlikely(!getHalfEdge(i)->valid_patch))
+          return false;
         if (unlikely(subdiv_mode == RTC_SUBDIVISION_MODE_NO_BOUNDARY)) {
           if (getHalfEdge(i)->faceHasBorder()) return false;
         }

kernels/subdiv/catmullclark_ring.h

@@ -151,6 +151,7 @@ namespace embree
 
       do
       {
+        assert(i+2 <= MAX_RING_EDGE_VALENCE);
         vertex_level = max(vertex_level,p->edge_level);
         crease_weight[i/2] = p->edge_crease_weight;
         assert(p->hasOpposite() || p->edge_crease_weight == float(inf));
@@ -178,6 +179,7 @@ namespace embree
           if (index0 < min_vertex_index) { min_vertex_index = index0; min_vertex_index_face = i>>1; }
 
           /*! mark first border edge and store dummy vertex for face between the two border edges */
+          assert(i+2 <= MAX_RING_EDGE_VALENCE);
           border_index = i;
           crease_weight[i/2] = inf; 
           ring[i++] = Vertex_t::loadu(vertices+index0*stride);
@@ -606,9 +608,11 @@ namespace embree
 	/* store first N-2 vertices of face */
 	unsigned int vn = 0;
         for (p = p_next; p!=p_prev; p=p->next()) {
+          assert(e < MAX_RING_EDGE_VALENCE);
           ring[e++] = Vertex_t::loadu(vertices+p->getStartVertexIndex()*stride);
           vn++;
 	}
+        assert(f < MAX_RING_FACE_VALENCE);
 	faces[f++] = Face(vn,crease_weight);
 	only_quads &= (vn == 2);
 
@@ -624,6 +628,8 @@ namespace embree
           if (vertex_index < min_vertex_index) { min_vertex_index = vertex_index; min_vertex_index_face = f; min_vertex_index_vertex = e; }
 
           /*! mark first border edge and store dummy vertex for face between the two border edges */
+          assert(f < MAX_RING_FACE_VALENCE);
+          assert(e+2 <= MAX_RING_EDGE_VALENCE);
           border_face = f;
 	  faces[f++] = Face(2,inf); 
           ring[e++] = Vertex_t::loadu(vertices+p->getStartVertexIndex()*stride);

kernels/subdiv/half_edge.h

@@ -48,7 +48,7 @@ namespace embree
 
     HalfEdge () 
       : vtx_index(-1), next_half_edge_ofs(0), prev_half_edge_ofs(0), opposite_half_edge_ofs(0), edge_crease_weight(0), 
-      vertex_crease_weight(0), edge_level(0), patch_type(COMPLEX_PATCH), vertex_type(REGULAR_VERTEX)
+      vertex_crease_weight(0), edge_level(0), patch_type(COMPLEX_PATCH), vertex_type(REGULAR_VERTEX), valid_patch(false)
     {
       static_assert(sizeof(HalfEdge) == 32, "invalid half edge size");
     }
@@ -354,6 +354,62 @@ namespace embree
 
       return faceValence <= MAX_RING_FACE_VALENCE && edgeValence <= MAX_RING_EDGE_VALENCE;
     }
+
+    /*! tests if the ring around the start vertex is within the supported size
+     *  limits. In contrast to validRing this test only depends on the topology
+     *  and not on the vertex positions. */
+    __forceinline bool validRingTopology() const
+    {
+      size_t faceValence = 0;
+      size_t edgeValence = 0;
+
+      const HalfEdge* p = this;
+      do
+      {
+        /* check size of current face */
+        const size_t n = p->numEdges();
+        if (n < 3 || n > MAX_PATCH_VALENCE)
+          return false;
+        edgeValence += n-2;
+
+        faceValence++;
+        p = p->prev();
+
+        /* continue with next face */
+        if (likely(p->hasOpposite()))
+          p = p->opposite();
+
+        /* if there is no opposite go the long way to the other side of the border */
+        else {
+          faceValence++;
+          edgeValence++;
+          p = this;
+          while (p->hasOpposite())
+            p = p->opposite()->next();
+        }
+
+        /* stop early for degenerated topology */
+        if (faceValence > MAX_RING_FACE_VALENCE || edgeValence > MAX_RING_EDGE_VALENCE)
+          return false;
+
+      } while (p != this);
+
+      return true;
+    }
+
+  public:
+
+    /*! tests if this patch and all its rings are within the supported size
+     *  limits. Patches that are not, cannot get evaluated. */
+    __forceinline bool validPatchTopology() const
+    {
+      size_t N = 1;
+      if (!this->validRingTopology()) return false;
+      for (const HalfEdge* p=this->next(); p!=this; p=p->next(), N++) {
+        if (!p->validRingTopology()) return false;
+      }
+      return N >= 3 && N <= MAX_PATCH_VALENCE;
+    }
 
   private:
     unsigned int vtx_index;         //!< index of edge start vertex
@@ -367,6 +423,7 @@ namespace embree
     float edge_level;               //!< subdivision factor for edge
     PatchType patch_type;           //!< stores type of subdiv patch
     VertexType vertex_type;         //!< stores type of the start vertex
-    char align[2];
+    bool valid_patch;               //!< stores if the patch can be evaluated
+    char align[1];
   };
 }