From d7731bd53ee82e960383860beeca65c94c96d8eb Mon Sep 17 00:00:00 2001
From: abhay-codes07 <abhaysingh0293@gmail.com>
Date: Fri, 18 Sep 2026 02:38:53 +0530
Subject: [PATCH] perf(dedup): batch MinHash shingle hashing into one
 vectorized pass
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`_make_minhash` folded each character-shingle into the sketch with a
separate `MinHash.update` call, and every call ran the 128-wide
permutation arithmetic on its own — `(a * hv + b) % MP & MASK` plus a
`np.minimum`, all on 128-element arrays. On graphify's own corpus that's
~200k `update` calls for ~3.8k sketches, and the per-call numpy dispatch
on tiny arrays dominated the build phase: `MinHash.update` was the single
hottest function at ~0.9s self-time.

Add `MinHash.update_batch`, which hashes every shingle, stacks the 32-bit
values into one array, and computes the permutations once on an `(S, 128)`
array before taking the column-wise minimum. The sketch is the
element-wise min over all shingles and `min` is associative, so batching
changes nothing about the result — and the `uint64` multiply wraps mod
2**64 exactly as the scalar path does (`a*hv` reaches ~2**93), a
wraparound broadcasting preserves element-wise, so the hash values are
bit-identical to the per-shingle loop.

Verified: `update_batch` produces bit-identical `hashvalues` to the
`update` loop across 500 randomized trials (including the empty case), and
the fully built+deduplicated graph of the 364-file corpus is byte-identical
to v8 (12110 nodes, 24897 edges; same 13 merges — 10 exact, 3 fuzzy).
`deduplicate_entities` drops from ~1.9s to ~1.1s.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JJbLfztSxm2tH5cJwBbe9q
---
 graphify/_minhash.py | 23 +++++++++++++++++++++++
 graphify/dedup.py    |  3 +--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/graphify/_minhash.py b/graphify/_minhash.py
index aabeb654f3..5d2a3e4196 100644
--- a/graphify/_minhash.py
+++ b/graphify/_minhash.py
@@ -48,6 +48,29 @@ def update(self, v: bytes) -> None:
         phv = np.bitwise_and((self._a * hv + self._b) % _MP, _MH)
         self.hashvalues = np.minimum(self.hashvalues, phv)
 
+    def update_batch(self, values: "list[bytes] | tuple[bytes, ...]") -> None:
+        """Fold many byte-strings into the sketch in one vectorized pass.
+
+        Equivalent to calling :meth:`update` once per element — the sketch is
+        the element-wise minimum of every element's permuted hash, and ``min``
+        is associative, so the order and the batching are irrelevant to the
+        result. But this does the 128-wide permutation arithmetic once on an
+        ``(S, 128)`` array instead of S times on ``(128,)`` arrays, which is
+        where the per-token loop spent almost all its time. The ``uint64``
+        multiply wraps mod 2**64 exactly as the scalar path does (a*hv reaches
+        ~2**93), and broadcasting preserves that wraparound element-wise, so the
+        hash values are bit-identical to the per-element loop.
+        """
+        if not values:
+            return
+        hvs = np.fromiter(
+            (struct.unpack("<I", hashlib.sha1(v).digest()[:4])[0] for v in values),
+            dtype=np.uint64,
+            count=len(values),
+        )
+        phv = np.bitwise_and((self._a * hvs[:, None] + self._b) % _MP, _MH)
+        self.hashvalues = np.minimum(self.hashvalues, phv.min(axis=0))
+
 
 def _lsh_integrate(f, lo: float, hi: float, n: int = 128) -> float:
     """Numerical integration — replaces scipy.integrate.quad for LSH param search."""
diff --git a/graphify/dedup.py b/graphify/dedup.py
index 7c74371ff3..8121466c0e 100644
--- a/graphify/dedup.py
+++ b/graphify/dedup.py
@@ -48,8 +48,7 @@ def _shingles(text: str, k: int = 3) -> set[str]:
 def _make_minhash(text: str, num_perm: int = 128) -> MinHash:
     # Strip spaces so "graph extractor" and "graphextractor" share shingles
     m = MinHash(num_perm=num_perm)
-    for shingle in _shingles(text.replace(" ", "")):
-        m.update(shingle.encode("utf-8"))
+    m.update_batch([shingle.encode("utf-8") for shingle in _shingles(text.replace(" ", ""))])
     return m