diff --git a/graphify/_minhash.py b/graphify/_minhash.py index aabeb654f..5d2a3e419 100644 --- a/graphify/_minhash.py +++ b/graphify/_minhash.py @@ -48,6 +48,29 @@ def update(self, v: bytes) -> None: phv = np.bitwise_and((self._a * hv + self._b) % _MP, _MH) self.hashvalues = np.minimum(self.hashvalues, phv) + def update_batch(self, values: "list[bytes] | tuple[bytes, ...]") -> None: + """Fold many byte-strings into the sketch in one vectorized pass. + + Equivalent to calling :meth:`update` once per element — the sketch is + the element-wise minimum of every element's permuted hash, and ``min`` + is associative, so the order and the batching are irrelevant to the + result. But this does the 128-wide permutation arithmetic once on an + ``(S, 128)`` array instead of S times on ``(128,)`` arrays, which is + where the per-token loop spent almost all its time. The ``uint64`` + multiply wraps mod 2**64 exactly as the scalar path does (a*hv reaches + ~2**93), and broadcasting preserves that wraparound element-wise, so the + hash values are bit-identical to the per-element loop. + """ + if not values: + return + hvs = np.fromiter( + (struct.unpack(" float: """Numerical integration — replaces scipy.integrate.quad for LSH param search.""" diff --git a/graphify/dedup.py b/graphify/dedup.py index 7c74371ff..8121466c0 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -48,8 +48,7 @@ def _shingles(text: str, k: int = 3) -> set[str]: def _make_minhash(text: str, num_perm: int = 128) -> MinHash: # Strip spaces so "graph extractor" and "graphextractor" share shingles m = MinHash(num_perm=num_perm) - for shingle in _shingles(text.replace(" ", "")): - m.update(shingle.encode("utf-8")) + m.update_batch([shingle.encode("utf-8") for shingle in _shingles(text.replace(" ", ""))]) return m