perf(utils): avoid quadratic suffix search in dedupe_keys

sqlite_utils/utils.py

@@ -634,14 +634,17 @@ def dedupe_keys(keys: Iterable[str]) -> list[str]:
         # No duplicates - the common case
         return keys
     seen: set = set()
+    next_suffix: dict[str, int] = {}
     result = []
     for key in keys:
         if key in seen:
-            new_key = key
-            suffix = 2
-            while new_key in seen or new_key in taken:
+            suffix = next_suffix.get(key, 2)
+            while True:
                 new_key = f"{key}_{suffix}"
                 suffix += 1
+                if new_key not in seen and new_key not in taken:
+                    break
+            next_suffix[key] = suffix
             key = new_key
         seen.add(key)
         result.append(key)

tests/test_utils.py

@@ -102,3 +102,9 @@ def test_flatten(input, expected):
 )
 def test_dedupe_keys(input, expected):
     assert utils.dedupe_keys(input) == expected
+
+
+def test_dedupe_keys_many_duplicates():
+    input_keys = ["col"] * 1000
+    expected = ["col"] + [f"col_{i}" for i in range(2, 1001)]
+    assert utils.dedupe_keys(input_keys) == expected