diff --git a/src/useSet.ts b/src/useSet.ts
index 9c88306cc9..2c98f70898 100644
--- a/src/useSet.ts
+++ b/src/useSet.ts
@@ -18,13 +18,21 @@ const useSet = <K>(initialSet = new Set<K>()): [Set<K>, Actions<K>] => {
   const stableActions = useMemo<StableActions<K>>(() => {
     const add = (item: K) => setSet((prevSet) => new Set([...Array.from(prevSet), item]));
     const remove = (item: K) =>
-      setSet((prevSet) => new Set(Array.from(prevSet).filter((i) => i !== item)));
+      setSet((prevSet) => {
+        const nextSet = new Set(prevSet);
+        nextSet.delete(item);
+        return nextSet;
+      });
     const toggle = (item: K) =>
-      setSet((prevSet) =>
-        prevSet.has(item)
-          ? new Set(Array.from(prevSet).filter((i) => i !== item))
-          : new Set([...Array.from(prevSet), item])
-      );
+      setSet((prevSet) => {
+        const nextSet = new Set(prevSet);
+        if (nextSet.has(item)) {
+          nextSet.delete(item);
+        } else {
+          nextSet.add(item);
+        }
+        return nextSet;
+      });
 
     return { add, remove, toggle, reset: () => setSet(initialSet), clear: () => setSet(new Set()) };
   }, [setSet]);
diff --git a/tests/useSet.test.ts b/tests/useSet.test.ts
index b4daa70d3e..9b6b0ed631 100644
--- a/tests/useSet.test.ts
+++ b/tests/useSet.test.ts
@@ -96,6 +96,47 @@ it('should remove existing key', () => {
   expect(result.current[0]).toEqual(new Set([1]));
 });
 
+it.each(['remove', 'toggle'] as const)(
+  'should %s NaN without mutating the previous set',
+  (action) => {
+    const initialSet = new Set([NaN, 1]);
+    const { result } = setUp(initialSet);
+    const previousSet = result.current[0];
+
+    act(() => {
+      result.current[1][action](NaN);
+    });
+
+    expect(result.current[0]).toEqual(new Set([1]));
+    expect(result.current[1].has(NaN)).toBe(false);
+    expect(result.current[0]).not.toBe(previousSet);
+    expect(previousSet).toEqual(new Set([NaN, 1]));
+    expect(initialSet).toEqual(new Set([NaN, 1]));
+
+    act(() => {
+      result.current[1].reset();
+    });
+
+    expect(result.current[0]).toEqual(new Set([NaN, 1]));
+  }
+);
+
+it('should toggle NaN on and off', () => {
+  const { result } = setUp(new Set<number>());
+
+  act(() => {
+    result.current[1].toggle(NaN);
+  });
+
+  expect(result.current[1].has(NaN)).toBe(true);
+
+  act(() => {
+    result.current[1].toggle(NaN);
+  });
+
+  expect(result.current[0].size).toBe(0);
+});
+
 it('should remove an existing key on toggle', () => {
   const { result } = setUp(new Set([1, 2]));
   const [, utils] = result.current;