From 96b75684c3115897ab3a7c0caf4f0b31aa9ca88d Mon Sep 17 00:00:00 2001
From: Anas Khan <83116240+anxkhn@users.noreply.github.com>
Date: Fri, 3 Jul 2026 16:27:46 +0530
Subject: [PATCH 01/11] fix: handle psutil.AccessDenied in get_memory_info

On Linux, get_memory_info reads a process's PSS via memory_full_info,
which psutil documents may require elevated privileges. In restricted
environments (hardened containers with hidepid/restricted /proc, or an
unreadable child subprocess) this raises psutil.AccessDenied, which is
not a subclass of psutil.NoSuchProcess and so was not caught by the
existing suppress around the child loop; the current-process read was
unguarded entirely. The exception propagated out of the recurring
system-info task, which has no error handling, silently stopping the
autoscaler's CPU/memory snapshots.

Suppress AccessDenied alongside NoSuchProcess when summing child memory,
and fall back to RSS for the current process when PSS is denied.

Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>
---
 src/crawlee/_utils/system.py     | 15 +++++++----
 tests/unit/_utils/test_system.py | 45 ++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 56eeaadf24..757b19e062 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -122,14 +122,19 @@ def get_memory_info() -> MemoryInfo:
     logger.debug('Calling get_memory_info()...')
     current_process = psutil.Process(os.getpid())
 
-    # Retrieve estimated memory usage of the current process.
-    current_size_bytes = _get_used_memory(current_process)
+    # Retrieve estimated memory usage of the current process. On Linux `_get_used_memory` reads PSS via
+    # `memory_full_info`, which can raise `AccessDenied` in restricted environments (e.g. hardened containers);
+    # fall back to RSS, which a process can always read for itself.
+    try:
+        current_size_bytes = _get_used_memory(current_process)
+    except psutil.AccessDenied:
+        current_size_bytes = int(current_process.memory_info().rss)
 
     # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
     for child in current_process.children(recursive=True):
-        # Ignore any NoSuchProcess exception that might occur if a child process ends before we retrieve
-        # its memory usage.
-        with suppress(psutil.NoSuchProcess):
+        # Ignore a child that ends before we retrieve its memory usage (`NoSuchProcess`) or that we are not
+        # allowed to inspect (`AccessDenied`, e.g. an unreadable subprocess in a restricted environment).
+        with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
             current_size_bytes += _get_used_memory(child)
 
     vm = psutil.virtual_memory()
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 1813b151a6..9e094f355f 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -5,8 +5,10 @@
 from multiprocessing.shared_memory import SharedMemory
 from typing import TYPE_CHECKING
 
+import psutil
 import pytest
 
+from crawlee._utils import system
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.system import get_cpu_info, get_memory_info
 
@@ -21,6 +23,49 @@ def test_get_memory_info_returns_valid_values() -> None:
     assert memory_info.current_size < memory_info.total_size
 
 
+def test_get_memory_info_skips_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A child process we are not allowed to inspect must be skipped, not abort the whole snapshot.
+
+    In restricted environments (e.g. hardened containers) reading a child's memory can raise
+    `psutil.AccessDenied`, which is not a subclass of `psutil.NoSuchProcess` and so was not suppressed.
+    """
+    child = psutil.Process()  # any process object works; only `_get_used_memory` behavior matters here
+
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child])
+
+    def fake_get_used_memory(process: psutil.Process) -> int:
+        if process is child:
+            raise psutil.AccessDenied(pid=child.pid)
+        return 100
+
+    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
+
+    memory_info = get_memory_info()
+
+    # The unreadable child is skipped, so only the current process (100) is counted.
+    assert memory_info.current_size == ByteSize(100)
+
+
+def test_get_memory_info_falls_back_to_rss_when_current_process_access_denied(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """If PSS for the current process is denied, fall back to RSS instead of crashing.
+
+    On Linux `_get_used_memory` reads PSS via `memory_full_info`, which may require elevated privileges.
+    """
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [])
+
+    def fake_get_used_memory(process: psutil.Process) -> int:
+        raise psutil.AccessDenied(pid=process.pid)
+
+    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
+
+    memory_info = get_memory_info()
+
+    # RSS of the current process is a positive value below total system memory.
+    assert ByteSize(0) < memory_info.current_size < memory_info.total_size
+
+
 def test_get_cpu_info_returns_valid_values() -> None:
     cpu_info = get_cpu_info()
     assert 0 <= cpu_info.used_ratio <= 1

From cd699de9a89d49da689e5a393e7c04416ad90725 Mon Sep 17 00:00:00 2001
From: Anas Khan <83116240+anxkhn@users.noreply.github.com>
Date: Mon, 27 Jul 2026 18:30:30 +0530
Subject: [PATCH 02/11] fix(system): preserve child RSS when PSS is denied

Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>
---
 src/crawlee/_utils/system.py     | 14 +++++++++-----
 tests/unit/_utils/test_system.py | 20 +++++++++++++-------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 757b19e062..139e1cea02 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -128,14 +128,18 @@ def get_memory_info() -> MemoryInfo:
     try:
         current_size_bytes = _get_used_memory(current_process)
     except psutil.AccessDenied:
+        logger.debug('PSS access denied for the current process, falling back to RSS.')
         current_size_bytes = int(current_process.memory_info().rss)
 
     # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
-    for child in current_process.children(recursive=True):
-        # Ignore a child that ends before we retrieve its memory usage (`NoSuchProcess`) or that we are not
-        # allowed to inspect (`AccessDenied`, e.g. an unreadable subprocess in a restricted environment).
-        with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
-            current_size_bytes += _get_used_memory(child)
+    with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
+        for child in current_process.children(recursive=True):
+            try:
+                current_size_bytes += _get_used_memory(child)
+            except psutil.AccessDenied:  # noqa: PERF203
+                logger.debug('PSS access denied for child process %s, falling back to RSS.', child.pid)
+                with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
+                    current_size_bytes += int(child.memory_info().rss)
 
     vm = psutil.virtual_memory()
 
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 9e094f355f..812d808db7 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -3,6 +3,7 @@
 import sys
 from multiprocessing import get_context, synchronize
 from multiprocessing.shared_memory import SharedMemory
+from types import SimpleNamespace
 from typing import TYPE_CHECKING
 
 import psutil
@@ -23,15 +24,11 @@ def test_get_memory_info_returns_valid_values() -> None:
     assert memory_info.current_size < memory_info.total_size
 
 
-def test_get_memory_info_skips_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
-    """A child process we are not allowed to inspect must be skipped, not abort the whole snapshot.
-
-    In restricted environments (e.g. hardened containers) reading a child's memory can raise
-    `psutil.AccessDenied`, which is not a subclass of `psutil.NoSuchProcess` and so was not suppressed.
-    """
+def test_get_memory_info_falls_back_to_rss_for_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
     child = psutil.Process()  # any process object works; only `_get_used_memory` behavior matters here
 
     monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child])
+    monkeypatch.setattr(child, 'memory_info', lambda: SimpleNamespace(rss=50))
 
     def fake_get_used_memory(process: psutil.Process) -> int:
         if process is child:
@@ -42,7 +39,16 @@ def fake_get_used_memory(process: psutil.Process) -> int:
 
     memory_info = get_memory_info()
 
-    # The unreadable child is skipped, so only the current process (100) is counted.
+    assert memory_info.current_size == ByteSize(150)
+
+
+@pytest.mark.parametrize('error', [psutil.AccessDenied(), psutil.NoSuchProcess(pid=1)])
+def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None:
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: (_ for _ in ()).throw(error))
+    monkeypatch.setattr(system, '_get_used_memory', lambda _process: 100)
+
+    memory_info = get_memory_info()
+
     assert memory_info.current_size == ByteSize(100)
 
 

From 8d257e5bf0ceaa675ee419ba05f178d0e075a8bb Mon Sep 17 00:00:00 2001
From: Anas Khan <83116240+anxkhn@users.noreply.github.com>
Date: Mon, 27 Jul 2026 22:07:51 +0530
Subject: [PATCH 03/11] fix(system): continue after child process exits

Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>
---
 src/crawlee/_utils/system.py     | 30 +++++++++++++++---------------
 tests/unit/_utils/test_system.py | 32 ++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 139e1cea02..6b321bb536 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -122,24 +122,16 @@ def get_memory_info() -> MemoryInfo:
     logger.debug('Calling get_memory_info()...')
     current_process = psutil.Process(os.getpid())
 
-    # Retrieve estimated memory usage of the current process. On Linux `_get_used_memory` reads PSS via
-    # `memory_full_info`, which can raise `AccessDenied` in restricted environments (e.g. hardened containers);
-    # fall back to RSS, which a process can always read for itself.
-    try:
-        current_size_bytes = _get_used_memory(current_process)
-    except psutil.AccessDenied:
-        logger.debug('PSS access denied for the current process, falling back to RSS.')
-        current_size_bytes = int(current_process.memory_info().rss)
+    current_size_bytes = _get_used_memory_with_rss_fallback(current_process)
 
     # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
+    children: list[psutil.Process] = []
     with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
-        for child in current_process.children(recursive=True):
-            try:
-                current_size_bytes += _get_used_memory(child)
-            except psutil.AccessDenied:  # noqa: PERF203
-                logger.debug('PSS access denied for child process %s, falling back to RSS.', child.pid)
-                with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
-                    current_size_bytes += int(child.memory_info().rss)
+        children = current_process.children(recursive=True)
+
+    for child in children:
+        with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
+            current_size_bytes += _get_used_memory_with_rss_fallback(child)
 
     vm = psutil.virtual_memory()
 
@@ -148,3 +140,11 @@ def get_memory_info() -> MemoryInfo:
         current_size=ByteSize(current_size_bytes),
         system_wide_used_size=ByteSize(vm.total - vm.available),
     )
+
+
+def _get_used_memory_with_rss_fallback(process: psutil.Process) -> int:
+    try:
+        return _get_used_memory(process)
+    except psutil.AccessDenied:
+        logger.debug('PSS access denied for process %s, falling back to RSS.', process.pid)
+        return int(process.memory_info().rss)
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 812d808db7..5b7c32d03d 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -25,7 +25,8 @@ def test_get_memory_info_returns_valid_values() -> None:
 
 
 def test_get_memory_info_falls_back_to_rss_for_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
-    child = psutil.Process()  # any process object works; only `_get_used_memory` behavior matters here
+    """Test that denied child PSS reads fall back to RSS."""
+    child = psutil.Process()
 
     monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child])
     monkeypatch.setattr(child, 'memory_info', lambda: SimpleNamespace(rss=50))
@@ -42,8 +43,15 @@ def fake_get_used_memory(process: psutil.Process) -> int:
     assert memory_info.current_size == ByteSize(150)
 
 
-@pytest.mark.parametrize('error', [psutil.AccessDenied(), psutil.NoSuchProcess(pid=1)])
+@pytest.mark.parametrize(
+    'error',
+    [
+        pytest.param(psutil.AccessDenied(), id='access denied'),
+        pytest.param(psutil.NoSuchProcess(pid=1), id='no such process'),
+    ],
+)
 def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None:
+    """Test that failure to list children does not abort memory collection."""
     monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: (_ for _ in ()).throw(error))
     monkeypatch.setattr(system, '_get_used_memory', lambda _process: 100)
 
@@ -52,6 +60,26 @@ def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.Mo
     assert memory_info.current_size == ByteSize(100)
 
 
+def test_get_memory_info_continues_after_child_exits(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Test that a child exiting mid-iteration does not hide later children."""
+    exited_child = psutil.Process()
+    live_child = psutil.Process()
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [exited_child, live_child])
+
+    def fake_get_used_memory(process: psutil.Process) -> int:
+        if process is exited_child:
+            raise psutil.NoSuchProcess(pid=process.pid)
+        if process is live_child:
+            return 40
+        return 100
+
+    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
+
+    memory_info = get_memory_info()
+
+    assert memory_info.current_size == ByteSize(140)
+
+
 def test_get_memory_info_falls_back_to_rss_when_current_process_access_denied(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

From c89e358780ed90af9c108b3d5c9b0827f5ef31bf Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 09:53:30 +0200
Subject: [PATCH 04/11] fix(system): harden the PSS to RSS memory fallback

- Move the fallback into the Linux-only `_get_used_memory`; elsewhere it re-ran the call that had just failed.
- Cover a missing or zero `pss` field, not just `AccessDenied`.
- Warn once about the degraded estimate instead of a debug record per snapshot, and log unlistable children.
- Test the real fallback instead of only a patched `_get_used_memory`.
---
 src/crawlee/_utils/system.py     |  49 ++++++++++----
 tests/unit/_utils/test_system.py | 107 +++++++++++++++++--------------
 2 files changed, 95 insertions(+), 61 deletions(-)

diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 6b321bb536..99146dd9ea 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -4,15 +4,17 @@
 import sys
 from contextlib import suppress
 from datetime import datetime, timezone
-from logging import getLogger
+from logging import WARNING, getLogger
 from typing import TYPE_CHECKING, Annotated
 
 import psutil
 from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator
 
 from crawlee._utils.byte_size import ByteSize
+from crawlee._utils.log import LoggerOnce
 
 logger = getLogger(__name__)
+logger_once = LoggerOnce(logger)
 
 if sys.platform == 'linux':
     """Get the most suitable available used memory metric.
@@ -26,7 +28,23 @@
     """
 
     def _get_used_memory(process: psutil.Process) -> int:
-        return int(process.memory_full_info().pss)
+        # A restricted environment may deny `/proc/<pid>/smaps` or not expose it at all - psutil then aliases
+        # `memory_full_info` to `memory_info`, which has no `pss`. An unreadable `smaps` parses to a PSS of zero.
+        try:
+            pss = getattr(process.memory_full_info(), 'pss', 0)
+        except psutil.AccessDenied:
+            pss = 0
+
+        if pss:
+            return int(pss)
+
+        # RSS counts shared memory in full for every process that maps it, so a sharing process tree is overestimated.
+        logger_once.log(
+            'Unable to read the PSS memory metric, falling back to RSS - shared memory may be counted repeatedly.',
+            key='pss_unavailable',
+            level=WARNING,
+        )
+        return int(process.memory_info().rss)
 else:
 
     def _get_used_memory(process: psutil.Process) -> int:
@@ -117,21 +135,32 @@ def get_cpu_info() -> CpuInfo:
 def get_memory_info() -> MemoryInfo:
     """Retrieve the current memory usage of the process and its children.
 
-    It utilizes the `psutil` library.
+    It utilizes the `psutil` library. The reported `current_size` is best-effort - processes that cannot be inspected
+    are left out of the sum and PSS may be substituted by RSS.
     """
     logger.debug('Calling get_memory_info()...')
     current_process = psutil.Process(os.getpid())
 
-    current_size_bytes = _get_used_memory_with_rss_fallback(current_process)
+    # Retrieve estimated memory usage of the current process.
+    current_size_bytes = _get_used_memory(current_process)
 
     # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
     children: list[psutil.Process] = []
-    with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
+    try:
         children = current_process.children(recursive=True)
+    except (psutil.NoSuchProcess, psutil.AccessDenied):
+        # A missing child list hides the whole subprocess tree from the estimate, so do not degrade silently.
+        logger_once.log(
+            'Unable to list child processes, their memory usage is excluded from the estimate.',
+            key='children_unavailable',
+            level=WARNING,
+        )
 
     for child in children:
+        # Skip a child that exits mid-loop (`NoSuchProcess`) or that we cannot inspect (`AccessDenied`); either way
+        # it drops out of the sum.
         with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
-            current_size_bytes += _get_used_memory_with_rss_fallback(child)
+            current_size_bytes += _get_used_memory(child)
 
     vm = psutil.virtual_memory()
 
@@ -140,11 +169,3 @@ def get_memory_info() -> MemoryInfo:
         current_size=ByteSize(current_size_bytes),
         system_wide_used_size=ByteSize(vm.total - vm.available),
     )
-
-
-def _get_used_memory_with_rss_fallback(process: psutil.Process) -> int:
-    try:
-        return _get_used_memory(process)
-    except psutil.AccessDenied:
-        logger.debug('PSS access denied for process %s, falling back to RSS.', process.pid)
-        return int(process.memory_info().rss)
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 5b7c32d03d..024e5b334e 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -17,62 +17,74 @@
     from collections.abc import Callable
 
 
-def test_get_memory_info_returns_valid_values() -> None:
-    memory_info = get_memory_info()
+class FakeProcess:
+    """Stand-in for `psutil.Process` that lets a test decide what a child process reports as its memory usage."""
 
-    assert ByteSize(0) < memory_info.total_size < ByteSize.from_tb(1)
-    assert memory_info.current_size < memory_info.total_size
+    def __init__(self, pid: int, used_memory: int | Exception) -> None:
+        self.pid = pid
+        self.used_memory = used_memory
 
 
-def test_get_memory_info_falls_back_to_rss_for_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Test that denied child PSS reads fall back to RSS."""
-    child = psutil.Process()
+def fake_get_used_memory(process: psutil.Process | FakeProcess) -> int:
+    """Report the scripted value for a `FakeProcess` child and a fixed value for the real current process."""
+    if not isinstance(process, FakeProcess):
+        return 100
 
-    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child])
-    monkeypatch.setattr(child, 'memory_info', lambda: SimpleNamespace(rss=50))
+    used_memory = process.used_memory
+    if isinstance(used_memory, Exception):
+        raise used_memory
+    return used_memory
 
-    def fake_get_used_memory(process: psutil.Process) -> int:
-        if process is child:
-            raise psutil.AccessDenied(pid=child.pid)
-        return 100
 
-    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
+def raise_access_denied(process: psutil.Process) -> object:
+    """Stand in for `psutil.Process.memory_full_info` in an environment that refuses to expose PSS."""
+    raise psutil.AccessDenied(pid=process.pid)
 
+
+def test_get_memory_info_returns_valid_values() -> None:
     memory_info = get_memory_info()
 
-    assert memory_info.current_size == ByteSize(150)
+    assert ByteSize(0) < memory_info.total_size < ByteSize.from_tb(1)
+    assert memory_info.current_size < memory_info.total_size
 
 
+@pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')
 @pytest.mark.parametrize(
-    'error',
+    'memory_full_info',
     [
-        pytest.param(psutil.AccessDenied(), id='access denied'),
-        pytest.param(psutil.NoSuchProcess(pid=1), id='no such process'),
+        pytest.param(raise_access_denied, id='access denied'),
+        pytest.param(lambda _process: SimpleNamespace(), id='pss field missing'),
+        pytest.param(lambda _process: SimpleNamespace(pss=0), id='pss reported as zero'),
     ],
 )
-def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None:
-    """Test that failure to list children does not abort memory collection."""
-    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: (_ for _ in ()).throw(error))
-    monkeypatch.setattr(system, '_get_used_memory', lambda _process: 100)
+def test_get_used_memory_falls_back_to_rss_when_pss_is_unavailable(
+    monkeypatch: pytest.MonkeyPatch, memory_full_info: Callable[[psutil.Process], object]
+) -> None:
+    """An unreadable PSS metric falls back to the RSS of the same process instead of raising or reporting zero."""
+    monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
+    monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=1024))
+
+    assert system._get_used_memory(psutil.Process()) == 1024
+
+
+def test_get_memory_info_skips_children_that_cannot_be_measured(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A child that cannot be inspected at all is left out of the sum instead of aborting the snapshot."""
+    children = [FakeProcess(pid=1001, used_memory=psutil.AccessDenied(pid=1001)), FakeProcess(pid=1002, used_memory=40)]
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: children)
+    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
 
     memory_info = get_memory_info()
 
-    assert memory_info.current_size == ByteSize(100)
+    assert memory_info.current_size == ByteSize(140)
 
 
 def test_get_memory_info_continues_after_child_exits(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Test that a child exiting mid-iteration does not hide later children."""
-    exited_child = psutil.Process()
-    live_child = psutil.Process()
-    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [exited_child, live_child])
-
-    def fake_get_used_memory(process: psutil.Process) -> int:
-        if process is exited_child:
-            raise psutil.NoSuchProcess(pid=process.pid)
-        if process is live_child:
-            return 40
-        return 100
-
+    """A child exiting mid-iteration does not hide the children measured after it."""
+    children = [
+        FakeProcess(pid=1001, used_memory=psutil.NoSuchProcess(pid=1001)),
+        FakeProcess(pid=1002, used_memory=40),
+    ]
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: children)
     monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
 
     memory_info = get_memory_info()
@@ -80,24 +92,25 @@ def fake_get_used_memory(process: psutil.Process) -> int:
     assert memory_info.current_size == ByteSize(140)
 
 
-def test_get_memory_info_falls_back_to_rss_when_current_process_access_denied(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """If PSS for the current process is denied, fall back to RSS instead of crashing.
-
-    On Linux `_get_used_memory` reads PSS via `memory_full_info`, which may require elevated privileges.
-    """
-    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [])
+@pytest.mark.parametrize(
+    'error',
+    [
+        pytest.param(psutil.AccessDenied(), id='access denied'),
+        pytest.param(psutil.NoSuchProcess(pid=1), id='no such process'),
+    ],
+)
+def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None:
+    """Failure to list the child processes does not abort the whole memory snapshot."""
 
-    def fake_get_used_memory(process: psutil.Process) -> int:
-        raise psutil.AccessDenied(pid=process.pid)
+    def raise_error(*_args: object, **_kwargs: object) -> list[psutil.Process]:
+        raise error
 
+    monkeypatch.setattr(psutil.Process, 'children', raise_error)
     monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
 
     memory_info = get_memory_info()
 
-    # RSS of the current process is a positive value below total system memory.
-    assert ByteSize(0) < memory_info.current_size < memory_info.total_size
+    assert memory_info.current_size == ByteSize(100)
 
 
 def test_get_cpu_info_returns_valid_values() -> None:

From 1401d00d8c88b9c99c606270834bf0d1466b406f Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 10:17:38 +0200
Subject: [PATCH 05/11] fix(system): harden the memory estimate against
 unreadable metrics

Inspecting a process can also fail with a bare `OSError` - psutil re-raises
`FileNotFoundError` for a live process whose `/proc` entry is missing - which
escaped the previous `psutil.AccessDenied` handling. Catch it wherever a process
is inspected, and make the PSS to RSS fallback report what actually happened:

- Warn about PSS only when psutil exposes no `pss` field at all, and latch that
  verdict instead of re-parsing `smaps` for every process on every sample. A
  single denied process falls back to RSS just for itself, and a `smaps` file
  that parses to zero is no longer reported as a missing metric.
- Report a child that cannot be measured at all separately from one that exited.
- Cap the estimate at the total memory of the machine, so summed RSS cannot pin
  the autoscaler in a critically overloaded state.
- Lock `LoggerOnce`, which is now reached from the metric sampling thread.
---
 src/crawlee/_utils/log.py        |  13 ++-
 src/crawlee/_utils/system.py     | 117 +++++++++++++++++++-------
 tests/unit/_utils/test_log.py    |  23 ++++++
 tests/unit/_utils/test_system.py | 137 +++++++++++++++++++++++--------
 4 files changed, 224 insertions(+), 66 deletions(-)

diff --git a/src/crawlee/_utils/log.py b/src/crawlee/_utils/log.py
index 04c932310f..9077526e3f 100644
--- a/src/crawlee/_utils/log.py
+++ b/src/crawlee/_utils/log.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import threading
 
 
 class LoggerOnce:
@@ -9,11 +10,14 @@ class LoggerOnce:
     Useful for diagnostic warnings that would otherwise spam the log when the same condition recurs (per-request
     misconfiguration warnings, repeated fallback paths, etc.). Deduplication scope follows the lifetime of the
     instance — a module-level instance gives process-wide dedup; an attribute on a class gives per-instance dedup.
+
+    Safe to call from multiple threads - some callers (e.g. system metric sampling) run in a worker thread.
     """
 
     def __init__(self, logger: logging.Logger) -> None:
         self._logger = logger
         self._seen: set[str] = set()
+        self._lock = threading.Lock()
 
     def log(self, message: str, *, key: str, level: int = logging.INFO) -> None:
         """Log `message` at `level` the first time `key` is seen on this instance; later calls are no-ops.
@@ -23,7 +27,10 @@ def log(self, message: str, *, key: str, level: int = logging.INFO) -> None:
             key: Deduplication key. Two calls with the same key emit at most once.
             level: Standard `logging` level (e.g. `logging.WARNING`). Defaults to `logging.INFO`.
         """
-        if key in self._seen:
-            return
-        self._seen.add(key)
+        # The check and the insert have to be atomic, otherwise two threads racing on the same key both emit.
+        with self._lock:
+            if key in self._seen:
+                return
+            self._seen.add(key)
+
         self._logger.log(level, message)
diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 99146dd9ea..49bfd4ba1e 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -16,41 +16,94 @@
 logger = getLogger(__name__)
 logger_once = LoggerOnce(logger)
 
-if sys.platform == 'linux':
-    """Get the most suitable available used memory metric.
+# Reading a memory metric of a process that is denied or gone raises either a `psutil.Error` or a bare `OSError` -
+# psutil re-raises `FileNotFoundError` as is when a `/proc` entry is missing for a process that is still alive.
+_METRIC_ERRORS = (psutil.Error, OSError)
+
 
-    `Proportional Set Size (PSS)`, is the amount of own memory and memory shared with other processes, accounted in a
-    way that the shared amount is divided evenly between the processes that share it. Available on Linux. Suitable for
-    avoiding overestimation by counting the same shared memory used by children processes multiple times.
+class _PssAvailability:
+    """Process-wide latch for whether the PSS memory metric exists on this system at all.
 
-    `Resident Set Size (RSS)` is the non-swapped physical memory a process has used; it includes shared memory. It
-    should be available everywhere.
+    Memory is sampled on a short recurring interval, so once psutil is known to not expose PSS there is no point in
+    asking for it again for every process on every sample. Only the system-wide verdict is latched; a single process
+    refusing to be inspected says nothing about the others.
     """
 
-    def _get_used_memory(process: psutil.Process) -> int:
-        # A restricted environment may deny `/proc/<pid>/smaps` or not expose it at all - psutil then aliases
-        # `memory_full_info` to `memory_info`, which has no `pss`. An unreadable `smaps` parses to a PSS of zero.
-        try:
-            pss = getattr(process.memory_full_info(), 'pss', 0)
-        except psutil.AccessDenied:
-            pss = 0
+    is_available = True
 
-        if pss:
-            return int(pss)
 
-        # RSS counts shared memory in full for every process that maps it, so a sharing process tree is overestimated.
-        logger_once.log(
-            'Unable to read the PSS memory metric, falling back to RSS - shared memory may be counted repeatedly.',
-            key='pss_unavailable',
-            level=WARNING,
-        )
+if sys.platform == 'linux':
+
+    def _get_used_memory(process: psutil.Process) -> int:
+        """Get the most suitable available used memory metric of a single process.
+
+        `Proportional Set Size (PSS)` is the amount of own memory and memory shared with other processes, accounted in
+        a way that the shared amount is divided evenly between the processes that share it. Available on Linux.
+        Suitable for avoiding overestimation by counting the same shared memory used by children processes multiple
+        times.
+
+        `Resident Set Size (RSS)` is the non-swapped physical memory a process has used; it includes shared memory. It
+        should be available everywhere, so it is used whenever PSS cannot be read. It counts shared memory in full for
+        every process that maps it, so a sharing process tree gets overestimated.
+
+        Raises:
+            psutil.Error: If the process refuses inspection or is gone.
+            OSError: If a `/proc` entry of the process is missing.
+        """
+        if _PssAvailability.is_available:
+            # A restricted environment may deny `/proc/<pid>/smaps`, which is a property of the single process, so it
+            # only falls back to RSS for that one process.
+            with suppress(*_METRIC_ERRORS):
+                # A system that does not expose `smaps` at all makes psutil alias `memory_full_info` to
+                # `memory_info`, whose result has no `pss` field.
+                pss = getattr(process.memory_full_info(), 'pss', None)
+
+                if pss is None:
+                    _PssAvailability.is_available = False
+                    logger_once.log(
+                        'Unable to read the PSS memory metric, falling back to RSS - shared memory may be counted '
+                        'repeatedly.',
+                        key='pss_unavailable',
+                        level=WARNING,
+                    )
+                # A `smaps` file can be empty for some processes, which parses to a PSS of zero. No live process
+                # really uses zero memory, so treat it as a missing reading rather than as a measurement.
+                elif pss > 0:
+                    return int(pss)
+
         return int(process.memory_info().rss)
 else:
 
     def _get_used_memory(process: psutil.Process) -> int:
+        """Get the used memory metric of a single process.
+
+        `Resident Set Size (RSS)` is the non-swapped physical memory a process has used; it includes shared memory, so
+        a process tree that shares memory gets overestimated. It is the only metric available outside of Linux.
+
+        Raises:
+            psutil.Error: If the process refuses inspection or is gone.
+            OSError: If the memory metric of the process cannot be read.
+        """
         return int(process.memory_info().rss)
 
 
+def _get_child_used_memory(child: psutil.Process) -> int:
+    """Get the used memory of a child process, or zero if the child cannot be measured at all."""
+    try:
+        return _get_used_memory(child)
+    except psutil.NoSuchProcess:
+        # A child that exits mid-measurement just drops out of the sum, which is business as usual.
+        return 0
+    except _METRIC_ERRORS:
+        # A child we cannot inspect at all drops out of the sum too, which does hide its memory usage.
+        logger_once.log(
+            'Unable to read the memory usage of a child process, it is excluded from the estimate.',
+            key='child_unmeasurable',
+            level=WARNING,
+        )
+        return 0
+
+
 class CpuInfo(BaseModel):
     """Information about the CPU usage."""
 
@@ -85,7 +138,11 @@ class MemoryUsageInfo(BaseModel):
         PlainSerializer(lambda size: size.bytes),
         Field(alias='currentSize'),
     ]
-    """Memory usage of the current Python process and its children."""
+    """Memory usage of the current Python process and its children.
+
+    This is a best-effort estimate - a process that cannot be inspected is left out of the sum, and the metric used
+    may be RSS, which counts memory shared between the processes repeatedly.
+    """
 
     # Workaround for Pydantic and type checkers when using Annotated with default_factory
     if TYPE_CHECKING:
@@ -136,7 +193,8 @@ def get_memory_info() -> MemoryInfo:
     """Retrieve the current memory usage of the process and its children.
 
     It utilizes the `psutil` library. The reported `current_size` is best-effort - processes that cannot be inspected
-    are left out of the sum and PSS may be substituted by RSS.
+    are left out of the sum, PSS may be substituted by RSS, and the result is capped at the total memory of the
+    machine.
     """
     logger.debug('Calling get_memory_info()...')
     current_process = psutil.Process(os.getpid())
@@ -148,7 +206,7 @@ def get_memory_info() -> MemoryInfo:
     children: list[psutil.Process] = []
     try:
         children = current_process.children(recursive=True)
-    except (psutil.NoSuchProcess, psutil.AccessDenied):
+    except _METRIC_ERRORS:
         # A missing child list hides the whole subprocess tree from the estimate, so do not degrade silently.
         logger_once.log(
             'Unable to list child processes, their memory usage is excluded from the estimate.',
@@ -157,13 +215,14 @@ def get_memory_info() -> MemoryInfo:
         )
 
     for child in children:
-        # Skip a child that exits mid-loop (`NoSuchProcess`) or that we cannot inspect (`AccessDenied`); either way
-        # it drops out of the sum.
-        with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
-            current_size_bytes += _get_used_memory(child)
+        current_size_bytes += _get_child_used_memory(child)
 
     vm = psutil.virtual_memory()
 
+    # Summing RSS counts memory shared between the processes repeatedly, which can add up to more than the machine
+    # has. Reporting more than the total would make the autoscaler throttle forever, so cap the estimate.
+    current_size_bytes = min(current_size_bytes, vm.total)
+
     return MemoryInfo(
         total_size=ByteSize(vm.total),
         current_size=ByteSize(current_size_bytes),
diff --git a/tests/unit/_utils/test_log.py b/tests/unit/_utils/test_log.py
index 40741a1f04..7c0968d886 100644
--- a/tests/unit/_utils/test_log.py
+++ b/tests/unit/_utils/test_log.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import threading
 from typing import TYPE_CHECKING
 
 from crawlee._utils.log import LoggerOnce
@@ -26,6 +27,28 @@ def test_duplicate_key_is_suppressed(caplog: pytest.LogCaptureFixture) -> None:
     assert [r.getMessage() for r in caplog.records] == ['first']
 
 
+def test_threads_racing_on_one_key_log_once(caplog: pytest.LogCaptureFixture) -> None:
+    """Threads reaching the same key at once emit a single record - metrics are sampled in a worker thread."""
+    logger = logging.getLogger('crawlee.tests.log_dedup_threads')
+    logger_once = LoggerOnce(logger)
+    thread_count = 16
+    ready = threading.Barrier(thread_count)
+
+    def log_once() -> None:
+        ready.wait()
+        logger_once.log('first', key='k1')
+
+    threads = [threading.Thread(target=log_once) for _ in range(thread_count)]
+
+    with caplog.at_level(logging.INFO, logger=logger.name):
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+
+    assert [r.getMessage() for r in caplog.records] == ['first']
+
+
 def test_distinct_keys_each_log(caplog: pytest.LogCaptureFixture) -> None:
     logger = logging.getLogger('crawlee.tests.log_dedup_distinct')
     logger_once = LoggerOnce(logger)
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 024e5b334e..ae489b438e 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import sys
 from multiprocessing import get_context, synchronize
 from multiprocessing.shared_memory import SharedMemory
@@ -11,6 +12,7 @@
 
 from crawlee._utils import system
 from crawlee._utils.byte_size import ByteSize
+from crawlee._utils.log import LoggerOnce
 from crawlee._utils.system import get_cpu_info, get_memory_info
 
 if TYPE_CHECKING:
@@ -24,16 +26,15 @@ def __init__(self, pid: int, used_memory: int | Exception) -> None:
         self.pid = pid
         self.used_memory = used_memory
 
+    def memory_full_info(self) -> object:
+        if isinstance(self.used_memory, Exception):
+            raise self.used_memory
+        return SimpleNamespace(pss=self.used_memory)
 
-def fake_get_used_memory(process: psutil.Process | FakeProcess) -> int:
-    """Report the scripted value for a `FakeProcess` child and a fixed value for the real current process."""
-    if not isinstance(process, FakeProcess):
-        return 100
-
-    used_memory = process.used_memory
-    if isinstance(used_memory, Exception):
-        raise used_memory
-    return used_memory
+    def memory_info(self) -> object:
+        if isinstance(self.used_memory, Exception):
+            raise self.used_memory
+        return SimpleNamespace(rss=self.used_memory)
 
 
 def raise_access_denied(process: psutil.Process) -> object:
@@ -41,6 +42,20 @@ def raise_access_denied(process: psutil.Process) -> object:
     raise psutil.AccessDenied(pid=process.pid)
 
 
+@pytest.fixture(autouse=True)
+def _isolated_module_state(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Reset the process-wide state of the module, so that dedup keys and the PSS latch do not leak between tests."""
+    monkeypatch.setattr(system, 'logger_once', LoggerOnce(system.logger))
+    monkeypatch.setattr(system._PssAvailability, 'is_available', True)
+
+
+@pytest.fixture
+def measured_current_process(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Make the real current process report a fixed memory usage of 100 bytes, whichever metric is used."""
+    monkeypatch.setattr(psutil.Process, 'memory_full_info', lambda _process: SimpleNamespace(pss=100))
+    monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=100))
+
+
 def test_get_memory_info_returns_valid_values() -> None:
     memory_info = get_memory_info()
 
@@ -50,67 +65,121 @@ def test_get_memory_info_returns_valid_values() -> None:
 
 @pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')
 @pytest.mark.parametrize(
-    'memory_full_info',
+    ('memory_full_info', 'expected_warning'),
     [
-        pytest.param(raise_access_denied, id='access denied'),
-        pytest.param(lambda _process: SimpleNamespace(), id='pss field missing'),
-        pytest.param(lambda _process: SimpleNamespace(pss=0), id='pss reported as zero'),
+        pytest.param(raise_access_denied, False, id='access denied'),
+        pytest.param(lambda _process: SimpleNamespace(pss=0), False, id='pss reported as zero'),
+        pytest.param(lambda _process: SimpleNamespace(), True, id='pss field missing'),
     ],
 )
 def test_get_used_memory_falls_back_to_rss_when_pss_is_unavailable(
-    monkeypatch: pytest.MonkeyPatch, memory_full_info: Callable[[psutil.Process], object]
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    memory_full_info: Callable[[psutil.Process], object],
+    *,
+    expected_warning: bool,
 ) -> None:
     """An unreadable PSS metric falls back to the RSS of the same process instead of raising or reporting zero."""
     monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
     monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=1024))
 
-    assert system._get_used_memory(psutil.Process()) == 1024
+    with caplog.at_level(logging.WARNING, logger=system.logger.name):
+        assert system._get_used_memory(psutil.Process()) == 1024
 
+    # Only a system that has no PSS at all is worth reporting - a single process denying it says nothing about the
+    # others, and an empty `smaps` parses to zero for a process that is on its way out.
+    warnings = [record.getMessage() for record in caplog.records if 'PSS' in record.getMessage()]
+    assert bool(warnings) == expected_warning
+    assert system._PssAvailability.is_available != expected_warning
 
-def test_get_memory_info_skips_children_that_cannot_be_measured(monkeypatch: pytest.MonkeyPatch) -> None:
-    """A child that cannot be inspected at all is left out of the sum instead of aborting the snapshot."""
-    children = [FakeProcess(pid=1001, used_memory=psutil.AccessDenied(pid=1001)), FakeProcess(pid=1002, used_memory=40)]
-    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: children)
-    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
 
-    memory_info = get_memory_info()
+@pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')
+def test_get_used_memory_stops_asking_for_pss_once_it_is_known_to_be_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A system without PSS is asked for it only once, not on every sample of every process."""
+    call_count = 0
 
-    assert memory_info.current_size == ByteSize(140)
+    def memory_full_info(_process: psutil.Process) -> object:
+        nonlocal call_count
+        call_count += 1
+        return SimpleNamespace()
+
+    monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
+    monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=1024))
+
+    for _ in range(3):
+        assert system._get_used_memory(psutil.Process()) == 1024
+
+    assert call_count == 1
 
 
-def test_get_memory_info_continues_after_child_exits(monkeypatch: pytest.MonkeyPatch) -> None:
-    """A child exiting mid-iteration does not hide the children measured after it."""
-    children = [
-        FakeProcess(pid=1001, used_memory=psutil.NoSuchProcess(pid=1001)),
-        FakeProcess(pid=1002, used_memory=40),
-    ]
+@pytest.mark.parametrize(
+    ('error', 'expected_warning'),
+    [
+        pytest.param(psutil.AccessDenied(pid=1001), True, id='access denied'),
+        pytest.param(psutil.NoSuchProcess(pid=1001), False, id='no such process'),
+        pytest.param(FileNotFoundError('/proc/1001/smaps'), True, id='proc entry missing'),
+    ],
+)
+@pytest.mark.usefixtures('measured_current_process')
+def test_get_memory_info_skips_children_that_cannot_be_measured(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    error: Exception,
+    *,
+    expected_warning: bool,
+) -> None:
+    """A child that cannot be measured is left out of the sum without hiding the children measured after it."""
+    children = [FakeProcess(pid=1001, used_memory=error), FakeProcess(pid=1002, used_memory=40)]
     monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: children)
-    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
 
-    memory_info = get_memory_info()
+    with caplog.at_level(logging.WARNING, logger=system.logger.name):
+        memory_info = get_memory_info()
 
     assert memory_info.current_size == ByteSize(140)
 
+    # A child that exits mid-iteration is business as usual, an uninspectable one hides its memory usage.
+    warnings = [record.getMessage() for record in caplog.records if 'child process' in record.getMessage()]
+    assert bool(warnings) == expected_warning
+
 
 @pytest.mark.parametrize(
     'error',
     [
         pytest.param(psutil.AccessDenied(), id='access denied'),
         pytest.param(psutil.NoSuchProcess(pid=1), id='no such process'),
+        pytest.param(FileNotFoundError('/proc/1/stat'), id='proc entry missing'),
     ],
 )
-def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None:
-    """Failure to list the child processes does not abort the whole memory snapshot."""
+@pytest.mark.usefixtures('measured_current_process')
+def test_get_memory_info_handles_failure_to_list_children(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, error: Exception
+) -> None:
+    """Failure to list the child processes does not abort the whole memory snapshot, but is reported."""
 
     def raise_error(*_args: object, **_kwargs: object) -> list[psutil.Process]:
         raise error
 
     monkeypatch.setattr(psutil.Process, 'children', raise_error)
-    monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory)
 
-    memory_info = get_memory_info()
+    with caplog.at_level(logging.WARNING, logger=system.logger.name):
+        memory_info = get_memory_info()
 
     assert memory_info.current_size == ByteSize(100)
+    assert [record.getMessage() for record in caplog.records if 'child processes' in record.getMessage()]
+
+
+@pytest.mark.usefixtures('measured_current_process')
+def test_get_memory_info_caps_the_estimate_at_the_total_memory_size(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Summing RSS of processes that share memory cannot report more memory than the machine has."""
+    total_size = psutil.virtual_memory().total
+    children = [FakeProcess(pid=1001, used_memory=total_size), FakeProcess(pid=1002, used_memory=total_size)]
+    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: children)
+
+    memory_info = get_memory_info()
+
+    assert memory_info.current_size == ByteSize(total_size)
 
 
 def test_get_cpu_info_returns_valid_values() -> None:

From a83f568d6ed20cb2e6946046fe9f5aa09c1759ba Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 12:18:55 +0200
Subject: [PATCH 06/11] fix(system): drop the memory estimate cap and reuse the
 already read RSS

---
 src/crawlee/_utils/log.py        |  2 +-
 src/crawlee/_utils/system.py     | 21 ++++++++++++---------
 tests/unit/_utils/test_log.py    |  3 ++-
 tests/unit/_utils/test_system.py | 32 ++++++++++++--------------------
 4 files changed, 27 insertions(+), 31 deletions(-)

diff --git a/src/crawlee/_utils/log.py b/src/crawlee/_utils/log.py
index 9077526e3f..d6bb2e8361 100644
--- a/src/crawlee/_utils/log.py
+++ b/src/crawlee/_utils/log.py
@@ -9,7 +9,7 @@ class LoggerOnce:
 
     Useful for diagnostic warnings that would otherwise spam the log when the same condition recurs (per-request
     misconfiguration warnings, repeated fallback paths, etc.). Deduplication scope follows the lifetime of the
-    instance — a module-level instance gives process-wide dedup; an attribute on a class gives per-instance dedup.
+    instance - a module-level instance gives process-wide dedup; an attribute on a class gives per-instance dedup.
 
     Safe to call from multiple threads - some callers (e.g. system metric sampling) run in a worker thread.
     """
diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 49bfd4ba1e..76386b6bfa 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -56,7 +56,8 @@ def _get_used_memory(process: psutil.Process) -> int:
             with suppress(*_METRIC_ERRORS):
                 # A system that does not expose `smaps` at all makes psutil alias `memory_full_info` to
                 # `memory_info`, whose result has no `pss` field.
-                pss = getattr(process.memory_full_info(), 'pss', None)
+                memory = process.memory_full_info()
+                pss = getattr(memory, 'pss', None)
 
                 if pss is None:
                     _PssAvailability.is_available = False
@@ -71,6 +72,10 @@ def _get_used_memory(process: psutil.Process) -> int:
                 elif pss > 0:
                     return int(pss)
 
+                # `memory_full_info` reads the RSS on its way to the PSS, so the fallback does not have to read it
+                # again.
+                return int(memory.rss)
+
         return int(process.memory_info().rss)
 else:
 
@@ -141,7 +146,8 @@ class MemoryUsageInfo(BaseModel):
     """Memory usage of the current Python process and its children.
 
     This is a best-effort estimate - a process that cannot be inspected is left out of the sum, and the metric used
-    may be RSS, which counts memory shared between the processes repeatedly.
+    may be RSS, which counts memory shared between the processes repeatedly. When only some of the processes expose
+    PSS, the sum mixes both metrics, so the memory those processes share with the rest of the tree is counted twice.
     """
 
     # Workaround for Pydantic and type checkers when using Annotated with default_factory
@@ -193,13 +199,14 @@ def get_memory_info() -> MemoryInfo:
     """Retrieve the current memory usage of the process and its children.
 
     It utilizes the `psutil` library. The reported `current_size` is best-effort - processes that cannot be inspected
-    are left out of the sum, PSS may be substituted by RSS, and the result is capped at the total memory of the
-    machine.
+    are left out of the sum, and PSS may be substituted by RSS for some or all of the processes.
     """
     logger.debug('Calling get_memory_info()...')
     current_process = psutil.Process(os.getpid())
 
-    # Retrieve estimated memory usage of the current process.
+    # Retrieve estimated memory usage of the current process. Deliberately not guarded - a process can always read
+    # its own RSS, and if it somehow cannot, failing the whole snapshot is safer than reporting a sum that is missing
+    # the main process: the autoscaler would read the gap as free memory and keep scaling up.
     current_size_bytes = _get_used_memory(current_process)
 
     # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
@@ -219,10 +226,6 @@ def get_memory_info() -> MemoryInfo:
 
     vm = psutil.virtual_memory()
 
-    # Summing RSS counts memory shared between the processes repeatedly, which can add up to more than the machine
-    # has. Reporting more than the total would make the autoscaler throttle forever, so cap the estimate.
-    current_size_bytes = min(current_size_bytes, vm.total)
-
     return MemoryInfo(
         total_size=ByteSize(vm.total),
         current_size=ByteSize(current_size_bytes),
diff --git a/tests/unit/_utils/test_log.py b/tests/unit/_utils/test_log.py
index 7c0968d886..63146e7a0b 100644
--- a/tests/unit/_utils/test_log.py
+++ b/tests/unit/_utils/test_log.py
@@ -32,7 +32,8 @@ def test_threads_racing_on_one_key_log_once(caplog: pytest.LogCaptureFixture) ->
     logger = logging.getLogger('crawlee.tests.log_dedup_threads')
     logger_once = LoggerOnce(logger)
     thread_count = 16
-    ready = threading.Barrier(thread_count)
+    # A timeout keeps a thread that never arrives from hanging the whole run - the assertion below fails instead.
+    ready = threading.Barrier(thread_count, timeout=10)
 
     def log_once() -> None:
         ready.wait()
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index ae489b438e..3810751031 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -29,7 +29,8 @@ def __init__(self, pid: int, used_memory: int | Exception) -> None:
     def memory_full_info(self) -> object:
         if isinstance(self.used_memory, Exception):
             raise self.used_memory
-        return SimpleNamespace(pss=self.used_memory)
+        # Mirrors psutil, whose full result carries the RSS alongside the PSS.
+        return SimpleNamespace(rss=self.used_memory, pss=self.used_memory)
 
     def memory_info(self) -> object:
         if isinstance(self.used_memory, Exception):
@@ -65,26 +66,29 @@ def test_get_memory_info_returns_valid_values() -> None:
 
 @pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')
 @pytest.mark.parametrize(
-    ('memory_full_info', 'expected_warning'),
+    ('memory_full_info', 'expected_size', 'expected_warning'),
     [
-        pytest.param(raise_access_denied, False, id='access denied'),
-        pytest.param(lambda _process: SimpleNamespace(pss=0), False, id='pss reported as zero'),
-        pytest.param(lambda _process: SimpleNamespace(), True, id='pss field missing'),
+        pytest.param(raise_access_denied, 2048, False, id='access denied'),
+        pytest.param(lambda _process: SimpleNamespace(rss=1024, pss=0), 1024, False, id='pss reported as zero'),
+        pytest.param(lambda _process: SimpleNamespace(rss=1024), 1024, True, id='pss field missing'),
     ],
 )
 def test_get_used_memory_falls_back_to_rss_when_pss_is_unavailable(
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
     memory_full_info: Callable[[psutil.Process], object],
+    expected_size: int,
     *,
     expected_warning: bool,
 ) -> None:
     """An unreadable PSS metric falls back to the RSS of the same process instead of raising or reporting zero."""
     monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
-    monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=1024))
+    # A value distinct from the RSS of the full result shows that the metric is re-read only when it has to be -
+    # a full result that was read successfully already carries the RSS.
+    monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=2048))
 
     with caplog.at_level(logging.WARNING, logger=system.logger.name):
-        assert system._get_used_memory(psutil.Process()) == 1024
+        assert system._get_used_memory(psutil.Process()) == expected_size
 
     # Only a system that has no PSS at all is worth reporting - a single process denying it says nothing about the
     # others, and an empty `smaps` parses to zero for a process that is on its way out.
@@ -103,7 +107,7 @@ def test_get_used_memory_stops_asking_for_pss_once_it_is_known_to_be_missing(
     def memory_full_info(_process: psutil.Process) -> object:
         nonlocal call_count
         call_count += 1
-        return SimpleNamespace()
+        return SimpleNamespace(rss=1024)
 
     monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
     monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=1024))
@@ -170,18 +174,6 @@ def raise_error(*_args: object, **_kwargs: object) -> list[psutil.Process]:
     assert [record.getMessage() for record in caplog.records if 'child processes' in record.getMessage()]
 
 
-@pytest.mark.usefixtures('measured_current_process')
-def test_get_memory_info_caps_the_estimate_at_the_total_memory_size(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Summing RSS of processes that share memory cannot report more memory than the machine has."""
-    total_size = psutil.virtual_memory().total
-    children = [FakeProcess(pid=1001, used_memory=total_size), FakeProcess(pid=1002, used_memory=total_size)]
-    monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: children)
-
-    memory_info = get_memory_info()
-
-    assert memory_info.current_size == ByteSize(total_size)
-
-
 def test_get_cpu_info_returns_valid_values() -> None:
     cpu_info = get_cpu_info()
     assert 0 <= cpu_info.used_ratio <= 1

From 2d9499b60ead5f16e470ce5209c75f6eb4b634d7 Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 13:03:05 +0200
Subject: [PATCH 07/11] test: drop the ineffective LoggerOnce race test and
 cover a zombie child

---
 tests/unit/_utils/test_log.py    | 24 ------------------------
 tests/unit/_utils/test_system.py |  1 +
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/tests/unit/_utils/test_log.py b/tests/unit/_utils/test_log.py
index 63146e7a0b..40741a1f04 100644
--- a/tests/unit/_utils/test_log.py
+++ b/tests/unit/_utils/test_log.py
@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-import threading
 from typing import TYPE_CHECKING
 
 from crawlee._utils.log import LoggerOnce
@@ -27,29 +26,6 @@ def test_duplicate_key_is_suppressed(caplog: pytest.LogCaptureFixture) -> None:
     assert [r.getMessage() for r in caplog.records] == ['first']
 
 
-def test_threads_racing_on_one_key_log_once(caplog: pytest.LogCaptureFixture) -> None:
-    """Threads reaching the same key at once emit a single record - metrics are sampled in a worker thread."""
-    logger = logging.getLogger('crawlee.tests.log_dedup_threads')
-    logger_once = LoggerOnce(logger)
-    thread_count = 16
-    # A timeout keeps a thread that never arrives from hanging the whole run - the assertion below fails instead.
-    ready = threading.Barrier(thread_count, timeout=10)
-
-    def log_once() -> None:
-        ready.wait()
-        logger_once.log('first', key='k1')
-
-    threads = [threading.Thread(target=log_once) for _ in range(thread_count)]
-
-    with caplog.at_level(logging.INFO, logger=logger.name):
-        for thread in threads:
-            thread.start()
-        for thread in threads:
-            thread.join()
-
-    assert [r.getMessage() for r in caplog.records] == ['first']
-
-
 def test_distinct_keys_each_log(caplog: pytest.LogCaptureFixture) -> None:
     logger = logging.getLogger('crawlee.tests.log_dedup_distinct')
     logger_once = LoggerOnce(logger)
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 02515d1001..b104c8c7e2 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -123,6 +123,7 @@ def memory_full_info(_process: psutil.Process) -> object:
     [
         pytest.param(psutil.AccessDenied(pid=1001), True, id='access denied'),
         pytest.param(psutil.NoSuchProcess(pid=1001), False, id='no such process'),
+        pytest.param(psutil.ZombieProcess(pid=1001), False, id='zombie process'),
         pytest.param(FileNotFoundError('/proc/1001/smaps'), True, id='proc entry missing'),
     ],
 )

From 278685d1883fcecde72700f08f0c78a4b47a0f3d Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 13:17:01 +0200
Subject: [PATCH 08/11] fix(system): report a per-process PSS denial instead of
 switching to RSS silently

---
 src/crawlee/_utils/system.py     | 19 +++++++++++++----
 tests/unit/_utils/test_system.py | 36 +++++++++++++++++++++++++-------
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 76386b6bfa..79e119b4d1 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -2,7 +2,6 @@
 
 import os
 import sys
-from contextlib import suppress
 from datetime import datetime, timezone
 from logging import WARNING, getLogger
 from typing import TYPE_CHECKING, Annotated
@@ -51,12 +50,24 @@ def _get_used_memory(process: psutil.Process) -> int:
             OSError: If a `/proc` entry of the process is missing.
         """
         if _PssAvailability.is_available:
-            # A restricted environment may deny `/proc/<pid>/smaps`, which is a property of the single process, so it
-            # only falls back to RSS for that one process.
-            with suppress(*_METRIC_ERRORS):
+            try:
                 # A system that does not expose `smaps` at all makes psutil alias `memory_full_info` to
                 # `memory_info`, whose result has no `pss` field.
                 memory = process.memory_full_info()
+            except (psutil.NoSuchProcess, psutil.ZombieProcess):
+                # A process that is gone is not refusing inspection, so let the RSS read below fail for it as usual.
+                pass
+            except _METRIC_ERRORS:
+                # A restricted environment may deny `/proc/<pid>/smaps`, which is a property of the single process, so
+                # only that one process falls back to RSS. Still worth reporting - when the denial covers the whole
+                # process tree, the estimate switches to RSS with nothing else to show it.
+                logger_once.log(
+                    'Unable to read the PSS memory metric of a process, falling back to RSS for it - shared memory '
+                    'may be counted repeatedly.',
+                    key='pss_denied',
+                    level=WARNING,
+                )
+            else:
                 pss = getattr(memory, 'pss', None)
 
                 if pss is None:
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index b104c8c7e2..02474eecd9 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -43,6 +43,11 @@ def raise_access_denied(process: psutil.Process) -> object:
     raise psutil.AccessDenied(pid=process.pid)
 
 
+def raise_no_such_process(process: psutil.Process) -> object:
+    """Stand in for a memory metric of a process that exits in the middle of being measured."""
+    raise psutil.NoSuchProcess(pid=process.pid)
+
+
 @pytest.fixture(autouse=True)
 def _isolated_module_state(monkeypatch: pytest.MonkeyPatch) -> None:
     """Reset the process-wide state of the module, so that dedup keys and the PSS latch do not leak between tests."""
@@ -66,11 +71,11 @@ def test_get_memory_info_returns_valid_values() -> None:
 
 @pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')
 @pytest.mark.parametrize(
-    ('memory_full_info', 'expected_size', 'expected_warning'),
+    ('memory_full_info', 'expected_size', 'expected_warning', 'pss_stays_available'),
     [
-        pytest.param(raise_access_denied, 2048, False, id='access denied'),
-        pytest.param(lambda _process: SimpleNamespace(rss=1024, pss=0), 1024, False, id='pss reported as zero'),
-        pytest.param(lambda _process: SimpleNamespace(rss=1024), 1024, True, id='pss field missing'),
+        pytest.param(raise_access_denied, 2048, True, True, id='access denied'),
+        pytest.param(lambda _process: SimpleNamespace(rss=1024, pss=0), 1024, False, True, id='pss reported as zero'),
+        pytest.param(lambda _process: SimpleNamespace(rss=1024), 1024, True, False, id='pss field missing'),
     ],
 )
 def test_get_used_memory_falls_back_to_rss_when_pss_is_unavailable(
@@ -80,6 +85,7 @@ def test_get_used_memory_falls_back_to_rss_when_pss_is_unavailable(
     expected_size: int,
     *,
     expected_warning: bool,
+    pss_stays_available: bool,
 ) -> None:
     """An unreadable PSS metric falls back to the RSS of the same process instead of raising or reporting zero."""
     monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
@@ -90,11 +96,27 @@ def test_get_used_memory_falls_back_to_rss_when_pss_is_unavailable(
     with caplog.at_level(logging.WARNING, logger=system.logger.name):
         assert system._get_used_memory(psutil.Process()) == expected_size
 
-    # Only a system that has no PSS at all is worth reporting - a single process denying it says nothing about the
-    # others, and an empty `smaps` parses to zero for a process that is on its way out.
+    # Silently switching the metric would leave an overestimated memory usage unexplained, so both a denial and a
+    # system without PSS are reported. An empty `smaps` parses to zero for a process on its way out, which is not.
     warnings = [record.getMessage() for record in caplog.records if 'PSS' in record.getMessage()]
     assert bool(warnings) == expected_warning
-    assert system._PssAvailability.is_available != expected_warning
+    # Only a system that has no PSS at all is latched - a single process denying it says nothing about the others.
+    assert system._PssAvailability.is_available == pss_stays_available
+
+
+@pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')
+def test_get_used_memory_does_not_report_a_vanished_process_as_a_pss_denial(
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A process that exits mid-measurement propagates rather than being reported as refusing to expose PSS."""
+    monkeypatch.setattr(psutil.Process, 'memory_full_info', raise_no_such_process)
+    monkeypatch.setattr(psutil.Process, 'memory_info', raise_no_such_process)
+
+    with caplog.at_level(logging.WARNING, logger=system.logger.name), pytest.raises(psutil.NoSuchProcess):
+        system._get_used_memory(psutil.Process())
+
+    assert not [record.getMessage() for record in caplog.records if 'PSS' in record.getMessage()]
 
 
 @pytest.mark.skipif(sys.platform != 'linux', reason='PSS is read only on Linux, elsewhere RSS is used directly')

From 0e526fc7e9f7866aaf0561f483fe027fa1c385bf Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 15:54:18 +0200
Subject: [PATCH 09/11] test: fill shared memory without a multi-gigabyte
 temporary list

---
 tests/unit/_utils/test_system.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 02474eecd9..381b42e717 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -231,7 +231,7 @@ def no_extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barr
         def extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barrier) -> None:
             memory = SharedMemory(size=extra_memory_size, create=True)
             assert memory.buf is not None
-            memory.buf[:] = bytearray([255 for _ in range(extra_memory_size)])
+            memory.buf[:] = b'\xff' * extra_memory_size
             print(f'Using the memory... {memory.buf[-1]}')
             ready.wait()
             measured.wait()
@@ -258,7 +258,7 @@ def get_additional_memory_estimation_while_running_processes(
             if use_shared_memory:
                 shared_memory = SharedMemory(size=extra_memory_size, create=True)
                 assert shared_memory.buf is not None
-                shared_memory.buf[:] = bytearray([255 for _ in range(extra_memory_size)])
+                shared_memory.buf[:] = b'\xff' * extra_memory_size
                 extra_args = [shared_memory]
             else:
                 extra_args = []

From c8497104b21cdb17b0359db4886766f3e5b916c3 Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Tue, 28 Jul 2026 16:57:07 +0200
Subject: [PATCH 10/11] test: run the memory estimation test in isolation and
 halve its peak memory

---
 tests/unit/_utils/test_system.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index 381b42e717..d58b878dfb 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -48,6 +48,20 @@ def raise_no_such_process(process: psutil.Process) -> object:
     raise psutil.NoSuchProcess(pid=process.pid)
 
 
+def fill_buffer(buffer: memoryview, size: int) -> None:
+    """Fill the first `size` bytes of a shared memory buffer, one chunk at a time.
+
+    Building the payload as a single object would double the peak memory usage of every process that fills a buffer,
+    which is enough for the memory estimation below to be thrown off by page reclaim on a loaded machine.
+    """
+    chunk_size = 1024 * 1024
+    chunk = b'\xff' * chunk_size
+
+    for offset in range(0, size, chunk_size):
+        end = min(offset + chunk_size, size)
+        buffer[offset:end] = chunk[: end - offset]
+
+
 @pytest.fixture(autouse=True)
 def _isolated_module_state(monkeypatch: pytest.MonkeyPatch) -> None:
     """Reset the process-wide state of the module, so that dedup keys and the PSS latch do not leak between tests."""
@@ -202,6 +216,9 @@ def test_get_cpu_info_returns_valid_values() -> None:
     assert 0 <= cpu_info.used_ratio <= 1
 
 
+# The estimation is asserted on absolute memory readings, which hold only as long as nothing else on the machine makes
+# the kernel reclaim the pages allocated below. Running alongside the other test workers is enough to break that.
+@pytest.mark.run_alone
 @pytest.mark.skipif(sys.platform != 'linux', reason='Improved estimation available only on Linux')
 def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
     """Test that memory usage estimation is not overestimating memory usage by counting shared memory multiple times.
@@ -231,7 +248,7 @@ def no_extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barr
         def extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barrier) -> None:
             memory = SharedMemory(size=extra_memory_size, create=True)
             assert memory.buf is not None
-            memory.buf[:] = b'\xff' * extra_memory_size
+            fill_buffer(memory.buf, extra_memory_size)
             print(f'Using the memory... {memory.buf[-1]}')
             ready.wait()
             measured.wait()
@@ -258,7 +275,7 @@ def get_additional_memory_estimation_while_running_processes(
             if use_shared_memory:
                 shared_memory = SharedMemory(size=extra_memory_size, create=True)
                 assert shared_memory.buf is not None
-                shared_memory.buf[:] = b'\xff' * extra_memory_size
+                fill_buffer(shared_memory.buf, extra_memory_size)
                 extra_args = [shared_memory]
             else:
                 extra_args = []

From 86b31c6348857bd61b2158763ae9c8e997714e79 Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Wed, 29 Jul 2026 08:16:55 +0200
Subject: [PATCH 11/11] address mantisus feedback

---
 src/crawlee/_utils/system.py     |  3 ++-
 tests/unit/_utils/test_system.py | 11 +++--------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py
index 79e119b4d1..45d0483679 100644
--- a/src/crawlee/_utils/system.py
+++ b/src/crawlee/_utils/system.py
@@ -54,8 +54,9 @@ def _get_used_memory(process: psutil.Process) -> int:
                 # A system that does not expose `smaps` at all makes psutil alias `memory_full_info` to
                 # `memory_info`, whose result has no `pss` field.
                 memory = process.memory_full_info()
-            except (psutil.NoSuchProcess, psutil.ZombieProcess):
+            except psutil.NoSuchProcess:
                 # A process that is gone is not refusing inspection, so let the RSS read below fail for it as usual.
+                # `ZombieProcess` is a subclass of `NoSuchProcess`, so a zombie lands here too.
                 pass
             except _METRIC_ERRORS:
                 # A restricted environment may deny `/proc/<pid>/smaps`, which is a property of the single process, so
diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py
index d58b878dfb..287de5000c 100644
--- a/tests/unit/_utils/test_system.py
+++ b/tests/unit/_utils/test_system.py
@@ -6,6 +6,7 @@
 from multiprocessing.shared_memory import SharedMemory
 from types import SimpleNamespace
 from typing import TYPE_CHECKING
+from unittest.mock import Mock
 
 import psutil
 import pytest
@@ -138,20 +139,14 @@ def test_get_used_memory_stops_asking_for_pss_once_it_is_known_to_be_missing(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     """A system without PSS is asked for it only once, not on every sample of every process."""
-    call_count = 0
-
-    def memory_full_info(_process: psutil.Process) -> object:
-        nonlocal call_count
-        call_count += 1
-        return SimpleNamespace(rss=1024)
-
+    memory_full_info = Mock(return_value=SimpleNamespace(rss=1024))
     monkeypatch.setattr(psutil.Process, 'memory_full_info', memory_full_info)
     monkeypatch.setattr(psutil.Process, 'memory_info', lambda _process: SimpleNamespace(rss=1024))
 
     for _ in range(3):
         assert system._get_used_memory(psutil.Process()) == 1024
 
-    assert call_count == 1
+    memory_full_info.assert_called_once()
 
 
 @pytest.mark.parametrize(