chore(deps): update dependency ruff to ~=0.16.0

pyproject.toml

@@ -137,7 +137,7 @@ dev = [
     "pytest-timeout<3.0.0",
     "pytest-xdist<4.0.0",
     "pytest<10.0.0",
-    "ruff~=0.15.0",
+    "ruff~=0.16.0",
     "setuptools", # setuptools are used by pytest, but not explicitly required
     "ty~=0.0.0",
     "types-beautifulsoup4<5.0.0",
@@ -183,6 +183,7 @@ ignore = [
     "BLE001",   # Do not catch blind exception
     "C901",     # `{name}` is too complex
     "COM812",   # This rule may cause conflicts when used with the formatter
+    "CPY001",   # Missing copyright notice at top of file
     "D100",     # Missing docstring in public module
     "D104",     # Missing docstring in public package
     "D107",     # Missing docstring in `__init__`

src/crawlee/_utils/sitemap.py

@@ -348,13 +348,13 @@ async def _process_raw_source(
 
 
 async def _fetch_and_process_sitemap(
+    *,
     http_client: HttpClient,
     source: SitemapSource,
     depth: int,
     visited_sitemap_urls: set[str],
     sources: list[SitemapSource],
     retries_left: int,
-    *,
     proxy_info: ProxyInfo | None = None,
     timeout: timedelta | None = None,
     emit_nested_sitemaps: bool,
@@ -564,12 +564,12 @@ async def parse_sitemap(
             visited_sitemap_urls.add(source['url'])
 
             async for result in _fetch_and_process_sitemap(
-                http_client,
-                source,
-                depth,
-                visited_sitemap_urls,
-                sources,
-                sitemap_retries,
+                http_client=http_client,
+                source=source,
+                depth=depth,
+                visited_sitemap_urls=visited_sitemap_urls,
+                sources=sources,
+                retries_left=sitemap_retries,
                 emit_nested_sitemaps=emit_nested_sitemaps,
                 enqueue_strategy=enqueue_strategy,
                 proxy_info=proxy_info,

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py

@@ -233,7 +233,7 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon
         self._static_parser = static_parser
 
     @classmethod
-    def with_beautifulsoup_static_parser(
+    def with_beautifulsoup_static_parser(  # noqa: PLR0917
         cls,
         rendering_type_predictor: RenderingTypePredictor | None = None,
         result_checker: Callable[[RequestHandlerRunResult], bool] | None = None,
@@ -396,7 +396,7 @@ async def _run_request_handler(self, context: BasicCrawlingContext) -> None:
                     self._context_result_map[context] = static_run.result
                     return
                 if static_run.exception:
-                    context.log.exception(
+                    context.log.error(
                         msg=f'Static crawler: failed for {context.request.url}', exc_info=static_run.exception
                     )
                 else:

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -1152,7 +1152,7 @@ async def _handle_request_retries(
         request = context.request
 
         if self._abort_on_error:
-            self._logger.exception('Aborting crawler run due to error (abort_on_error=True)', exc_info=error)
+            self._logger.error('Aborting crawler run due to error (abort_on_error=True)', exc_info=error)
             self._failed = True
 
         if self._should_retry_request(context, error):

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -123,7 +123,7 @@ def __init__(
         browser_launch_options: Mapping[str, Any] | None = None,
         browser_new_context_options: Mapping[str, Any] | None = None,
         goto_options: GotoOptions | None = None,
-        fingerprint_generator: FingerprintGenerator | None | Literal['default'] = 'default',
+        fingerprint_generator: FingerprintGenerator | Literal['default'] | None = 'default',
         headless: bool | None = None,
         use_incognito_pages: bool | None = None,
         navigation_timeout: timedelta | None = None,

src/crawlee/events/_event_manager.py

@@ -166,7 +166,7 @@ def on(self, *, event: Event, listener: EventListener[Any]) -> None:
         """
         signature = inspect.signature(listener)
 
-        @wraps(cast('Callable[..., None | Awaitable[None]]', listener))
+        @wraps(cast('Callable[..., Awaitable[None] | None]', listener))
         async def listener_wrapper(event_data: EventData) -> None:
             try:
                 bound_args = signature.bind(event_data)
@@ -262,7 +262,7 @@ async def wait_for_listeners() -> None:
             results = await asyncio.gather(*self._listener_tasks, return_exceptions=True)
             for result in results:
                 if isinstance(result, Exception):
-                    logger.exception('Event listener raised an exception.', exc_info=result)
+                    logger.error('Event listener raised an exception.', exc_info=result)
 
         tasks = [asyncio.create_task(wait_for_listeners(), name=f'Task-{wait_for_listeners.__name__}')]
 

src/crawlee/events/_types.py

@@ -112,11 +112,11 @@ class EventCrawlerStatusData(BaseModel):
 EventListener = (
     Callable[
         [TEvent],
-        None | Coroutine[Any, Any, None],
+        Coroutine[Any, Any, None] | None,
     ]
     | Callable[
         [],
-        None | Coroutine[Any, Any, None],
+        Coroutine[Any, Any, None] | None,
     ]
 )
 """An event listener function - it can be both sync and async and may accept zero or one argument."""

src/crawlee/http_clients/_httpx.py

@@ -264,6 +264,7 @@ async def stream(
 
     def _build_request(
         self,
+        *,
         client: httpx.AsyncClient,
         url: str,
         method: HttpMethod,

src/crawlee/proxy_configuration.py

@@ -265,4 +265,4 @@ def __call__(
         self,
         session_id: str | None = None,
         request: Request | None = None,
-    ) -> str | None | Awaitable[str | None]: ...
+    ) -> str | Awaitable[str | None] | None: ...

src/crawlee/storage_clients/_redis/_request_queue_client.py

@@ -117,7 +117,7 @@ def __init__(
         self._reclaim_stale_script: AsyncScript | None = None
         self._add_requests_script: AsyncScript | None = None
 
-        self._next_reclaim_stale: None | datetime = None
+        self._next_reclaim_stale: datetime | None = None
 
     @property
     def _added_filter_key(self) -> str:

src/crawlee/storage_clients/_sql/_storage_client.py

@@ -73,7 +73,7 @@ def __init__(
         self._connection_string = connection_string
         self._engine = engine
         self._initialized = False
-        self.session_maker: None | async_sessionmaker[AsyncSession] = None
+        self.session_maker: async_sessionmaker[AsyncSession] | None = None
 
         self._listeners_registered = False
         self._dialect_name: str | None = None

src/crawlee/storages/_dataset.py

@@ -320,7 +320,7 @@ async def export_to(
         **kwargs: Unpack[ExportDataCsvKwargs],
     ) -> None: ...
 
-    async def export_to(
+    async def export_to(  # noqa: PLR0917
         self,
         key: str,
         content_type: Literal['json', 'csv'] = 'json',

tests/unit/_utils/test_system.py

@@ -76,7 +76,7 @@ def get_additional_memory_estimation_while_running_processes(
             processes = []
             ready = ctx.Barrier(parties=count + 1)
             measured = ctx.Barrier(parties=count + 1)
-            shared_memory: None | SharedMemory = None
+            shared_memory: SharedMemory | None = None
             memory_before = get_memory_info().current_size
 
             if use_shared_memory:

tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py

@@ -85,7 +85,7 @@ class _SimpleRenderingTypePredictor(RenderingTypePredictor):
     def __init__(
         self,
         rendering_types: Iterator[RenderingType] | None = None,
-        detection_probability_recommendation: None | Iterator[float] = None,
+        detection_probability_recommendation: Iterator[float] | None = None,
     ) -> None:
         super().__init__()
 

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -373,7 +373,7 @@ async def failed_request_handler(context: BasicCrawlingContext, error: Exception
         pytest.param('POST', 'post', b'Hello, world!', id='post send_request'),
     ],
 )
-async def test_send_request_works(server_url: URL, method: HttpMethod, path: str, payload: None | bytes) -> None:
+async def test_send_request_works(server_url: URL, method: HttpMethod, path: str, payload: bytes | None) -> None:
     response_data: dict[str, Any] = {}
 
     crawler = BasicCrawler(max_request_retries=3)
@@ -1995,7 +1995,7 @@ async def request_handler(context: BasicCrawlingContext) -> None:
 @dataclass
 class _CrawlerInput:
     requests: list[str]
-    id: None | int = None
+    id: int | None = None
 
 
 def _process_run_crawlers(crawler_inputs: list[_CrawlerInput], storage_dir: str) -> list[StatisticsState]:

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -235,7 +235,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
     ],
 )
 async def test_chromium_headless_headers(
-    header_network: dict, fingerprint_generator: None | FingerprintGenerator | Literal['default'], server_url: URL
+    header_network: dict, fingerprint_generator: FingerprintGenerator | Literal['default'] | None, server_url: URL
 ) -> None:
     browser_type: BrowserType = 'chromium'
     crawler = PlaywrightCrawler(headless=True, browser_type=browser_type, fingerprint_generator=fingerprint_generator)