feat(sensor): make User-Agent configurable per feed

README.md

@@ -56,6 +56,12 @@ sensor:
     feed_url: https://www.nu.nl/rss/Algemeen
     local_time: true
     show_topn: 1
+
+  # Some servers block or stall unknown clients; send a browser User-Agent instead
+  - platform: feedparser
+    name: Picky Server Feed
+    feed_url: https://example.com/rss
+    user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36'
 ```
 
 If you wish the integration to look for enclosures in the feed entries, add `image` to `inclusions` list. Do not use `enclosure`.
@@ -76,6 +82,7 @@ key | description
 **inclusions (Optional)** | List of fields to include from populating the list
 **exclusions (Optional)** | List of fields to exclude from populating the list
 **scan_interval (Optional)** | Update interval in hours
+**user_agent (Optional)** | User-Agent header sent when fetching the feed **Default** `Home Assistant Feed-parser Integration <version>`
 
 ***
 

custom_components/feedparser/sensor.py

@@ -36,6 +36,7 @@
 CONF_EXCLUSIONS = "exclusions"
 CONF_SHOW_TOPN = "show_topn"
 CONF_REMOVE_SUMMARY_IMG = "remove_summary_image"
+CONF_USER_AGENT = "user_agent"
 
 DEFAULT_DATE_FORMAT = "%a, %b %d %I:%M %p"
 DEFAULT_SCAN_INTERVAL = timedelta(hours=1)
@@ -55,6 +56,7 @@
         vol.Optional(CONF_INCLUSIONS, default=[]): vol.All(cv.ensure_list, [cv.string]),
         vol.Optional(CONF_EXCLUSIONS, default=[]): vol.All(cv.ensure_list, [cv.string]),
         vol.Optional(CONF_SCAN_INTERVAL, default=DEFAULT_SCAN_INTERVAL): cv.time_period,
+        vol.Optional(CONF_USER_AGENT, default=USER_AGENT): cv.string,
     },
 )
 
@@ -80,6 +82,7 @@ async def async_setup_platform(
                 exclusions=config[CONF_EXCLUSIONS],
                 scan_interval=config[CONF_SCAN_INTERVAL],
                 local_time=config[CONF_LOCAL_TIME],
+                user_agent=config[CONF_USER_AGENT],
             ),
         ],
         update_before_add=True,
@@ -104,6 +107,7 @@ def __init__(
         inclusions: list[str | None],
         scan_interval: timedelta,
         local_time: bool,
+        user_agent: str = USER_AGENT,
     ) -> None:
         """Initialize the Feedparser sensor."""
         self._feed = feed
@@ -116,6 +120,7 @@ def __init__(
         self._exclusions = exclusions
         self._scan_interval = scan_interval
         self._local_time = local_time
+        self._user_agent = user_agent
         self._entries: list[dict[str, str]] = []
         self._attr_extra_state_attributes = {"entries": self._entries}
         self._attr_attribution = "Data retrieved using RSS feedparser"
@@ -129,15 +134,16 @@ def __repr__(self: FeedParserSensor) -> str:
             f"remove_summary_image={self._remove_summary_image}, "
             f"inclusions={self._inclusions}, "
             f"exclusions={self._exclusions}, scan_interval={self._scan_interval}, "
-            f'local_time={self._local_time}, date_format="{self._date_format}")'
+            f'local_time={self._local_time}, date_format="{self._date_format}", '
+            f'user_agent="{self._user_agent}")'
         )
 
     def update(self: FeedParserSensor) -> None:
         """Parse the feed and update the state of the sensor."""
         _LOGGER.debug("Feed %s: Polling feed data from %s", self.name, self._feed)
         s: requests.Session = requests.Session()
         s.mount("file://", FileAdapter())
-        s.headers.update({"User-Agent": USER_AGENT})
+        s.headers.update({"User-Agent": self._user_agent})
         res: requests.Response = s.get(self._feed)
         res.raise_for_status()
         parsed_feed: FeedParserDict = feedparser.parse(res.text)

pyproject.toml

@@ -144,7 +144,7 @@ target-version = "py311"
 "tests/**" = ["S101"]
 
 [tool.ruff.pylint]
-max-args = 10
+max-args = 11
 
 [[tool.mypy.overrides]]
 module = "feedparser.*"

tests/test_sensors.py

@@ -3,22 +3,27 @@
 import re
 from contextlib import nullcontext, suppress
 from datetime import UTC, datetime
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import feedparser
 import pytest
+import requests
 from constants import DATE_FORMAT, URLS_HEADERS_REQUIRED
 from feedsource import FeedSource
 
 from custom_components.feedparser.sensor import (
+    CONF_USER_AGENT,
     DEFAULT_SCAN_INTERVAL,
     DEFAULT_THUMBNAIL,
     IMAGE_REGEX,
+    PLATFORM_SCHEMA,
+    USER_AGENT,
     FeedParserSensor,
 )
 
 if TYPE_CHECKING:
     import time
+    from collections.abc import Callable
 
 
 def test_simple(feed_sensor: FeedParserSensor) -> None:
@@ -223,3 +228,59 @@ def test_image_not_in_entries(feed: FeedSource) -> None:
     assert feed_sensor.feed_entries
     # assert that the sensor does not include the image in its feed entries
     assert all("image" not in e for e in feed_sensor.feed_entries)
+
+
+@pytest.fixture()
+def sent_user_agents(monkeypatch: pytest.MonkeyPatch) -> list[str | bytes | None]:
+    """Record the User-Agent header of every request the sensor sends."""
+    sent: list[str | bytes | None] = []
+    original_get: Callable[..., requests.Response] = requests.Session.get
+
+    def spy_get(
+        self: requests.Session,
+        url: str,
+        *args: Any,  # noqa: ANN401
+        **kwargs: Any,  # noqa: ANN401
+    ) -> requests.Response:
+        response = original_get(self, url, *args, **kwargs)
+        sent.append(response.request.headers.get("User-Agent"))
+        return response
+
+    monkeypatch.setattr(requests.Session, "get", spy_get)
+    return sent
+
+
+def test_default_user_agent(
+    feed: FeedSource,
+    sent_user_agents: list[str | bytes | None],
+) -> None:
+    """Test that the integration's User-Agent is sent when none is configured."""
+    feed_sensor = FeedParserSensor(**feed.sensor_config_local_feed)
+    feed_sensor.update()
+    assert sent_user_agents == [USER_AGENT]
+
+
+def test_custom_user_agent(
+    feed: FeedSource,
+    sent_user_agents: list[str | bytes | None],
+) -> None:
+    """Test that a configured User-Agent replaces the default one."""
+    custom_user_agent = "Mozilla/5.0 (compatible; FeedTest/1.0)"
+    feed_sensor = FeedParserSensor(
+        **feed.sensor_config_local_feed | {"user_agent": custom_user_agent},
+    )
+    feed_sensor.update()
+    assert sent_user_agents == [custom_user_agent]
+
+
+def test_user_agent_config_schema() -> None:
+    """Test that user_agent is optional and defaults to the integration's agent."""
+    base_config = {
+        "platform": "feedparser",
+        "name": "Test Feed",
+        "feed_url": "https://example.com/feed.xml",
+    }
+    assert PLATFORM_SCHEMA(base_config)[CONF_USER_AGENT] == USER_AGENT
+
+    custom_config = base_config | {CONF_USER_AGENT: "Custom/1.0"}
+    assert PLATFORM_SCHEMA(custom_config)[CONF_USER_AGENT] == "Custom/1.0"