diff --git a/README.md b/README.md index 41024518..bcb30466 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,12 @@ sensor: feed_url: https://www.nu.nl/rss/Algemeen local_time: true show_topn: 1 + + # Some servers block or stall unknown clients; send a browser User-Agent instead + - platform: feedparser + name: Picky Server Feed + feed_url: https://example.com/rss + user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36' ``` If you wish the integration to look for enclosures in the feed entries, add `image` to `inclusions` list. Do not use `enclosure`. @@ -76,6 +82,7 @@ key | description **inclusions (Optional)** | List of fields to include from populating the list **exclusions (Optional)** | List of fields to exclude from populating the list **scan_interval (Optional)** | Update interval in hours +**user_agent (Optional)** | User-Agent header sent when fetching the feed **Default** `Home Assistant Feed-parser Integration ` *** diff --git a/custom_components/feedparser/sensor.py b/custom_components/feedparser/sensor.py index 4477bbee..9cf2a05f 100644 --- a/custom_components/feedparser/sensor.py +++ b/custom_components/feedparser/sensor.py @@ -36,6 +36,7 @@ CONF_EXCLUSIONS = "exclusions" CONF_SHOW_TOPN = "show_topn" CONF_REMOVE_SUMMARY_IMG = "remove_summary_image" +CONF_USER_AGENT = "user_agent" DEFAULT_DATE_FORMAT = "%a, %b %d %I:%M %p" DEFAULT_SCAN_INTERVAL = timedelta(hours=1) @@ -55,6 +56,7 @@ vol.Optional(CONF_INCLUSIONS, default=[]): vol.All(cv.ensure_list, [cv.string]), vol.Optional(CONF_EXCLUSIONS, default=[]): vol.All(cv.ensure_list, [cv.string]), vol.Optional(CONF_SCAN_INTERVAL, default=DEFAULT_SCAN_INTERVAL): cv.time_period, + vol.Optional(CONF_USER_AGENT, default=USER_AGENT): cv.string, }, ) @@ -80,6 +82,7 @@ async def async_setup_platform( exclusions=config[CONF_EXCLUSIONS], scan_interval=config[CONF_SCAN_INTERVAL], local_time=config[CONF_LOCAL_TIME], + user_agent=config[CONF_USER_AGENT], ), ], update_before_add=True, @@ -104,6 +107,7 @@ def __init__( inclusions: list[str | None], scan_interval: timedelta, local_time: bool, + user_agent: str = USER_AGENT, ) -> None: """Initialize the Feedparser sensor.""" self._feed = feed @@ -116,6 +120,7 @@ def __init__( self._exclusions = exclusions self._scan_interval = scan_interval self._local_time = local_time + self._user_agent = user_agent self._entries: list[dict[str, str]] = [] self._attr_extra_state_attributes = {"entries": self._entries} self._attr_attribution = "Data retrieved using RSS feedparser" @@ -129,7 +134,8 @@ def __repr__(self: FeedParserSensor) -> str: f"remove_summary_image={self._remove_summary_image}, " f"inclusions={self._inclusions}, " f"exclusions={self._exclusions}, scan_interval={self._scan_interval}, " - f'local_time={self._local_time}, date_format="{self._date_format}")' + f'local_time={self._local_time}, date_format="{self._date_format}", ' + f'user_agent="{self._user_agent}")' ) def update(self: FeedParserSensor) -> None: @@ -137,7 +143,7 @@ def update(self: FeedParserSensor) -> None: _LOGGER.debug("Feed %s: Polling feed data from %s", self.name, self._feed) s: requests.Session = requests.Session() s.mount("file://", FileAdapter()) - s.headers.update({"User-Agent": USER_AGENT}) + s.headers.update({"User-Agent": self._user_agent}) res: requests.Response = s.get(self._feed) res.raise_for_status() parsed_feed: FeedParserDict = feedparser.parse(res.text) diff --git a/pyproject.toml b/pyproject.toml index 97409d97..de4c56b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,7 +144,7 @@ target-version = "py311" "tests/**" = ["S101"] [tool.ruff.pylint] -max-args = 10 +max-args = 11 [[tool.mypy.overrides]] module = "feedparser.*" diff --git a/tests/test_sensors.py b/tests/test_sensors.py index 07fad38f..d90f7580 100644 --- a/tests/test_sensors.py +++ b/tests/test_sensors.py @@ -3,22 +3,27 @@ import re from contextlib import nullcontext, suppress from datetime import UTC, datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import feedparser import pytest +import requests from constants import DATE_FORMAT, URLS_HEADERS_REQUIRED from feedsource import FeedSource from custom_components.feedparser.sensor import ( + CONF_USER_AGENT, DEFAULT_SCAN_INTERVAL, DEFAULT_THUMBNAIL, IMAGE_REGEX, + PLATFORM_SCHEMA, + USER_AGENT, FeedParserSensor, ) if TYPE_CHECKING: import time + from collections.abc import Callable def test_simple(feed_sensor: FeedParserSensor) -> None: @@ -223,3 +228,59 @@ def test_image_not_in_entries(feed: FeedSource) -> None: assert feed_sensor.feed_entries # assert that the sensor does not include the image in its feed entries assert all("image" not in e for e in feed_sensor.feed_entries) + + +@pytest.fixture() +def sent_user_agents(monkeypatch: pytest.MonkeyPatch) -> list[str | bytes | None]: + """Record the User-Agent header of every request the sensor sends.""" + sent: list[str | bytes | None] = [] + original_get: Callable[..., requests.Response] = requests.Session.get + + def spy_get( + self: requests.Session, + url: str, + *args: Any, # noqa: ANN401 + **kwargs: Any, # noqa: ANN401 + ) -> requests.Response: + response = original_get(self, url, *args, **kwargs) + sent.append(response.request.headers.get("User-Agent")) + return response + + monkeypatch.setattr(requests.Session, "get", spy_get) + return sent + + +def test_default_user_agent( + feed: FeedSource, + sent_user_agents: list[str | bytes | None], +) -> None: + """Test that the integration's User-Agent is sent when none is configured.""" + feed_sensor = FeedParserSensor(**feed.sensor_config_local_feed) + feed_sensor.update() + assert sent_user_agents == [USER_AGENT] + + +def test_custom_user_agent( + feed: FeedSource, + sent_user_agents: list[str | bytes | None], +) -> None: + """Test that a configured User-Agent replaces the default one.""" + custom_user_agent = "Mozilla/5.0 (compatible; FeedTest/1.0)" + feed_sensor = FeedParserSensor( + **feed.sensor_config_local_feed | {"user_agent": custom_user_agent}, + ) + feed_sensor.update() + assert sent_user_agents == [custom_user_agent] + + +def test_user_agent_config_schema() -> None: + """Test that user_agent is optional and defaults to the integration's agent.""" + base_config = { + "platform": "feedparser", + "name": "Test Feed", + "feed_url": "https://example.com/feed.xml", + } + assert PLATFORM_SCHEMA(base_config)[CONF_USER_AGENT] == USER_AGENT + + custom_config = base_config | {CONF_USER_AGENT: "Custom/1.0"} + assert PLATFORM_SCHEMA(custom_config)[CONF_USER_AGENT] == "Custom/1.0"