From 4c1d425ab7eaa19f31e1aaea166bac4251a6d76d Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Aug 2026 12:27:25 +0200 Subject: [PATCH] fix: add User-Agent value for data collection HTTP requests Signed-off-by: tdruez --- component_catalog/tests/test_models.py | 15 +++++++++++++++ dejacode_toolkit/download.py | 7 ++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index 195062b5..05b1f1f3 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -2473,6 +2473,21 @@ def test_collect_package_data(self, mock_get): } self.assertEqual(expected_data, collect_package_data(download_url)) + @mock.patch("requests.get") + def test_collect_package_data_user_agent(self, mock_get): + mock_get.return_value = mock.Mock( + content=b"\x00", + headers={"content-length": 1}, + status_code=200, + url="http://domain.com/a.zip", + ) + + collect_package_data("http://domain.com/a.zip") + self.assertEqual( + download.USER_AGENT, + mock_get.call_args.kwargs["headers"]["User-Agent"], + ) + def test_package_create_save_set_usage_policy_from_license(self): from policy.models import AssociatedPolicy from policy.models import UsagePolicy diff --git a/dejacode_toolkit/download.py b/dejacode_toolkit/download.py index d19c0217..ef4b94d5 100644 --- a/dejacode_toolkit/download.py +++ b/dejacode_toolkit/download.py @@ -26,14 +26,19 @@ CONTENT_MAX_LENGTH = 536870912 # 512 MB DEFAULT_TIMEOUT = 5 +# Some sites (e.g. crates.io) reject requests without an identifying User-Agent. +USER_AGENT = "DejaCode (https://github.com/aboutcode-org/dejacode)" + class DataCollectionException(Exception): pass def collect_package_data(url): + headers = {"User-Agent": USER_AGENT} + try: - response = requests.get(url, timeout=DEFAULT_TIMEOUT, stream=True) + response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT, stream=True) except (TimeoutError, requests.RequestException) as e: raise DataCollectionException(e)