From 49ec401ef86a3e02ee2513cd88965cb1dbc23fbc Mon Sep 17 00:00:00 2001 From: litescraper <322634161+litescraper@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:55:05 -0500 Subject: [PATCH] feat: support configurable Serper base URL --- README.md | 3 +++ scrapegraphai/utils/research_web.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 386c41b10..97fc8f5a5 100644 --- a/README.md +++ b/README.md @@ -258,3 +258,6 @@ ScrapeGraphAI is licensed under the MIT License. See the [LICENSE](https://githu Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com) [Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d) + + +Support any Serper.dev-compatible endpoint (like litescrape.com, serpbase.dev, serpensapi.org, and others). Set `SERPER_BASE_URL` to the provider base URL (without `/search`) and `SERPER_API_KEY` to a key issued by that provider. The default base URL is `https://google.serper.dev`. diff --git a/scrapegraphai/utils/research_web.py b/scrapegraphai/utils/research_web.py index 7efb497cb..5f3c4c669 100644 --- a/scrapegraphai/utils/research_web.py +++ b/scrapegraphai/utils/research_web.py @@ -3,6 +3,7 @@ error handling, validation, and security features. """ +import os import random import re import time @@ -394,7 +395,8 @@ def _search_serper( try: response = requests.post( - "https://google.serper.dev/search", + (os.getenv("SERPER_BASE_URL") or "https://google.serper.dev").rstrip("/") + + "/search", json=data, headers=headers, timeout=timeout,