diff --git a/.agents/skills/scrapingbee-cli/reference/scrape/options.md b/.agents/skills/scrapingbee-cli/reference/scrape/options.md index 37ee290..4ec30ec 100644 --- a/.agents/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.agents/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (15 credits — the API's error message for non-custom-google Google requests wrongly claims 20; 15 is correct), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/.github/skills/scrapingbee-cli/reference/scrape/options.md b/.github/skills/scrapingbee-cli/reference/scrape/options.md index 37ee290..4ec30ec 100644 --- a/.github/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.github/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (15 credits — the API's error message for non-custom-google Google requests wrongly claims 20; 15 is correct), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 948ca10..b0cbbe0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,9 @@ name: CI on: push: branches: [main] + # No branches filter: stacked PRs (a PR based on another PR's branch) must + # run CI too — with [main] here they silently get no checks at all. pull_request: - branches: [main] permissions: contents: read diff --git a/.kiro/skills/scrapingbee-cli/reference/scrape/options.md b/.kiro/skills/scrapingbee-cli/reference/scrape/options.md index 37ee290..4ec30ec 100644 --- a/.kiro/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.kiro/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (15 credits — the API's error message for non-custom-google Google requests wrongly claims 20; 15 is correct), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/.opencode/skills/scrapingbee-cli/reference/scrape/options.md b/.opencode/skills/scrapingbee-cli/reference/scrape/options.md index 37ee290..4ec30ec 100644 --- a/.opencode/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.opencode/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (15 credits — the API's error message for non-custom-google Google requests wrongly claims 20; 15 is correct), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e283a..f239fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Discovery crawls saved zero pages when the site was smaller than `--max-pages`** — Scrapy 2.13 removed the `spider` argument of `ExecutionEngine.crawl()` (deprecated since 2.10), so a discovery-phase crawl (`--return-page-text`, `--extract-rules`, `--ai-query`, screenshot without `--json-response`) whose save queue never reached the `--max-pages` cap (site smaller than the cap, or no cap) raised `TypeError` on every queued save; the error handler logged and continued, so the crawl "succeeded" with nothing saved. Crawls that hit the cap dispatched saves through a different path and were unaffected — which is why this went unnoticed. Save dispatch now uses the current Scrapy call signature, and a contract test fails CI loudly if a future Scrapy changes it again. +- **Crawl discovery prompt suggested a flag that doesn't exist** — the double-credit warning told users to pass `--yes`; the actual flag is `--confirm yes`. - **`-H` headers were silently dropped on POST/PUT** — custom headers are now `Spb-`-prefixed on every method (idempotently), which is the only form the API forwards to the target. Previously the prefix was only added on GET, so POST/PUT headers never reached the target — and a user `Authorization` header could clobber the CLI's own API authentication. Already-prefixed headers are passed through unchanged, so `-H "Spb-X: 1"` no longer double-prefixes. - **Empty subtitles warned about** — `youtube-subtitles` with a `--language`/`--subtitle-origin` that matches nothing returns HTTP 200 with an empty `subtitles` object (not 404) and still charges 5 credits; the CLI now prints a warning instead of silent empty JSON. diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md index 37ee290..4ec30ec 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (15 credits — the API's error message for non-custom-google Google requests wrongly claims 20; 15 is correct), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/src/scrapingbee_cli/commands/crawl.py b/src/scrapingbee_cli/commands/crawl.py index 3d5c37d..b0aba29 100644 --- a/src/scrapingbee_cli/commands/crawl.py +++ b/src/scrapingbee_cli/commands/crawl.py @@ -546,7 +546,7 @@ def crawl_cmd( " an extra HTML-only discovery request — approximately doubling credits.\n\n" " Tip: Use --save-pattern '.*' to crawl with HTML (cheap, finds all links)\n" " and apply your full settings only to pages that match the pattern.\n" - " Pass --yes to skip this prompt in scripts.\n", + " Pass --confirm yes to skip this prompt in scripts.\n", err=True, ) try: diff --git a/src/scrapingbee_cli/crawl.py b/src/scrapingbee_cli/crawl.py index e70a059..8334ef0 100644 --- a/src/scrapingbee_cli/crawl.py +++ b/src/scrapingbee_cli/crawl.py @@ -654,7 +654,12 @@ def _on_spider_idle(self, spider) -> None: self._save_pending += 1 self._save_queue_next += 1 try: - engine.crawl(self._make_save_request(url), spider) + # Scrapy 2.13 removed the ``spider`` argument of + # ``ExecutionEngine.crawl`` — request-only is the sole call + # shape on every Scrapy this spider runs on (its ``start()`` + # entry point requires >= 2.13). test_crawl.py's contract + # test fails loudly if a future Scrapy changes the signature. + engine.crawl(self._make_save_request(url)) except Exception as e: # Log, don't swallow: if engine.crawl()'s signature shifts under a # Scrapy bump, every queued save would silently fail and the user @@ -1139,7 +1144,7 @@ def _on_save_error(self, failure) -> None: self._save_queue_next += 1 self._save_pending += 1 try: - engine.crawl(self._make_save_request(url), self) + engine.crawl(self._make_save_request(url)) except Exception as e: self.logger.warning("Failed to dispatch backfill save for %s: %s", url, e) if self._save_pending > 0: diff --git a/tests/unit/test_crawl.py b/tests/unit/test_crawl.py index b707c5f..f705d89 100644 --- a/tests/unit/test_crawl.py +++ b/tests/unit/test_crawl.py @@ -444,6 +444,29 @@ def test_return_page_markdown_does_not_require_discovery(self): assert _requires_discovery_phase({"return_page_markdown": "true"}) is False +class TestEngineCrawlContract: + """The installed Scrapy's ExecutionEngine.crawl must be request-only. + + Scrapy 2.13 removed the ``spider`` argument; the spider dispatches saves + with ``engine.crawl(request)``. A stale call shape raises TypeError, + which the save-dispatch error handlers swallow — every queued save + silently fails and a discovery crawl saves zero pages (the 1.6.0 + regression this guards against). If a future Scrapy changes the + signature again, this test fails loudly at upgrade time instead. + """ + + def test_installed_scrapy_engine_crawl_is_request_only(self): + import inspect + + from scrapy.core.engine import ExecutionEngine + + params = list(inspect.signature(ExecutionEngine.crawl).parameters) + assert params == ["self", "request"], ( + f"ExecutionEngine.crawl signature changed to {params}; update the " + "engine.crawl() dispatch sites in crawl.py to match" + ) + + class TestExtractHrefsExceptionHandling: """Tests that _extract_hrefs_from_response handles non-HTML gracefully."""