From 48127be480cba8d8cadb4011b7beabd2574e86d0 Mon Sep 17 00:00:00 2001 From: Nirupam Bhowmick <48842933+heydryft@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:13:12 +0000 Subject: [PATCH] feat(models): add Runcrate as an OpenAI-compatible provider Adds a Runcrate ChatOpenAI wrapper pinned to https://api.runcrate.ai/v1 and wires it into AbstractGraph, so `runcrate//` (for example runcrate/deepseek-ai/DeepSeek-V4.1-Flash) resolves to Runcrate with the vendor-prefixed model id kept intact. Adds context windows for six models to models_tokens, a SmartScraper example and tests. Co-Authored-By: Claude Opus 5.5 --- .../runcrate/smart_scraper_runcrate.py | 47 +++++++++++++++++++ scrapegraphai/graphs/abstract_graph.py | 7 ++- scrapegraphai/helpers/models_tokens.py | 8 ++++ scrapegraphai/models/__init__.py | 3 +- scrapegraphai/models/runcrate.py | 23 +++++++++ tests/test_runcrate_model.py | 31 ++++++++++++ 6 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 examples/smart_scraper_graph/runcrate/smart_scraper_runcrate.py create mode 100644 scrapegraphai/models/runcrate.py create mode 100644 tests/test_runcrate_model.py diff --git a/examples/smart_scraper_graph/runcrate/smart_scraper_runcrate.py b/examples/smart_scraper_graph/runcrate/smart_scraper_runcrate.py new file mode 100644 index 000000000..e36a4aa6f --- /dev/null +++ b/examples/smart_scraper_graph/runcrate/smart_scraper_runcrate.py @@ -0,0 +1,47 @@ +""" +Basic example of scraping pipeline using SmartScraper with Runcrate +""" + +import json +import os + +from dotenv import load_dotenv + +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +# ************************************************ +# Define the configuration for the graph +# ************************************************ + + +graph_config = { + "llm": { + "api_key": os.getenv("RUNCRATE_API_KEY"), + "model": "runcrate/deepseek-ai/DeepSeek-V4.1-Flash", + }, + "verbose": True, + "headless": True, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="Extract me the first article", + source="https://www.wired.com", + config=graph_config, +) + +result = smart_scraper_graph.run() +print(json.dumps(result, indent=4)) + +# ************************************************ +# Get graph execution info +# ************************************************ + +graph_exec_info = smart_scraper_graph.get_execution_info() +print(prettify_exec_info(graph_exec_info)) diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index 21f90460c..89d1baaf2 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -13,7 +13,7 @@ from pydantic import BaseModel from ..helpers import models_tokens -from ..models import XAI, CLoD, DeepSeek, MiniMax, Nvidia, OneApi +from ..models import XAI, CLoD, DeepSeek, MiniMax, Nvidia, OneApi, Runcrate from ..utils.logging import get_logger, set_verbosity_info, set_verbosity_warning logger = get_logger(__name__) @@ -175,6 +175,7 @@ def _create_llm(self, llm_config: dict) -> object: "togetherai", "xai", "minimax", + "runcrate", } if "/" in llm_params["model"]: @@ -243,6 +244,7 @@ def _create_llm(self, llm_config: dict) -> object: "clod", "xai", "minimax", + "runcrate", }: if llm_params["model_provider"] == "bedrock": llm_params["model_kwargs"] = { @@ -263,6 +265,9 @@ def _create_llm(self, llm_config: dict) -> object: if model_provider == "minimax": return MiniMax(**llm_params) + if model_provider == "runcrate": + return Runcrate(**llm_params) + if model_provider == "ernie": from langchain_community.chat_models import ErnieBotChat diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index 5a81ecaa7..a574e7a2d 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -352,6 +352,14 @@ "gpt-5.4-mini": 400000, "gpt-5.4": 1000000, }, + "runcrate": { + "deepseek-ai/DeepSeek-V4.1-Flash": 1048576, + "zai-org/GLM-5.3": 1048576, + "zai-org/GLM-5.3-Flash": 1048576, + "Qwen/Qwen3.8-Flash": 1000000, + "openai/gpt-oss-120b": 131072, + "moonshotai/Kimi-K2.6": 262144, + }, "deepseek": { "deepseek-chat": 128000, "deepseek-coder": 128000, diff --git a/scrapegraphai/models/__init__.py b/scrapegraphai/models/__init__.py index db01cd2e4..9e4b22eb5 100644 --- a/scrapegraphai/models/__init__.py +++ b/scrapegraphai/models/__init__.py @@ -11,6 +11,7 @@ from .oneapi import OneApi from .openai_itt import OpenAIImageToText from .openai_tts import OpenAITextToSpeech +from .runcrate import Runcrate from .xai import XAI -__all__ = ["AtlasCloud", "CheaperInference", "DeepSeek", "MiniMax", "OneApi", "OpenAIImageToText", "OpenAITextToSpeech", "CLoD", "XAI", "Nvidia"] +__all__ = ["AtlasCloud", "CheaperInference", "DeepSeek", "MiniMax", "OneApi", "OpenAIImageToText", "OpenAITextToSpeech", "Runcrate", "CLoD", "XAI", "Nvidia"] diff --git a/scrapegraphai/models/runcrate.py b/scrapegraphai/models/runcrate.py new file mode 100644 index 000000000..2e3939c6f --- /dev/null +++ b/scrapegraphai/models/runcrate.py @@ -0,0 +1,23 @@ +""" +Runcrate Module +""" + +from langchain_openai import ChatOpenAI + + +class Runcrate(ChatOpenAI): + """ + A wrapper for ChatOpenAI configured for Runcrate's OpenAI-compatible + inference API. Model ids keep their vendor prefix, for example + ``deepseek-ai/DeepSeek-V4.1-Flash``. + + Args: + llm_config (dict): Configuration parameters for the language model. + """ + + def __init__(self, **llm_config): + if "api_key" in llm_config: + llm_config["openai_api_key"] = llm_config.pop("api_key") + llm_config["openai_api_base"] = "https://api.runcrate.ai/v1" + + super().__init__(**llm_config) diff --git a/tests/test_runcrate_model.py b/tests/test_runcrate_model.py new file mode 100644 index 000000000..b58baa58f --- /dev/null +++ b/tests/test_runcrate_model.py @@ -0,0 +1,31 @@ +"""Tests for the Runcrate model configuration.""" + +from unittest.mock import Mock, patch + +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.models import Runcrate + + +def test_runcrate_model_sets_openai_compatible_base_url(): + """Runcrate should map api_key and pin its base URL.""" + model = Runcrate(api_key="test-key", model="deepseek-ai/DeepSeek-V4.1-Flash") + + assert str(model.openai_api_base).rstrip("/") == "https://api.runcrate.ai/v1" + assert model.openai_api_key.get_secret_value() == "test-key" + + +def test_runcrate_provider_keeps_vendor_prefixed_model_id(): + """`runcrate//` resolves to Runcrate with the full model id.""" + config = { + "llm": { + "model": "runcrate/deepseek-ai/DeepSeek-V4.1-Flash", + "api_key": "test-key", + } + } + with patch.object(SmartScraperGraph, "_create_graph", return_value=Mock(nodes=[])): + graph = SmartScraperGraph("prompt", "https://example.com", config) + + assert isinstance(graph.llm_model, Runcrate) + assert graph.llm_model.model_name == "deepseek-ai/DeepSeek-V4.1-Flash" + assert graph.model_token == 1048576 + assert not getattr(graph, "model_tokens_defaulted", False)