Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions examples/smart_scraper_graph/runcrate/smart_scraper_runcrate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""
Basic example of scraping pipeline using SmartScraper with Runcrate
"""

import json
import os

from dotenv import load_dotenv

from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.utils import prettify_exec_info

load_dotenv()

# ************************************************
# Define the configuration for the graph
# ************************************************


graph_config = {
"llm": {
"api_key": os.getenv("RUNCRATE_API_KEY"),
"model": "runcrate/deepseek-ai/DeepSeek-V4.1-Flash",
},
"verbose": True,
"headless": True,
}

# ************************************************
# Create the SmartScraperGraph instance and run it
# ************************************************

smart_scraper_graph = SmartScraperGraph(
prompt="Extract me the first article",
source="https://www.wired.com",
config=graph_config,
)

result = smart_scraper_graph.run()
print(json.dumps(result, indent=4))

# ************************************************
# Get graph execution info
# ************************************************

graph_exec_info = smart_scraper_graph.get_execution_info()
print(prettify_exec_info(graph_exec_info))
7 changes: 6 additions & 1 deletion scrapegraphai/graphs/abstract_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pydantic import BaseModel

from ..helpers import models_tokens
from ..models import XAI, CLoD, DeepSeek, MiniMax, Nvidia, OneApi
from ..models import XAI, CLoD, DeepSeek, MiniMax, Nvidia, OneApi, Runcrate
from ..utils.logging import get_logger, set_verbosity_info, set_verbosity_warning

logger = get_logger(__name__)
Expand Down Expand Up @@ -175,6 +175,7 @@ def _create_llm(self, llm_config: dict) -> object:
"togetherai",
"xai",
"minimax",
"runcrate",
}

if "/" in llm_params["model"]:
Expand Down Expand Up @@ -243,6 +244,7 @@ def _create_llm(self, llm_config: dict) -> object:
"clod",
"xai",
"minimax",
"runcrate",
}:
if llm_params["model_provider"] == "bedrock":
llm_params["model_kwargs"] = {
Expand All @@ -263,6 +265,9 @@ def _create_llm(self, llm_config: dict) -> object:
if model_provider == "minimax":
return MiniMax(**llm_params)

if model_provider == "runcrate":
return Runcrate(**llm_params)

if model_provider == "ernie":
from langchain_community.chat_models import ErnieBotChat

Expand Down
8 changes: 8 additions & 0 deletions scrapegraphai/helpers/models_tokens.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,14 @@
"gpt-5.4-mini": 400000,
"gpt-5.4": 1000000,
},
"runcrate": {
"deepseek-ai/DeepSeek-V4.1-Flash": 1048576,
"zai-org/GLM-5.3": 1048576,
"zai-org/GLM-5.3-Flash": 1048576,
"Qwen/Qwen3.8-Flash": 1000000,
"openai/gpt-oss-120b": 131072,
"moonshotai/Kimi-K2.6": 262144,
},
"deepseek": {
"deepseek-chat": 128000,
"deepseek-coder": 128000,
Expand Down
3 changes: 2 additions & 1 deletion scrapegraphai/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from .oneapi import OneApi
from .openai_itt import OpenAIImageToText
from .openai_tts import OpenAITextToSpeech
from .runcrate import Runcrate
from .xai import XAI

__all__ = ["AtlasCloud", "CheaperInference", "DeepSeek", "MiniMax", "OneApi", "OpenAIImageToText", "OpenAITextToSpeech", "CLoD", "XAI", "Nvidia"]
__all__ = ["AtlasCloud", "CheaperInference", "DeepSeek", "MiniMax", "OneApi", "OpenAIImageToText", "OpenAITextToSpeech", "Runcrate", "CLoD", "XAI", "Nvidia"]
23 changes: 23 additions & 0 deletions scrapegraphai/models/runcrate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""
Runcrate Module
"""

from langchain_openai import ChatOpenAI


class Runcrate(ChatOpenAI):
"""
A wrapper for ChatOpenAI configured for Runcrate's OpenAI-compatible
inference API. Model ids keep their vendor prefix, for example
``deepseek-ai/DeepSeek-V4.1-Flash``.

Args:
llm_config (dict): Configuration parameters for the language model.
"""

def __init__(self, **llm_config):
if "api_key" in llm_config:
llm_config["openai_api_key"] = llm_config.pop("api_key")
llm_config["openai_api_base"] = "https://api.runcrate.ai/v1"

super().__init__(**llm_config)
31 changes: 31 additions & 0 deletions tests/test_runcrate_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
"""Tests for the Runcrate model configuration."""

from unittest.mock import Mock, patch

from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.models import Runcrate


def test_runcrate_model_sets_openai_compatible_base_url():
"""Runcrate should map api_key and pin its base URL."""
model = Runcrate(api_key="test-key", model="deepseek-ai/DeepSeek-V4.1-Flash")

assert str(model.openai_api_base).rstrip("/") == "https://api.runcrate.ai/v1"
assert model.openai_api_key.get_secret_value() == "test-key"


def test_runcrate_provider_keeps_vendor_prefixed_model_id():
"""`runcrate/<vendor>/<model>` resolves to Runcrate with the full model id."""
config = {
"llm": {
"model": "runcrate/deepseek-ai/DeepSeek-V4.1-Flash",
"api_key": "test-key",
}
}
with patch.object(SmartScraperGraph, "_create_graph", return_value=Mock(nodes=[])):
graph = SmartScraperGraph("prompt", "https://example.com", config)

assert isinstance(graph.llm_model, Runcrate)
assert graph.llm_model.model_name == "deepseek-ai/DeepSeek-V4.1-Flash"
assert graph.model_token == 1048576
assert not getattr(graph, "model_tokens_defaulted", False)