From bf0722666809188a23904ff857ed8d3b4b4e44bf Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 22 Sep 2026 20:29:35 -0300 Subject: [PATCH] docs(sdk): add Route Task to Model guide Document the route_task_to_model tool (ClassifyAndSwitchLLMTool) and its meta-profile shapes (structured classes + direct prompt), referencing the new examples/01_standalone_sdk/59_route_task_to_model.py in software-agent-sdk PR #4287. Co-authored-by: openhands --- docs.json | 1 + sdk/guides/llm-route-task-to-model.mdx | 230 +++++++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 sdk/guides/llm-route-task-to-model.mdx diff --git a/docs.json b/docs.json index 4a6ad027..c3277bc0 100644 --- a/docs.json +++ b/docs.json @@ -366,6 +366,7 @@ "sdk/guides/llm-subscriptions", "sdk/guides/llm-registry", "sdk/guides/llm-routing", + "sdk/guides/llm-route-task-to-model", "sdk/guides/llm-reasoning", "sdk/guides/llm-gpt5-preset", "sdk/guides/llm-streaming", diff --git a/sdk/guides/llm-route-task-to-model.mdx b/sdk/guides/llm-route-task-to-model.mdx new file mode 100644 index 00000000..6cdccca6 --- /dev/null +++ b/sdk/guides/llm-route-task-to-model.mdx @@ -0,0 +1,230 @@ +--- +title: Route Task to Model +description: Route each task to the best LLM profile with the built-in route_task_to_model tool. +--- + +import RunExampleCode from "/sdk/shared-snippets/how-to-run-example.mdx"; + +> A ready-to-run example is available [here](#ready-to-run-example)! + +The `route_task_to_model` tool (a.k.a. `ClassifyAndSwitchLLMTool`) lets an agent +start on a default LLM profile and switch to a better-suited profile per task. +When the agent calls the tool, a lightweight **classifier** LLM inspects the +recent conversation, picks the most suitable saved LLM profile for the task, and +switches the conversation to that profile before the agent continues. + +A **meta-profile** declaratively describes how to classify a task. It supports +two shapes: + +- **Structured classes** — a fixed set of `{description, model}` rows; the + classifier returns a class index. +- **Direct prompt** — a `prompt_template` rendered with `{{ instance_text }}` + and `{{ model_table }}`; the classifier returns the model name directly as + JSON. This is the shape used by the Pareto prompt meta-profiles. + +Both modes require the target models to exist as saved LLM profiles (the tool +switches to them by name) or to be supplied inline via `meta_profile_llms`. + +### Wiring the tool through settings + +Enable the tool and set the active meta-profile on `OpenHandsAgentSettings` — +that is all it takes to wire `route_task_to_model` into the agent: + +```python icon="python" wrap focus={4-6} +settings = OpenHandsAgentSettings( + llm=profile_store.load(DEFAULT_PROFILE), + enable_classify_and_switch_llm_tool=True, + active_meta_profile="structured", + meta_profile=structured_meta, +) +agent = settings.create_agent() +``` + +### Structured-classes meta-profile + +The classifier returns a 1-based class index; `model` is the saved profile name +to switch to. `classifier_model` is itself a saved profile name used to run the +classification call. + +```python icon="python" wrap focus={2-9} +structured_meta = MetaProfile( + classifier_model=CLASSIFIER_PROFILE, + classes=[ + MetaProfileClass( + description="Simple lookups, small edits, formatting", + model=CHEAP_PROFILE, + ), + MetaProfileClass( + description="Multi-file reasoning, debugging, architecture", + model=STRONG_PROFILE, + ), + ], +) +``` + +### Direct-prompt meta-profile + +Instead of fixed classes, give the classifier a free-form prompt template and a +model table. The classifier returns the model name directly as JSON +`{"model": "", "reason": "..."}`. + +```python icon="python" wrap focus={2-12} +direct_meta = MetaProfile( + classifier_model=CLASSIFIER_PROFILE, + prompt_template=( + "Pick the best model for the task below.\n\n" + "{{ model_table }}\n\n" + "Task:\n{{ instance_text }}\n\n" + 'Return ONLY JSON: {"model": "", "reason": ""}' + ), + model_table=( + f"- {CHEAP_PROFILE}: fast and cheap, good for simple tasks\n" + f"- {STRONG_PROFILE}: slower and stronger, good for hard tasks" + ), +) +``` + +## Ready-to-run Example + + +This example is available on GitHub: [examples/01_standalone_sdk/59_route_task_to_model.py](https://github.com/OpenHands/software-agent-sdk/blob/main/examples/01_standalone_sdk/59_route_task_to_model.py) + + +Route each task to the best LLM profile with the built-in `route_task_to_model` +tool, demonstrating both the structured-classes and direct-prompt meta-profile +shapes: + +```python icon="python" expandable examples/01_standalone_sdk/59_route_task_to_model.py +import os + +from pydantic import SecretStr + +from openhands.sdk import LLM, Conversation, OpenHandsAgentSettings +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.meta_profile_store import MetaProfile, MetaProfileClass + + +DEFAULT_BASE_URL = "https://llm-proxy.app.all-hands.dev" + +# Saved profile names. The route_task_to_model tool switches the conversation +# to one of these by name, so each must exist in the LLMProfileStore below. +DEFAULT_PROFILE = "example-router-default" +CHEAP_PROFILE = "example-router-cheap" +STRONG_PROFILE = "example-router-strong" +CLASSIFIER_PROFILE = "example-router-classifier" + +CHEAP_MODEL = "openai/gpt-5.5" +STRONG_MODEL = "openai/prod/claude-sonnet-4-5-20250929" + +api_key = os.getenv("LLM_API_KEY") +assert api_key is not None, "LLM_API_KEY environment variable is not set." +base_url = os.getenv("LLM_BASE_URL", DEFAULT_BASE_URL) + + +# ── 1. Save the LLM profiles the router will choose between ────────────── +profile_store = LLMProfileStore() +for name, model, usage_id in [ + (DEFAULT_PROFILE, CHEAP_MODEL, "router-default"), + (CHEAP_PROFILE, CHEAP_MODEL, "router-cheap"), + (STRONG_PROFILE, STRONG_MODEL, "router-strong"), + (CLASSIFIER_PROFILE, CHEAP_MODEL, "router-classifier"), +]: + profile_store.save( + name, + LLM( + model=model, + api_key=SecretStr(api_key), + base_url=base_url, + usage_id=usage_id, + ), + include_secrets=True, + ) + +try: + # ── 2. Define a meta-profile: structured classes ────────────────────── + # The classifier returns a 1-based class index; ``model`` is the saved + # profile name to switch to. ``classifier_model`` is itself a saved + # profile name used to run the classification call. + structured_meta = MetaProfile( + classifier_model=CLASSIFIER_PROFILE, + classes=[ + MetaProfileClass( + description="Simple lookups, small edits, formatting", + model=CHEAP_PROFILE, + ), + MetaProfileClass( + description="Multi-file reasoning, debugging, architecture", + model=STRONG_PROFILE, + ), + ], + ) + + # ── 3. Build the agent via OpenHandsAgentSettings ───────────────────── + # Enabling the tool + setting the active meta-profile name is all it takes + # to wire route_task_to_model into the agent. The agent starts on the + # default profile and switches only when it calls the tool. + settings = OpenHandsAgentSettings( + llm=profile_store.load(DEFAULT_PROFILE), + enable_classify_and_switch_llm_tool=True, + active_meta_profile="structured", + meta_profile=structured_meta, + ) + agent = settings.create_agent() + + conversation = Conversation(agent=agent, workspace=os.getcwd()) + print(f"Starting model: {conversation.agent.llm.model}") + + conversation.send_message( + "Call the route_task_to_model tool now. After it returns, answer in one " + "short sentence naming the model the tool switched to." + ) + conversation.run() + + print(f"Active model after routing: {conversation.agent.llm.model}") + + for usage_id, metrics in conversation.state.stats.usage_to_metrics.items(): + print(f" [{usage_id}] cost=${metrics.accumulated_cost:.6f}") + combined = conversation.state.stats.get_combined_metrics() + print(f"Total cost: ${combined.accumulated_cost:.6f}") + print(f"EXAMPLE_COST: {combined.accumulated_cost}") + + # ── 4. (Info) Direct-prompt meta-profile shape ──────────────────────── + # Instead of fixed classes, you give the classifier a free-form prompt + # template and a model table. The classifier returns the model name + # directly as JSON ``{"model": "", "reason": "..."}``. This is the + # shape used by the Pareto prompt meta-profiles. + direct_meta = MetaProfile( + classifier_model=CLASSIFIER_PROFILE, + prompt_template=( + "Pick the best model for the task below.\n\n" + "{{ model_table }}\n\n" + "Task:\n{{ instance_text }}\n\n" + 'Return ONLY JSON: {"model": "", "reason": ""}' + ), + model_table=( + f"- {CHEAP_PROFILE}: fast and cheap, good for simple tasks\n" + f"- {STRONG_PROFILE}: slower and stronger, good for hard tasks" + ), + ) + print() + print("Direct-prompt meta-profile (not executed here):") + print(direct_meta.model_dump_json(indent=2)) + +finally: + for name in [ + DEFAULT_PROFILE, + CHEAP_PROFILE, + STRONG_PROFILE, + CLASSIFIER_PROFILE, + ]: + profile_store.delete(name) +``` + + + + +## Next Steps + +- **[Model Routing](/sdk/guides/llm-routing)** — Route requests based on content (e.g., multimodal vs text-only) +- **[LLM Profile Store](/sdk/guides/llm-profile-store)** — Save and load reusable LLM configurations +- **[LLM Metrics](/sdk/guides/metrics)** — Track token usage and costs