diff --git a/pyproject.toml b/pyproject.toml index a7c6567..28e1667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,9 @@ dev = [ # tests/test_network_guard.py unwraps exception groups, which Python 3.10 # has only through this backport. "exceptiongroup>=1.0; python_version < '3.11'", + # Validates each tool spec in the tool_calling catalog against the JSON + # Schema 2020-12 meta-schema, in tests/engine/probes/test_tool_catalog.py. + "jsonschema>=4.0,<5", ] # Empty on purpose: the gate runs a pinned pip-licenses through `uv tool run`, # so this group exists only to scope an isolated environment holding the diff --git a/src/usersim/engine/assets/tool_calling/README.md b/src/usersim/engine/assets/tool_calling/README.md index c84de03..334986e 100644 --- a/src/usersim/engine/assets/tool_calling/README.md +++ b/src/usersim/engine/assets/tool_calling/README.md @@ -241,17 +241,32 @@ other changes are listed per source: every distinct tool spec in the English `dev.json` split of [`data/English-dataset`](https://github.com/JoeYing1019/UltraTool/tree/5d340adc26d5881725308398dd5dfc6441eb0156/data/English-dataset) (commit `5d340ad`), with one toolset per upstream `domain` label. Tool names - and descriptions are unchanged. The `arguments` field is renamed - `parameters`, and the `results` (output) schema is removed, including where - upstream nests it in the arguments of `seat_preference_query` and - `select_and_change_ticket`. Argument schemas are otherwise unchanged, except - that `population_growth_rate` in `analyze_impact_on_economy` has type - `number`, where upstream writes `float`, which is not a JSON Schema type. + are unchanged. The `arguments` field is renamed `parameters`, and the + `results` (output) schema is removed, including where upstream nests it in + the arguments of `seat_preference_query` and `select_and_change_ticket`. + UltraTool's tools have no implementation, so each spec is checked against its + own description, the `dev.json` plan steps that call it and the `results` of + the tools that feed it: + - Every array declares its item type (179 parameters). A parameter whose + description, examples or plan calls show a number, a boolean or a list has + that type (98), among them `population_growth_rate` in + `analyze_impact_on_economy`, where upstream writes `float`. + - 103 parameters that a call needs and its schema lacks, such as the origin + and destination of `book_best_flight`, are added, named and typed after the + matching parameter of a related tool, and 33 tools require the inputs they + cannot run without. + - `example` is written `examples`, `min` is `minimum`, `dependencies` is + `dependentRequired`, `nullable` is removed, an enum lists each value once + and the values the tool's plans use, and a `format` outside JSON Schema becomes a `pattern`, a JSON Schema + format or text in the description (32 formats). + - Every nested field has a description (29 added), and 72 descriptions that + contradict their tool, or that leave a list's layout unstated, are + rewritten. - **ToolEyes** ([Ye et al., 2025](https://aclanthology.org/2025.coling-main.12/)): all 95 toolsets in [`Tool_Library`](https://github.com/Junjie-Ye/ToolEyes/tree/c818fcebe017951570a25e552a7984e40093fd52/Tool_Library) (commit `c818fce`), read from each toolset's `config_gpt4.json` and named - after its folders. Specs are unchanged and keep their upstream order. The two + after its folders. Specs keep their upstream order. The two agent-control functions that every upstream toolset includes, `ask_to_user` and `finish`, are removed. Three toolsets repeat a name upstream: the second `drivers` in `API F1` is `teams` and the second `blacklist_ip` in @@ -260,16 +275,34 @@ other changes are listed per source: identical `classify_grocery_product` specs. In `Execute`, two parameter names are the ones the functions' signatures in `tool.py` use: `code` in `execute_pycode` (upstream `code:`) and `return_err_output` in - `execute_bash_code` (upstream `return_err_outputpersistent=False`). + `execute_bash_code` (upstream `return_err_outputpersistent=False`). Each + spec is checked against the function of the same name in its toolset's + `tool.py`, the code that calls the API: + - A parameter that upstream types `string` has the type the function's + signature gives it, or that its use shows where the signature gives none + (669 parameters, most of them integers and booleans), and each array + declares its item type (80). + - A tool or parameter description that contradicts the function, such as the + nine CoinMarketCap tools that share one tool's text or Ergast's ids + described as names, or that is empty, such as 13 search tools, is written + from the endpoint the function calls (228 descriptions). + - `required` follows the signature where upstream's list differs from it + (14 tools), five parameters the functions take are added, and `random`, + which `search_recipes_by_ingredients` does not take, is removed. - **API-Bank** ([Li et al., 2023](https://aclanthology.org/2023.emnlp-main.187/)): 100 of the 101 APIs listed in [`api-bank/data/all_apis.csv`](https://github.com/AlibabaResearch/DAMO-ConvAI/blob/faf754671f5b850e1b4d1ff870db37a6a2480df6/api-bank/data/all_apis.csv) - (commit `faf7546`), all except `ImageCaption`. Descriptions and parameter - descriptions are unchanged. Tool names are the upstream class names in lower - case (`AddAlarm` becomes `addalarm`); parameter types are written as JSON - Schema types (`str` as `string`, `int` as `integer`, `datetime` as `string`, - `List[str]` as `array`); 32 tools gain a `required` list; and output - parameters are removed. The `All` toolset holds every API, and nine topical + (commit `faf7546`), all except `ImageCaption`. Tool names are the upstream + class names in lower case (`AddAlarm` becomes `addalarm`); parameter types + are written as JSON Schema types (`str` as `string`, `int` as `integer`, + `datetime` as `string`, `List[str]` as `array`); and output parameters are + removed. A tool requires the inputs that its signature in the `expressions` + column takes without a default, apart from inputs that upstream text marks + optional or that the class's `call` method does not take, and eight tools + gain the `token` input that their signature takes and their `api_info` + omits. Descriptions and parameter descriptions are upstream's, apart from 15 + corrected against the class's code and the CSV, such as `gettoday`, which + gets the current date. The `All` toolset holds every API, and nine topical toolsets repeat subsets of them. - **RestGPT** ([Song et al., 2023](https://arxiv.org/abs/2306.06624)): all 40 Spotify and 54 TMDB operations in RestBench's OpenAPI specs under @@ -279,14 +312,27 @@ other changes are listed per source: path, and the fields of its JSON request body; a body field that repeats a query parameter appears once, as the query parameter. A parameter is required when upstream marks it so, and the Spotify spec writes that mark as - the string `"true"` or `"false"`. Parameter schemas keep their type, `enum`, - `default` and `items`. Descriptions are joined onto one line; each tool + the string `"true"` or `"false"`. Three tools differ: `uris` in + `post_playlists_playlist_id_tracks` and each track's `uri` in + `delete_playlists_playlist_id_tracks` are required, and `get_recommendations` + requires none of its three seeds, since it needs at least one, as its + description says. Parameter schemas keep their type, `enum` and `items`, and + each `default` has its parameter's type (21 Spotify `limit` and `offset` + defaults that upstream writes as strings). + Descriptions are joined onto one line; each tool description ends with the operation's method and URL, and each parameter description ends with its location (for example `Location: query` or `Location: body`). One operation and 16 parameters that have no upstream description were given a short one: four query parameters and the 12 TMDB path parameters. In TMDB's `get_discover_tv`, the values that `with_status` - and `with_type` list for a string parameter are written as strings. + and `with_type` list for a string parameter are written as strings, and + `with_release_type`, which takes several release types, is a string. 13 + parameters that an operation's description promises and its spec omits are + added, typed and worded as elsewhere in the same spec: `include_image_language` + on the TMDB TV, season and episode images tools, `timezone` on + `get_tv_airing_today`, `append_to_response` on four details tools, and + `language` on the person and images tools. Eight descriptions that name the + wrong resource, such as the TV filters of `get_discover_tv`, are rewritten. - **AutoTools** ([Shi et al., 2025](https://arxiv.org/abs/2405.16533)): all 83 endpoints in the `food`, `game`, `music` and `weather` files under [`tools`](https://github.com/mangopy/AutoTools/tree/09c88df0959b2baf03e570aeff4365ea9f4d8b64/tools) @@ -297,12 +343,20 @@ other changes are listed per source: Parameters come from each endpoint's `parameters` list, which the `game` file names `params`. Descriptions are trimmed; each tool description ends with the endpoint's method and URL, and each parameter description ends with - its location (for example `Location: query`). Parameter types are written in - lower case, `float` as `number` and an empty type as `string`; a parameter - named in the endpoint's URL is required; and the sample responses are - removed. One `game` endpoint, `get_game_reviewed_today`, has its description - in the upstream `url` field, so its tool description ends with that text and - the API's host. + where it is sent, `Location: query` or `Location: path`. Upstream's `in` + field marks a needed parameter `query` or `required` and an optional one + `optional`: a parameter is required when it fills the endpoint's URL or, in + the `music`, `game` and `weather` files, when its `in` marks it needed (the + `food` file marks every parameter `query`). Parameter types are written in + lower case, `float` as `number` and an empty type as `string`, apart from + four numeric parameters that are integers, and each default has its + parameter's type. Parameter descriptions that upstream copies from another + parameter, such as a menu item's `item_id` described as a page number, are + written from the endpoint. The sample responses are removed. One `game` + endpoint, `get_game_reviewed_today`, has its description in the upstream + `url` field; its tool description is that text followed by + `HTTP GET https://opencritic-api.p.rapidapi.com/game/reviewed-today`, the + path the API's other game lists follow. ### Toolsets written for this project diff --git a/src/usersim/engine/assets/tool_calling/toolsets_seed.parquet b/src/usersim/engine/assets/tool_calling/toolsets_seed.parquet index bf9874c..1450074 100644 Binary files a/src/usersim/engine/assets/tool_calling/toolsets_seed.parquet and b/src/usersim/engine/assets/tool_calling/toolsets_seed.parquet differ diff --git a/tests/engine/probes/test_tool_catalog.py b/tests/engine/probes/test_tool_catalog.py index 311203f..c59a08a 100644 --- a/tests/engine/probes/test_tool_catalog.py +++ b/tests/engine/probes/test_tool_catalog.py @@ -4,7 +4,8 @@ """Tests for the tool catalog the tool_calling probe ships. Each row's tools reach the assistant model unchanged, as OpenAI-style function -specs, so every spec in ``toolsets_seed.parquet`` must be one a provider accepts. +specs, so every spec in ``toolsets_seed.parquet`` must be valid JSON Schema that +a provider accepts, with every tool and parameter described and typed. """ import json @@ -14,20 +15,13 @@ import pandas as pd import pytest +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError from usersim.engine.core.missing import is_missing from usersim.engine.probes.tool_calling.generator import ToolCallingProbe JSON_SCHEMA_TYPES = frozenset({"string", "number", "integer", "boolean", "array", "object", "null"}) -_PYTHON_TYPES = { - "string": str, - "number": (int, float), - "integer": int, - "boolean": bool, - "array": list, - "object": dict, - "null": type(None), -} _FUNCTION_NAME = re.compile(r"^[a-zA-Z0-9_-]{1,64}$") _PARAMETER_NAME = re.compile(r"^[a-zA-Z0-9_.-]{1,64}$") #: The method and URL a REST tool's description ends with. @@ -63,9 +57,8 @@ def _tools(toolsets) -> Iterator[tuple[str, dict]]: def _fits(value, json_type: str) -> bool: - if isinstance(value, bool) and json_type in ("number", "integer"): - return False - return isinstance(value, _PYTHON_TYPES[json_type]) + """Whether JSON Schema's ``type`` keyword accepts the value (a boolean is no integer).""" + return Draft202012Validator({"type": json_type}).is_valid(value) def test_num_tools_counts_each_toolsets_specs(toolsets): @@ -155,3 +148,103 @@ def test_each_placeholder_in_an_endpoint_is_a_required_parameter(toolsets): wrong.append(f"{where}: {name}") assert checked, "no tool description ends with an endpoint, so no placeholder was checked" assert not wrong, f"endpoint placeholders that are not required parameters: {wrong[:10]}" + + +#: The JSON Schema formats the catalog uses; anything else is a description, not a format. +_FORMATS = frozenset({"date-time", "date", "time", "email", "uri", "uuid"}) +#: Keywords a parameter schema may carry: JSON Schema 2020-12 validation and annotation keywords. +_KEYWORDS = frozenset( + { + "type", + "description", + "properties", + "required", + "items", + "enum", + "default", + "format", + "examples", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "pattern", + "minLength", + "maxLength", + "minItems", + "maxItems", + "uniqueItems", + "contentMediaType", + "dependentRequired", + "additionalProperties", + "title", + "const", + } +) + + +def test_every_parameters_schema_is_valid_json_schema(toolsets): + """Draft 2020-12 is the dialect providers validate a tool's parameters against.""" + wrong = [] + for where, f in _tools(toolsets): + try: + Draft202012Validator.check_schema(f["parameters"]) + except SchemaError as e: + wrong.append(f"{where}: {e.message}") + assert not wrong, f"parameters schemas that are not valid JSON Schema: {wrong[:10]}" + + +def test_every_array_declares_its_items(toolsets): + wrong = [ + f"{where}{path}" + for where, f in _tools(toolsets) + for path, schema in _schemas(f["parameters"], "") + if schema.get("type") == "array" and not isinstance(schema.get("items"), dict) + ] + assert not wrong, f"arrays without an item schema: {wrong[:10]}" + + +def test_schemas_use_json_schema_keywords_and_formats(toolsets): + keywords, formats = [], [] + for where, f in _tools(toolsets): + for path, schema in [("", f["parameters"]), *_schemas(f["parameters"], "")]: + keywords += [f"{where}{path}: {k}" for k in schema if k not in _KEYWORDS] + if "format" in schema and schema["format"] not in _FORMATS: + formats.append(f"{where}{path}: {schema['format']!r}") + assert not keywords, f"keywords outside JSON Schema: {keywords[:10]}" + assert not formats, f"formats outside JSON Schema: {formats[:10]}" + + +def test_defaults_and_examples_have_the_declared_type(toolsets): + wrong = [ + f"{where}{path}: {schema['type']} {value!r}" + for where, f in _tools(toolsets) + for path, schema in _schemas(f["parameters"], "") + if schema.get("type") in JSON_SCHEMA_TYPES + for value in ([schema["default"]] if "default" in schema else []) + list(schema.get("examples", [])) + if not _fits(value, schema["type"]) + ] + assert not wrong, f"defaults or examples of another type: {wrong[:10]}" + + +def test_enum_values_are_distinct(toolsets): + wrong = [ + f"{where}{path}" + for where, f in _tools(toolsets) + for path, schema in _schemas(f["parameters"], "") + if "enum" in schema and len({json.dumps(v) for v in schema["enum"]}) != len(schema["enum"]) + ] + assert not wrong, f"enums that repeat a value: {wrong[:10]}" + + +def test_every_tool_and_parameter_is_described(toolsets): + tools = [where for where, f in _tools(toolsets) if not str(f.get("description") or "").strip()] + params = [ + f"{where}{path}" + for where, f in _tools(toolsets) + for path, schema in _schemas(f["parameters"], "") + if not path.endswith("[]") and not str(schema.get("description") or "").strip() + ] + assert not tools, f"tools without a description: {tools[:10]}" + assert not params, f"parameters without a description: {params[:10]}" diff --git a/uv.lock b/uv.lock index a60f408..bebed22 100644 --- a/uv.lock +++ b/uv.lock @@ -4343,6 +4343,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "jsonschema" }, { name = "jupyter" }, { name = "pre-commit" }, { name = "pytest" }, @@ -4374,6 +4375,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "exceptiongroup", marker = "python_full_version < '3.11'", specifier = ">=1.0" }, + { name = "jsonschema", specifier = ">=4.0,<5" }, { name = "jupyter" }, { name = "pre-commit", specifier = ">=4.0,<5" }, { name = "pytest", specifier = ">=8.0" },