fix(assets): correct the tool catalog's specs against their sources

pyproject.toml

@@ -115,6 +115,9 @@ dev = [
     # tests/test_network_guard.py unwraps exception groups, which Python 3.10
     # has only through this backport.
     "exceptiongroup>=1.0; python_version < '3.11'",
+    # Validates each tool spec in the tool_calling catalog against the JSON
+    # Schema 2020-12 meta-schema, in tests/engine/probes/test_tool_catalog.py.
+    "jsonschema>=4.0,<5",
 ]
 # Empty on purpose: the gate runs a pinned pip-licenses through `uv tool run`,
 # so this group exists only to scope an isolated environment holding the

src/usersim/engine/assets/tool_calling/README.md

@@ -241,17 +241,32 @@ other changes are listed per source:
   every distinct tool spec in the English `dev.json` split of
   [`data/English-dataset`](https://github.com/JoeYing1019/UltraTool/tree/5d340adc26d5881725308398dd5dfc6441eb0156/data/English-dataset)
   (commit `5d340ad`), with one toolset per upstream `domain` label. Tool names
-  and descriptions are unchanged. The `arguments` field is renamed
-  `parameters`, and the `results` (output) schema is removed, including where
-  upstream nests it in the arguments of `seat_preference_query` and
-  `select_and_change_ticket`. Argument schemas are otherwise unchanged, except
-  that `population_growth_rate` in `analyze_impact_on_economy` has type
-  `number`, where upstream writes `float`, which is not a JSON Schema type.
+  are unchanged. The `arguments` field is renamed `parameters`, and the
+  `results` (output) schema is removed, including where upstream nests it in
+  the arguments of `seat_preference_query` and `select_and_change_ticket`.
+  UltraTool's tools have no implementation, so each spec is checked against its
+  own description, the `dev.json` plan steps that call it and the `results` of
+  the tools that feed it:
+  - Every array declares its item type (179 parameters). A parameter whose
+    description, examples or plan calls show a number, a boolean or a list has
+    that type (98), among them `population_growth_rate` in
+    `analyze_impact_on_economy`, where upstream writes `float`.
+  - 103 parameters that a call needs and its schema lacks, such as the origin
+    and destination of `book_best_flight`, are added, named and typed after the
+    matching parameter of a related tool, and 33 tools require the inputs they
+    cannot run without.
+  - `example` is written `examples`, `min` is `minimum`, `dependencies` is
+    `dependentRequired`, `nullable` is removed, an enum lists each value once
+    and the values the tool's plans use, and a `format` outside JSON Schema becomes a `pattern`, a JSON Schema
+    format or text in the description (32 formats).
+  - Every nested field has a description (29 added), and 72 descriptions that
+    contradict their tool, or that leave a list's layout unstated, are
+    rewritten.
 - **ToolEyes** ([Ye et al., 2025](https://aclanthology.org/2025.coling-main.12/)):
   all 95 toolsets in
   [`Tool_Library`](https://github.com/Junjie-Ye/ToolEyes/tree/c818fcebe017951570a25e552a7984e40093fd52/Tool_Library)
   (commit `c818fce`), read from each toolset's `config_gpt4.json` and named
-  after its folders. Specs are unchanged and keep their upstream order. The two
+  after its folders. Specs keep their upstream order. The two
   agent-control functions that every upstream toolset includes, `ask_to_user`
   and `finish`, are removed. Three toolsets repeat a name upstream: the second
   `drivers` in `API F1` is `teams` and the second `blacklist_ip` in
@@ -260,16 +275,34 @@ other changes are listed per source:
   identical `classify_grocery_product` specs. In `Execute`, two parameter names
   are the ones the functions' signatures in `tool.py` use: `code` in
   `execute_pycode` (upstream `code:`) and `return_err_output` in
-  `execute_bash_code` (upstream `return_err_outputpersistent=False`).
+  `execute_bash_code` (upstream `return_err_outputpersistent=False`). Each
+  spec is checked against the function of the same name in its toolset's
+  `tool.py`, the code that calls the API:
+  - A parameter that upstream types `string` has the type the function's
+    signature gives it, or that its use shows where the signature gives none
+    (669 parameters, most of them integers and booleans), and each array
+    declares its item type (80).
+  - A tool or parameter description that contradicts the function, such as the
+    nine CoinMarketCap tools that share one tool's text or Ergast's ids
+    described as names, or that is empty, such as 13 search tools, is written
+    from the endpoint the function calls (228 descriptions).
+  - `required` follows the signature where upstream's list differs from it
+    (14 tools), five parameters the functions take are added, and `random`,
+    which `search_recipes_by_ingredients` does not take, is removed.
 - **API-Bank** ([Li et al., 2023](https://aclanthology.org/2023.emnlp-main.187/)):
   100 of the 101 APIs listed in
   [`api-bank/data/all_apis.csv`](https://github.com/AlibabaResearch/DAMO-ConvAI/blob/faf754671f5b850e1b4d1ff870db37a6a2480df6/api-bank/data/all_apis.csv)
-  (commit `faf7546`), all except `ImageCaption`. Descriptions and parameter
-  descriptions are unchanged. Tool names are the upstream class names in lower
-  case (`AddAlarm` becomes `addalarm`); parameter types are written as JSON
-  Schema types (`str` as `string`, `int` as `integer`, `datetime` as `string`,
-  `List[str]` as `array`); 32 tools gain a `required` list; and output
-  parameters are removed. The `All` toolset holds every API, and nine topical
+  (commit `faf7546`), all except `ImageCaption`. Tool names are the upstream
+  class names in lower case (`AddAlarm` becomes `addalarm`); parameter types
+  are written as JSON Schema types (`str` as `string`, `int` as `integer`,
+  `datetime` as `string`, `List[str]` as `array`); and output parameters are
+  removed. A tool requires the inputs that its signature in the `expressions`
+  column takes without a default, apart from inputs that upstream text marks
+  optional or that the class's `call` method does not take, and eight tools
+  gain the `token` input that their signature takes and their `api_info`
+  omits. Descriptions and parameter descriptions are upstream's, apart from 15
+  corrected against the class's code and the CSV, such as `gettoday`, which
+  gets the current date. The `All` toolset holds every API, and nine topical
   toolsets repeat subsets of them.
 - **RestGPT** ([Song et al., 2023](https://arxiv.org/abs/2306.06624)):
   all 40 Spotify and 54 TMDB operations in RestBench's OpenAPI specs under
@@ -279,14 +312,27 @@ other changes are listed per source:
   path, and the fields of its JSON request body; a body field that repeats a
   query parameter appears once, as the query parameter. A parameter is
   required when upstream marks it so, and the Spotify spec writes that mark as
-  the string `"true"` or `"false"`. Parameter schemas keep their type, `enum`,
-  `default` and `items`. Descriptions are joined onto one line; each tool
+  the string `"true"` or `"false"`. Three tools differ: `uris` in
+  `post_playlists_playlist_id_tracks` and each track's `uri` in
+  `delete_playlists_playlist_id_tracks` are required, and `get_recommendations`
+  requires none of its three seeds, since it needs at least one, as its
+  description says. Parameter schemas keep their type, `enum` and `items`, and
+  each `default` has its parameter's type (21 Spotify `limit` and `offset`
+  defaults that upstream writes as strings).
+  Descriptions are joined onto one line; each tool
   description ends with the operation's method and URL, and each parameter
   description ends with its location (for example `Location: query` or
   `Location: body`). One operation and 16 parameters that have no upstream
   description were given a short one: four query parameters and the 12 TMDB
   path parameters. In TMDB's `get_discover_tv`, the values that `with_status`
-  and `with_type` list for a string parameter are written as strings.
+  and `with_type` list for a string parameter are written as strings, and
+  `with_release_type`, which takes several release types, is a string. 13
+  parameters that an operation's description promises and its spec omits are
+  added, typed and worded as elsewhere in the same spec: `include_image_language`
+  on the TMDB TV, season and episode images tools, `timezone` on
+  `get_tv_airing_today`, `append_to_response` on four details tools, and
+  `language` on the person and images tools. Eight descriptions that name the
+  wrong resource, such as the TV filters of `get_discover_tv`, are rewritten.
 - **AutoTools** ([Shi et al., 2025](https://arxiv.org/abs/2405.16533)):
   all 83 endpoints in the `food`, `game`, `music` and `weather` files under
   [`tools`](https://github.com/mangopy/AutoTools/tree/09c88df0959b2baf03e570aeff4365ea9f4d8b64/tools)
@@ -297,12 +343,20 @@ other changes are listed per source:
   Parameters come from each endpoint's `parameters` list, which the `game`
   file names `params`. Descriptions are trimmed; each tool description ends
   with the endpoint's method and URL, and each parameter description ends with
-  its location (for example `Location: query`). Parameter types are written in
-  lower case, `float` as `number` and an empty type as `string`; a parameter
-  named in the endpoint's URL is required; and the sample responses are
-  removed. One `game` endpoint, `get_game_reviewed_today`, has its description
-  in the upstream `url` field, so its tool description ends with that text and
-  the API's host.
+  where it is sent, `Location: query` or `Location: path`. Upstream's `in`
+  field marks a needed parameter `query` or `required` and an optional one
+  `optional`: a parameter is required when it fills the endpoint's URL or, in
+  the `music`, `game` and `weather` files, when its `in` marks it needed (the
+  `food` file marks every parameter `query`). Parameter types are written in
+  lower case, `float` as `number` and an empty type as `string`, apart from
+  four numeric parameters that are integers, and each default has its
+  parameter's type. Parameter descriptions that upstream copies from another
+  parameter, such as a menu item's `item_id` described as a page number, are
+  written from the endpoint. The sample responses are removed. One `game`
+  endpoint, `get_game_reviewed_today`, has its description in the upstream
+  `url` field; its tool description is that text followed by
+  `HTTP GET https://opencritic-api.p.rapidapi.com/game/reviewed-today`, the
+  path the API's other game lists follow.
 
 ### Toolsets written for this project
 

tests/engine/probes/test_tool_catalog.py

@@ -4,7 +4,8 @@
 """Tests for the tool catalog the tool_calling probe ships.
 
 Each row's tools reach the assistant model unchanged, as OpenAI-style function
-specs, so every spec in ``toolsets_seed.parquet`` must be one a provider accepts.
+specs, so every spec in ``toolsets_seed.parquet`` must be valid JSON Schema that
+a provider accepts, with every tool and parameter described and typed.
 """
 
 import json
@@ -14,20 +15,13 @@
 
 import pandas as pd
 import pytest
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError
 
 from usersim.engine.core.missing import is_missing
 from usersim.engine.probes.tool_calling.generator import ToolCallingProbe
 
 JSON_SCHEMA_TYPES = frozenset({"string", "number", "integer", "boolean", "array", "object", "null"})
-_PYTHON_TYPES = {
-    "string": str,
-    "number": (int, float),
-    "integer": int,
-    "boolean": bool,
-    "array": list,
-    "object": dict,
-    "null": type(None),
-}
 _FUNCTION_NAME = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")
 _PARAMETER_NAME = re.compile(r"^[a-zA-Z0-9_.-]{1,64}$")
 #: The method and URL a REST tool's description ends with.
@@ -63,9 +57,8 @@ def _tools(toolsets) -> Iterator[tuple[str, dict]]:
 
 
 def _fits(value, json_type: str) -> bool:
-    if isinstance(value, bool) and json_type in ("number", "integer"):
-        return False
-    return isinstance(value, _PYTHON_TYPES[json_type])
+    """Whether JSON Schema's ``type`` keyword accepts the value (a boolean is no integer)."""
+    return Draft202012Validator({"type": json_type}).is_valid(value)
 
 
 def test_num_tools_counts_each_toolsets_specs(toolsets):
@@ -155,3 +148,103 @@ def test_each_placeholder_in_an_endpoint_is_a_required_parameter(toolsets):
                 wrong.append(f"{where}: {name}")
     assert checked, "no tool description ends with an endpoint, so no placeholder was checked"
     assert not wrong, f"endpoint placeholders that are not required parameters: {wrong[:10]}"
+
+
+#: The JSON Schema formats the catalog uses; anything else is a description, not a format.
+_FORMATS = frozenset({"date-time", "date", "time", "email", "uri", "uuid"})
+#: Keywords a parameter schema may carry: JSON Schema 2020-12 validation and annotation keywords.
+_KEYWORDS = frozenset(
+    {
+        "type",
+        "description",
+        "properties",
+        "required",
+        "items",
+        "enum",
+        "default",
+        "format",
+        "examples",
+        "minimum",
+        "maximum",
+        "exclusiveMinimum",
+        "exclusiveMaximum",
+        "multipleOf",
+        "pattern",
+        "minLength",
+        "maxLength",
+        "minItems",
+        "maxItems",
+        "uniqueItems",
+        "contentMediaType",
+        "dependentRequired",
+        "additionalProperties",
+        "title",
+        "const",
+    }
+)
+
+
+def test_every_parameters_schema_is_valid_json_schema(toolsets):
+    """Draft 2020-12 is the dialect providers validate a tool's parameters against."""
+    wrong = []
+    for where, f in _tools(toolsets):
+        try:
+            Draft202012Validator.check_schema(f["parameters"])
+        except SchemaError as e:
+            wrong.append(f"{where}: {e.message}")
+    assert not wrong, f"parameters schemas that are not valid JSON Schema: {wrong[:10]}"
+
+
+def test_every_array_declares_its_items(toolsets):
+    wrong = [
+        f"{where}{path}"
+        for where, f in _tools(toolsets)
+        for path, schema in _schemas(f["parameters"], "")
+        if schema.get("type") == "array" and not isinstance(schema.get("items"), dict)
+    ]
+    assert not wrong, f"arrays without an item schema: {wrong[:10]}"
+
+
+def test_schemas_use_json_schema_keywords_and_formats(toolsets):
+    keywords, formats = [], []
+    for where, f in _tools(toolsets):
+        for path, schema in [("", f["parameters"]), *_schemas(f["parameters"], "")]:
+            keywords += [f"{where}{path}: {k}" for k in schema if k not in _KEYWORDS]
+            if "format" in schema and schema["format"] not in _FORMATS:
+                formats.append(f"{where}{path}: {schema['format']!r}")
+    assert not keywords, f"keywords outside JSON Schema: {keywords[:10]}"
+    assert not formats, f"formats outside JSON Schema: {formats[:10]}"
+
+
+def test_defaults_and_examples_have_the_declared_type(toolsets):
+    wrong = [
+        f"{where}{path}: {schema['type']} {value!r}"
+        for where, f in _tools(toolsets)
+        for path, schema in _schemas(f["parameters"], "")
+        if schema.get("type") in JSON_SCHEMA_TYPES
+        for value in ([schema["default"]] if "default" in schema else []) + list(schema.get("examples", []))
+        if not _fits(value, schema["type"])
+    ]
+    assert not wrong, f"defaults or examples of another type: {wrong[:10]}"
+
+
+def test_enum_values_are_distinct(toolsets):
+    wrong = [
+        f"{where}{path}"
+        for where, f in _tools(toolsets)
+        for path, schema in _schemas(f["parameters"], "")
+        if "enum" in schema and len({json.dumps(v) for v in schema["enum"]}) != len(schema["enum"])
+    ]
+    assert not wrong, f"enums that repeat a value: {wrong[:10]}"
+
+
+def test_every_tool_and_parameter_is_described(toolsets):
+    tools = [where for where, f in _tools(toolsets) if not str(f.get("description") or "").strip()]
+    params = [
+        f"{where}{path}"
+        for where, f in _tools(toolsets)
+        for path, schema in _schemas(f["parameters"], "")
+        if not path.endswith("[]") and not str(schema.get("description") or "").strip()
+    ]
+    assert not tools, f"tools without a description: {tools[:10]}"
+    assert not params, f"parameters without a description: {params[:10]}"