From a6b13fce9c3fd97ab08981fa666debea39320dc8 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 21:30:15 -0700 Subject: [PATCH 01/17] docs(openspec): propose the Vale authoring polish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Vale rule that is malformed fails loudly. A Vale rule that is wrong passes verify, passes test, and reports nothing forever. Authoring the first house-style rules against this repository produced three of those in a row, each green on every local gate. Measured against the pinned 3.18.0, the engine does not close them: `extends: nonsense` and `scope: fenced` both verify clean and match nothing, while a field belonging to another check type throws E201 and suppresses every other Vale rule, because Vale reads one assembled config per run. The change adds a schema layer to the Vale verify path and closes the recipe gaps that produce a silent rule. Vale publishes no JSON Schema — the machine-readable field knowledge sits behind its paid hosted MCP — so the schema is hand-authored, and a corpus of rule files run through both the binary and the schema is what keeps the transcription honest. Delivery shape: stacked, merging forward. The recipe text lands alone, then the schema and its verify layer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .../vale-authoring-polish/.openspec.yaml | 2 + .../changes/vale-authoring-polish/design.md | 100 ++++++++++++++++++ .../changes/vale-authoring-polish/proposal.md | 45 ++++++++ .../specs/cli-agent-authoring/spec.md | 79 ++++++++++++++ .../specs/cli-rule-validation/spec.md | 66 ++++++++++++ .../changes/vale-authoring-polish/tasks.md | 65 ++++++++++++ 6 files changed, 357 insertions(+) create mode 100644 openspec/changes/vale-authoring-polish/.openspec.yaml create mode 100644 openspec/changes/vale-authoring-polish/design.md create mode 100644 openspec/changes/vale-authoring-polish/proposal.md create mode 100644 openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md create mode 100644 openspec/changes/vale-authoring-polish/specs/cli-rule-validation/spec.md create mode 100644 openspec/changes/vale-authoring-polish/tasks.md diff --git a/openspec/changes/vale-authoring-polish/.openspec.yaml b/openspec/changes/vale-authoring-polish/.openspec.yaml new file mode 100644 index 00000000..e685d45e --- /dev/null +++ b/openspec/changes/vale-authoring-polish/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-25 diff --git a/openspec/changes/vale-authoring-polish/design.md b/openspec/changes/vale-authoring-polish/design.md new file mode 100644 index 00000000..98b39aa4 --- /dev/null +++ b/openspec/changes/vale-authoring-polish/design.md @@ -0,0 +1,100 @@ +## Context + +`verify` runs three layers against an `sg` rule: the ast-grep JSON schema, the Taskless required fields, and then the tests. For a `vale` rule it runs the last two only. `packages/cli/src/rules/vale/verify.ts` is 318 lines of fixture behavior — bucket coverage, then which fixtures fired — plus a `level` check and the `.vale.ini` presence checks. Nothing reads the rule's structure. + +That gap is not theoretical, and the engine does not close it. Measured against the pinned Vale 3.18.0: + +| Rule defect | `verify` | `test` | `check` | +| --------------------------------- | -------------- | --------------------------- | ------------------------------ | +| `level: bananas` | **reports it** | — | — | +| `extends: nonsense` | `ok: true` | "fail fixture did not fire" | silent | +| `scope: fenced` | `ok: true` | "fail fixture did not fire" | silent | +| `tokens` on an `occurrence` check | `ok: true` | `E201: has invalid keys` | **every Vale rule suppressed** | + +The two middle rows are the shape that matters. The author is told their pattern did not match, so they go and edit the pattern. The fourth row is worse in a different direction: Vale reads one assembled config per run, so one stray field takes the whole engine's reporting down. + +The ast-grep side avoids this because ast-grep publishes `schemas/rule.json` per tag, and `packages/cli/scripts/fetch-ast-grep-schema.ts` pins it to the vendored version. Vale publishes nothing equivalent. Its repository has no `schemas/` directory; the machine-readable field knowledge exists behind the hosted MCP server at `api.vale.sh/mcp`, which is part of the paid Vale CMS product. + +## Goals / Non-Goals + +**Goals:** + +- `verify` rejects a Vale rule whose `extends`, `scope`, or per-check fields the binary will not honor, before Vale is invoked. +- The schema's claims are held to the vendored binary, so a Vale upgrade fails loudly rather than silently invalidating them. +- The `create-vale-rule` recipe documents the silent failures, which is the only place they can be prevented for the cases a schema cannot see. + +**Non-Goals:** + +- Depending on Vale's hosted MCP server at author or verify time. It is remote and paid; `verify` must work offline. +- Validating regex _semantics_. Whether a token is too broad, or whether `landed on` also matches "the plane landed on time", is judgment. That belongs in the recipe. +- Changing what `check` reports for any rule that is valid today. +- Reworking the `.vale.ini`. Converting it to JSON and schema-validating it was considered and rejected — `verify` already catches both of its silent failures ("no `.vale.ini`, so nothing scopes it" and "never enables `.`"), which is what a schema there would buy. + +## Decisions + +### Hand-author the schema, pinned to `VALE_VERSION` + +There is no upstream artifact to generate from, so the choice is between transcribing the docs and not having a schema. Transcribe — but treat the transcription as untrusted. + +_Alternative considered: derive the schema by probing the binary at build time._ Attractive, because it cannot drift. Rejected because Vale does not report its accepted values; the only probe available is "author a rule and see whether it fires", which distinguishes "invalid" from "valid but did not match" only through a fixture that is itself hand-written. That is the vendor-contract test, and it belongs in the test suite rather than in a generator. + +_Alternative considered: query the hosted MCP `explain_check` at build time and vendor the result._ Rejected. It makes a build step depend on a paid third-party service, and the output would still need pinning and checking. + +### The vendor-contract test is what makes the transcription safe + +`packages/cli/test/vale-vendor-contract.test.ts` already pins Vale's givens by invoking the vendored binary directly rather than through `runVale`. Extend it, so that every value the schema asserts is one the binary was measured accepting. A Vale bump that adds or removes a check type then fails a test that names the field, instead of leaving the schema quietly wrong. + +This is the same relationship `capabilities.ts` already has with the format tiers, and the precedent is deliberate: an engine claim in this repository is quoted from a pinned binary rather than from release notes. + +### `scope` is a grammar, not an enum + +The recipe presents `scope` as a single value. Measured, the binary accepts a bare string, a list, `~` negation, and `&` chaining — `~code`, `[~code]`, and `text & ~code` all parse and behave. A schema modeling `scope` as a flat enum would reject valid rules, which is worse than the gap it closes, so the enum applies to the _operands_ and the schema accepts the operators around them. + +The vocabulary is also larger than the recipe's list: `heading` with levels, the `table.*` and `figure.caption` forms, `list`, `paragraph`, `sentence`, `blockquote`, `alt`, `summary`, `raw`, the v3.17.0+ inline scopes (`link`, `code`, `strong`, `emphasis`), `text.class.`, the v3.18.0+ `meta` and `meta.class.`, and `comment.line` / `comment.block`. + +`scope` is the highest-value field in the schema, because it is the one field nothing downstream ever validates. An invalid scope is not an error anywhere in the stack — it is a rule that matches nothing. + +### The corpus is the artifact; the schema is derived from it + +The schema is a transcription, so the thing that makes it true is not how carefully it is written — it is a corpus of rule files, each run through **both** the vendored binary and the schema, asserting the two agree. Build the corpus first and let the schema fall out of it, rather than writing the schema and adding tests afterwards. + +Each corpus entry is a minimal rule YAML plus the verdict the binary was measured giving it. The test asserts `schemaAccepts === binaryAccepts` for every entry, and both directions of disagreement are failures: + +- **Schema accepts, binary rejects** — the gap this change exists to close. +- **Schema rejects, binary accepts** — worse, because it blocks work that would have functioned. This is the direction to be paranoid about, and it is why an unclear value is accepted rather than rejected. + +**Deciding the binary's verdict is the tedious part, and it cannot be read off an exit code.** Vale does not report "this is not a scope." An unrecognized `extends` or `scope` parses clean and produces a rule that matches nothing — indistinguishable, from the outside, from a valid rule whose pattern did not fire. So every corpus entry needs a **positive control**: a fixture document the rule _must_ flag when the construct is valid. "Fired" then means accepted, "did not fire" means the binary ignored it, and `E201` means rejected outright. Three outcomes, not two. + +That control has to be written per entry, because it depends on what the rule matches — a `scope: heading` entry needs a heading, a `scope: table.cell` entry needs a table. There is no way to generate it from the check type alone, and this is the bulk of the work. + +Keep the corpus declarative — a table of `{ check, field, value, fixture, expected }` rather than a test per case — so that a Vale upgrade is a re-run rather than a re-authoring, and so a gap in coverage is visible as a missing row rather than as an absent test nobody notices. + +_Alternative considered: assert the schema against the docs and skip the corpus._ Rejected. It would encode the same transcription twice and prove only that we copied consistently. The corpus is the only thing that can catch a docs page that is wrong or stale about the pinned version — and the docs are already inconsistent with the MCP guide about how many check types exist. + +### Follow the `sg` shape rather than inventing one + +`verify` already has a `schema` layer with a `LayerResult` and an error list. The Vale path reports through the same `VerifyResult`. Reuse both, so the two engines fail the same way and the JSON output shape does not gain a special case. + +### Resolve the check-type count against the binary, not the docs + +The docs enumerate eleven check types. The MCP guide says `scaffold_rule` covers twelve. Do not pick a number: enumerate against the vendored binary and record what was measured. An enum that omits a real check type rejects valid rules. + +## Risks / Trade-offs + +- **The schema drifts from a Vale upgrade.** → The corpus is re-run against the new binary and the disagreement names the field. This is the whole reason the corpus is a requirement rather than a nicety. +- **The corpus is the expensive part and will be under-built.** Every entry needs a hand-written positive control, and an entry that silently lacks one asserts nothing — it "passes" because nothing fired on either side. → Assert that each entry's control fires for at least one known-valid variant, so a control that can never fire is itself a test failure rather than a quiet pass. +- **The schema is stricter than the binary and rejects a valid rule.** → Worse than the gap it closes, because it blocks work that would have functioned. Mitigated by deriving every enumerated value from a measurement, and by the `scope` grammar decision above. When a value's status is unclear, accept it. +- **A rule in the wild fails `verify` after this lands.** → Only if it was already inert or already suppressing the engine. Both are defects the author wants to know about. No rule that reports findings today can start failing. +- **Recipe growth.** `create-vale-rule` is already the longest recipe, and this adds to it. → The additions are failure-mode statements with measurements attached, which is the material the recipe is densest in already. If length becomes the problem, the split is by engine surface, not by trimming the measured parts. + +## Migration Plan + +No migration. The schema layer is additive, and it only rejects rules that the engine was already going to ignore or fail on. + +The stack lands forward in two units, per the proposal: the recipe text first, then the schema and its `verify` layer. If the second unit needs to be reverted, the first stands on its own. + +## Open Questions + +- **Eleven check types or twelve?** To be resolved by enumeration against the vendored binary during implementation. +- **How much of the per-check field table to encode.** The full table makes `E201` unreachable; a partial one still leaves the engine-wide suppression possible for the fields it omits. Recommend the full table, since `E201` is the failure with the widest blast radius, but the cost is that every check type must be transcribed and measured rather than just the common header. +- **Whether `verify` should also report each matcher's selected-file count.** Out of scope here, but adjacent: a matcher glob that reaches nothing produces a rule that is well formed, enabled, green, and inert — the one silent failure this change does not close. diff --git a/openspec/changes/vale-authoring-polish/proposal.md b/openspec/changes/vale-authoring-polish/proposal.md new file mode 100644 index 00000000..8ddca3b6 --- /dev/null +++ b/openspec/changes/vale-authoring-polish/proposal.md @@ -0,0 +1,45 @@ +## Why + +A Vale rule that is _malformed_ fails loudly. A Vale rule that is _wrong_ passes `verify`, passes `test`, and reports nothing forever — and nobody re-checks a green rule. Authoring the first house-style rules against this repository produced three of those in a row, each green on every local gate: + +- An em-dash rule whose token can never match, because Vale wraps tokens in word boundaries and an em dash is non-word on both sides. It needed `nonword: true`, which nothing asked for. +- A rule about a shell command that caught zero of two real violations, because commands in a README live in fenced blocks and Vale skips those by default. +- A rule scoped to `packages/cli/src/**/*.ts` whose clean report was indistinguishable from a matcher glob that reached nothing. + +Measured against the pinned Vale 3.18.0, the engine does not close these. `extends: nonsense` and `scope: fenced` both pass `verify` and simply match nothing; a field belonging to a different check type throws `E201`, which takes **every** Vale rule in the project down because Vale reads one config per run. The `create-vale-rule` recipe carries the knowledge that exists, but it is incomplete in the specific places that produce silent rules. + +## What Changes + +- **`verify` validates a Vale rule file structurally**, before Vale sees it, against a schema that models the check types and their fields. Today it validates `level` and the presence of the rule's `.vale.ini`, and nothing else — `extends: nonsense` verifies clean. +- **The schema is hand-authored and pinned to `VALE_VERSION`**, because there is nothing to generate it from. Vale publishes no JSON Schema; the machine-readable field knowledge exists only behind its paid cloud MCP at `api.vale.sh/mcp`, which `verify` cannot depend on. The vendor-contract test is what stops the schema drifting from the binary. +- **`scope` gains a real enum**, including the hierarchical forms, the v3.17.0+ inline scopes, and the v3.18.0+ `meta` scopes — and accepts the `~` negation and `&` chaining syntax that the recipe never documents and that a naive enum would reject. +- **The `create-vale-rule` recipe closes the gaps that produce silent rules**: the measured scope table, `nonword` for punctuation tokens, how to scope a rule _out_ (the recipe only explains scoping in), fixture design for a subject that normally appears in code, and the `limit`/`vocab` fields it omits. +- **No breaking change.** Every rule that verifies today continues to verify; the schema only rejects rules the engine was already going to ignore or fail on. + +## Capabilities + +### New Capabilities + +None. This tightens two existing capabilities rather than introducing a surface. + +### Modified Capabilities + +- `cli-rule-validation`: the `verify` component table currently says a Vale rule is checked "against Vale's own validation." Measured, that covers `level` and not `extends` or `scope`. The requirement changes to name schema validation as its own layer for `vale`, matching how `sg` is already checked against the ast-grep schema. +- `cli-agent-authoring`: the Vale authoring recipe requirement gains the scope, token, and fixture guidance whose absence is what produces a green-but-inert rule. + +## Impact + +- `packages/cli/src/rules/vale/verify.ts` — today entirely fixture behavior (bucket coverage, which fixtures fired); gains a schema layer. +- A new generated-or-authored schema artifact alongside `packages/cli/src/generated/ast-grep-rule-schema.json`, plus whatever pins it to `VALE_VERSION`. +- `packages/cli/test/vale-vendor-contract.test.ts` — extended to hold the schema's claims against the pinned binary. +- `packages/cli/src/agent/create-vale-rule.txt` — the recipe text. +- Closes #171 and #170; #167 is the worked case that motivated both. + +## Delivery shape + +**Stacked, merging forward.** Two units, each independently safe in production: + +1. **The recipe gaps.** Text only, no behavior change, and it is the half that helps an author today. It can land alone. +2. **The schema and its `verify` layer.** Code, tests, and the vendor-contract additions. + +Landing 1 alone leaves `check`, `verify`, and the test suite exactly as they are, so the stack merges forward rather than down. The changeset goes on the bottom branch and grows as the second unit lands. diff --git a/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md b/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md new file mode 100644 index 00000000..9818a953 --- /dev/null +++ b/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md @@ -0,0 +1,79 @@ +## MODIFIED Requirements + +### Requirement: The Vale authoring recipe covers rule, scope, and fixtures + +The `create-vale-rule` recipe SHALL instruct the agent to produce three artifacts, and SHALL state that a rule is incomplete without all three: + +1. A Vale style file at `.taskless/rules/vale//.yml`. +2. That rule's own `.taskless/rules/vale//.vale.ini`, declaring the matchers that scope it and enabling it as `. = YES`. +3. `pass/` and `fail/` fixture documents under `.taskless/rules/vale//.tests/`. + +The recipe SHALL state that scope is declared in the rule's own config, that no shared file is edited, and that the project-wide config is assembled rather than authored. + +It SHALL direct the agent to check its work by running `verify` and then `test` against the rule's directory path. + +The previous version of this requirement taught the agent to add a matcher to a single project-wide `.vale.ini`. Executing that recipe against sandboxed agents found every one of its silent failures in that step and nowhere else — an assignment above the first matcher, a glob that missed the fixture extension, three names that had to agree with nothing reporting when they didn't. The layout change removes the step rather than documenting it further. + +The recipe SHALL additionally carry the guidance below. Each item is a failure observed while authoring rules against this repository, and each produced a rule that passed `verify` and `test` while reporting nothing: + +- **What each `scope` reaches**, as measured: `text` sees prose, `code` sees inline code spans, `[code, text]` sees both, and `raw` sees prose, inline code, and fenced blocks. `raw` subsumes the other two. +- **That `scope` is per-rule.** Vale assembles one config per run, which invites the assumption that scopes interact. They do not. +- **That a rule scoped to `raw` cannot be suppressed** by Vale's `` directive, because it reads the unparsed document. A rule about a command needs `raw`, so it trades away per-case exemption. +- **That a token made only of punctuation needs `nonword: true`**, because Vale wraps every token in word boundaries. +- **How to scope a rule out**, not only in: a second matcher assigning `. = NO`. +- **That a bare word finds senses you did not mean**, and that narrowing to a collocation is checked by writing the `pass/` fixture from the literal sense first. +- **That fixture design follows the rule's subject**: when the subject normally appears in code, the `fail/` fixture SHALL contain it inline, fenced, and in prose. +- **The `limit` and `vocab` common fields**, which the recipe's field table omits. + +#### Scenario: Authoring produces all three artifacts + +- **WHEN** the agent follows `create-vale-rule` +- **THEN** it writes the style file, the rule's own config with a scoping matcher, and both fixture buckets + +#### Scenario: No shared file is edited + +- **WHEN** the agent scopes a rule +- **THEN** it writes matchers into that rule's own config +- **AND** it SHALL NOT be directed to edit a project-wide Vale config + +#### Scenario: The recipe names the commands that check the work + +- **WHEN** the agent has written the three artifacts +- **THEN** the recipe SHALL direct it to run `verify` and `test` against the rule's path + +#### Scenario: An unscoped rule is not silently accepted + +- **WHEN** the agent writes a style file without a matcher enabling it in the rule's own config +- **THEN** the recipe SHALL identify this as incomplete +- **AND** `verify` SHALL report it + +#### Scenario: A rule about a command reaches fenced blocks + +- **WHEN** the agent authors a rule whose subject is a command, flag, or package name +- **THEN** the recipe SHALL direct it to a scope that reaches fenced blocks +- **AND** the `fail/` fixture SHALL carry the subject inline, fenced, and in prose + +#### Scenario: A punctuation token is not left unable to match + +- **WHEN** the agent authors a rule whose token contains no word characters +- **THEN** the recipe SHALL direct it to set `nonword: true` + +## ADDED Requirements + +### Requirement: An authoring recipe states the failure a rule cannot report itself + +An authoring recipe SHALL document the ways a rule of its engine can be well formed, enabled, green on its fixtures, and still report nothing. + +A malformed rule is caught by `verify` and needs no recipe. A rule that is merely _wrong_ is caught by nothing, ships green, and is discovered only when someone notices it has never fired — so the recipe is the only place that failure can be prevented. + +#### Scenario: The recipe names the silent failures for its engine + +- **WHEN** an authoring recipe is written or revised +- **THEN** it SHALL name the failures that pass every local gate for that engine +- **AND** each SHALL be stated as an observed behavior of the pinned engine rather than as a caution in principle + +#### Scenario: Passing fixtures are not presented as proof of reach + +- **WHEN** the recipe directs the agent to run `test` +- **THEN** it SHALL state that fixtures run under an isolating config +- **AND** that a clean `check` SHALL be confirmed against a real file before the rule is believed to be working diff --git a/openspec/changes/vale-authoring-polish/specs/cli-rule-validation/spec.md b/openspec/changes/vale-authoring-polish/specs/cli-rule-validation/spec.md new file mode 100644 index 00000000..406264fe --- /dev/null +++ b/openspec/changes/vale-authoring-polish/specs/cli-rule-validation/spec.md @@ -0,0 +1,66 @@ +## MODIFIED Requirements + +### Requirement: Verify checks a rule's required components + +`verify` SHALL check that a rule has the components its engine requires and that they are well formed, and SHALL NOT require fixtures or test cases to exist. + +The two commands split because they have different preconditions. An agent part-way through authoring has a rule and no fixtures yet, and needs to know the rule itself is valid before it can write a meaningful test for it. + +Per engine, `verify` SHALL check: + +| Engine | Components | +| --------- | ---------------------------------------------------------------------------------------------------- | +| `sg` | `.yml` against the ast-grep schema and the Taskless required fields | +| `vale` | `.yml` against the Vale rule schema and the Taskless required fields, and the rule's `.vale.ini` | +| `runtime` | `check.ts` present, and at least one capture rule under `captures/` | + +The `vale` row previously read "against Vale's own validation." Measured against the pinned 3.18.0 binary, that covers less than it claims: `level: bananas` is reported, while `extends: nonsense` and `scope: fenced` both verify clean and produce a rule that matches nothing. Vale validates a rule when it _runs_ one, and it runs one field at a time — so a name it does not recognize is not an error, it is a check that never fires. Schema validation is therefore its own layer for `vale`, as it already is for `sg`. + +#### Scenario: A rule with no fixtures still verifies + +- **WHEN** `verify` runs against a rule whose fixture buckets are empty or absent +- **THEN** it SHALL report on the rule's components only +- **AND** the absence of fixtures SHALL NOT be a verify failure + +#### Scenario: A malformed rule reports its own error + +- **WHEN** a Vale style declares a `level` outside `suggestion`/`warning`/`error` +- **THEN** `verify` SHALL report that error, naming the field + +#### Scenario: An unrecognized extension point is rejected + +- **WHEN** a Vale style declares an `extends` that is not one of Vale's check types +- **THEN** `verify` SHALL report it, naming the field and the accepted values +- **AND** it SHALL NOT report the rule as valid + +#### Scenario: An unrecognized scope is rejected + +- **WHEN** a Vale style declares a `scope` that is not one of Vale's scope values +- **THEN** `verify` SHALL report it, naming the field +- **AND** a scope using the `~` negation or `&` chaining syntax over recognized values SHALL be accepted + +#### Scenario: A field belonging to another check type is rejected + +- **WHEN** a Vale style declares a field its `extends` does not accept, such as `tokens` on an `occurrence` check +- **THEN** `verify` SHALL report it before Vale is invoked + +The failure it prevents is not a local one: Vale reports this as `E201: has invalid keys` and reads one assembled config per run, so a single rule with a stray field suppresses every other Vale rule's findings. + +## ADDED Requirements + +### Requirement: The Vale rule schema is pinned to the vendored binary + +The Vale rule schema SHALL be authored in this repository and pinned to `VALE_VERSION`, and a vendor-contract test SHALL hold its claims against the vendored binary. + +Vale publishes no JSON Schema for its check types. The machine-readable field knowledge exists only behind its hosted MCP server, which is a paid product and unavailable to `verify`. The schema is therefore a transcription, and a transcription drifts — so the binary, not the documentation, SHALL be the authority for what the schema asserts. + +#### Scenario: A Vale upgrade that invalidates the schema fails loudly + +- **WHEN** `VALE_VERSION` is raised to a version whose accepted check types or scopes differ from the schema +- **THEN** the vendor-contract test SHALL fail +- **AND** the failure SHALL name the field whose accepted values changed + +#### Scenario: The schema is not derived from documentation alone + +- **WHEN** a value is added to the schema's `extends` or `scope` enumerations +- **THEN** it SHALL be one the vendored binary was measured accepting diff --git a/openspec/changes/vale-authoring-polish/tasks.md b/openspec/changes/vale-authoring-polish/tasks.md new file mode 100644 index 00000000..a168ef31 --- /dev/null +++ b/openspec/changes/vale-authoring-polish/tasks.md @@ -0,0 +1,65 @@ +## 1. Measure the binary before writing anything down + +- [ ] 1.1 Enumerate the check types the vendored Vale accepts, and resolve the docs' eleven against the MCP guide's twelve. Record the measured list and how it was obtained. +- [ ] 1.2 Enumerate the accepted `scope` operands, including the hierarchical forms, the v3.17.0+ inline scopes, and the v3.18.0+ `meta` scopes. Confirm `~` negation and `&` chaining parse. +- [ ] 1.3 For each check type, measure which fields it accepts and which it rejects with `E201`. This is the per-check table the schema encodes. +- [ ] 1.4 Confirm the failure modes the design asserts still hold on the pinned binary: `extends: nonsense` and `scope: fenced` verify clean and match nothing; a foreign field throws `E201` and suppresses every other Vale rule's findings. + +## 2. Recipe gaps (unit 1, lands alone) + +- [ ] 2.1 Add the measured scope table to `create-vale-rule`, and state that `raw` subsumes `code` and `text`. +- [ ] 2.2 State that `scope` is per-rule and that rules do not interact, despite the single assembled config. +- [ ] 2.3 State that a `raw`-scoped rule cannot be suppressed by ``, and that a rule about a command needs `raw`. +- [ ] 2.4 Add `nonword: true` for punctuation-only tokens, with the em-dash rule as the worked case. +- [ ] 2.5 Document scoping a rule _out_ with a second matcher assigning `NO`. +- [ ] 2.6 Add the collocation guidance: narrow a banned word to a collocation, and write the `pass/` fixture from the literal sense first. +- [ ] 2.7 Add the fixture rule for a subject that appears in code: `fail/` must carry it inline, fenced, and in prose. +- [ ] 2.8 Add `limit` and `vocab` to the common-fields table, and note that Vale requires `.yml` rather than `.yaml`. +- [ ] 2.9 State that fixtures run under an isolating config, so passing tests do not prove the matcher reaches any real file. +- [ ] 2.10 Update the recipe's cross-reference tests if the added sections change what they assert. +- [ ] 2.11 Write the changeset on this branch, so the stack inherits it. + +## 3. The schema (unit 2) + +- [ ] 3.1 Author the schema from the measurements in group 1, alongside `packages/cli/src/generated/ast-grep-rule-schema.json`. Pin it to `VALE_VERSION`. +- [ ] 3.2 Model the common header fields: `extends`, `message`, `level`, `scope`, `link`, `limit`, `vocab`. +- [ ] 3.3 Model `scope` as a grammar over an enum of operands, accepting a string, a list, `~`, and `&`. +- [ ] 3.4 Model the per-check field tables, so a field belonging to another check type is rejected before `E201` can suppress the engine. +- [ ] 3.5 Decide and record how strictly to treat a value whose status the measurements left unclear. The design's rule is to accept it. + +## 4. Wire it into verify + +- [ ] 4.1 Add the schema layer to the Vale verify path, reporting through the existing `LayerResult` and `VerifyResult` shapes so both engines fail the same way. +- [ ] 4.2 Ensure the error names the field and the accepted values, rather than reporting a raw schema path. +- [ ] 4.3 Confirm the layer runs before Vale is invoked, so a rule that would throw `E201` never reaches the binary. +- [ ] 4.4 Confirm `test` still runs `verify` first and stops on its failure. + +## 5. The corpus, and the differential test that makes the schema true + +Build this before the schema is finalized — the schema is derived from it. This is the largest group and the one that will be under-built if rushed. + +- [ ] 5.1 Design the corpus entry shape: a minimal rule YAML, the fixture that acts as its positive control, and the measured verdict. Keep it a declarative table so a Vale upgrade is a re-run rather than a re-authoring. +- [ ] 5.2 Establish the three-outcome verdict, since an exit code cannot express it: **fired** (accepted), **did not fire** (the binary ignored the construct), **`E201`** (rejected outright). An unrecognized `extends` or `scope` parses clean, so "did not fire" is the signal that the construct is invalid. +- [ ] 5.3 Write a positive control per entry. It depends on what the rule matches — a `scope: heading` entry needs a heading, `scope: table.cell` needs a table — so it cannot be generated from the check type. This is the bulk of the work. +- [ ] 5.4 Guard against a vacuous entry: assert each control fires for at least one known-valid variant, so a control that can never fire is a test failure rather than a quiet pass. +- [ ] 5.5 Cover every check type from 1.1, every scope operand from 1.2, and the per-check field tables from 1.3, including the `~` and `&` forms. +- [ ] 5.6 Write the differential test: for every entry, assert `schemaAccepts === binaryAccepts`. Report both directions distinctly — schema-too-lax is the gap being closed, schema-too-strict blocks valid work and is the worse failure. +- [ ] 5.7 Wire the corpus into `packages/cli/test/vale-vendor-contract.test.ts`, or a sibling beside it, following the existing convention of invoking the vendored binary directly rather than through `runVale`. +- [ ] 5.8 Add the version-bump case: raising `VALE_VERSION` past a change in accepted check types or scopes fails a test that names the field. +- [ ] 5.9 Verify the suite is non-vacuous by reverting the schema and watching exactly the expected entries fail. + +## 6. Regression coverage + +- [ ] 6.1 Test that `extends: nonsense` fails `verify`, naming the field and the accepted values. +- [ ] 6.2 Test that an unrecognized `scope` fails `verify`, and that `~` and `&` over recognized operands pass. +- [ ] 6.3 Test that a foreign field for the declared check type fails `verify`. +- [ ] 6.4 Test that every rule under `.taskless/rules/vale/` still verifies, so no valid rule regressed. +- [ ] 6.5 Test that a rule with no fixtures still verifies, preserving the existing requirement. + +## 7. Land it + +- [ ] 7.1 `pnpm typecheck`, `pnpm lint`, `pnpm test` pass. +- [ ] 7.2 `pnpm build`, then author a deliberately-broken rule and confirm the real CLI reports what the specs require. Record the actual output. +- [ ] 7.3 Extend the changeset on the bottom branch with unit 2's scope. +- [ ] 7.4 Open the stack as two PRs merging forward, the recipe unit first. +- [ ] 7.5 Archive the change on the tip PR. From cc5afa855779d217b54214d41fefb324015aaffe Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 21:43:25 -0700 Subject: [PATCH 02/17] docs(cli): close the create-vale-rule gaps that produce a silent rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every addition is a behaviour measured against the pinned Vale 3.18.0 binary, not a caution in principle: the scope vocabulary and what each value reaches, `raw` subsuming `code` and `text`, `~`/`&` and the silent no-op a negation over an unknown operand becomes, `scope` being per-rule, a `raw` rule being unsuppressable by Vale's directive, `nonword: true` for a punctuation-only token, scoping a rule out with a second matcher, collocation narrowing, three-way fixtures for a subject that lives in code, `limit`/`vocab`, `.yml` over `.yaml`, and that a green `test` proves nothing about the matcher's reach. Check types are now enumerated from the binary's own rejection message. There are twelve, not eleven — the docs fold `readability` into `metric`. The per-check field tables are measured the same way, correcting two published claims. Refs #170 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .changeset/vale-authoring-polish.md | 38 +++ .../changes/vale-authoring-polish/design.md | 86 +++++- .../changes/vale-authoring-polish/tasks.md | 30 +-- packages/cli/src/agent/create-vale-rule.txt | 247 ++++++++++++++++-- 4 files changed, 362 insertions(+), 39 deletions(-) create mode 100644 .changeset/vale-authoring-polish.md diff --git a/.changeset/vale-authoring-polish.md b/.changeset/vale-authoring-polish.md new file mode 100644 index 00000000..d09e76b6 --- /dev/null +++ b/.changeset/vale-authoring-polish.md @@ -0,0 +1,38 @@ +--- +"@taskless/cli": patch +--- + +Close the gaps in `create-vale-rule` that produce a rule which is green +everywhere and reports nothing. + +A malformed Vale rule fails loudly. A Vale rule that is merely _wrong_ passes +`verify`, passes `test`, and never fires — and nobody re-checks a green rule. +The recipe now documents each of those failures as an observed behavior of the +pinned Vale binary rather than as a caution in principle: + +- **The measured `scope` vocabulary**, with what each value actually reaches. + `raw` subsumes `code` and `text`; `~` negation and `&` chaining are accepted; + and a negation over a scope Vale does not know (`~fenced`) is a silent no-op + that removes the exclusion you wrote the rule for. +- **`scope` is per-rule.** Taskless assembles one config per run, which invites + the assumption that scopes interact. They do not. +- **A `raw`-scoped rule cannot be suppressed** by ``, + because it reads the unparsed document — so a rule about a shell command needs + `raw` and trades away per-case exemption. +- **A punctuation-only token needs `nonword: true`**, because Vale wraps every + token in word boundaries and an em dash has no word character on either side. +- **How to scope a rule _out_**, with a second matcher assigning `NO`. +- **Collocation guidance** for a banned word, checked by writing the `pass/` + fixture from the literal sense first. +- **Fixture design for a subject that appears in code**: `fail/` must carry it + inline, fenced, and in prose. +- **`limit` and `vocab`** in the common-fields table, and that Vale loads only + `.yml` — a style file renamed to `.yaml` is silently not loaded at all. +- **Fixtures run under an isolating config**, so a green `test` is not evidence + the rule's matcher glob reaches any real file. + +Vale's check types are now enumerated from the binary rather than the docs. +There are **twelve**, not eleven: the docs fold `readability` into `metric`. +The per-check field tables are measured the same way, which corrects two +published claims — `capitalization` rejects `prefixes`/`suffixes`, and +`occurrence` rejects `exceptions`. diff --git a/openspec/changes/vale-authoring-polish/design.md b/openspec/changes/vale-authoring-polish/design.md index 98b39aa4..3931bd7e 100644 --- a/openspec/changes/vale-authoring-polish/design.md +++ b/openspec/changes/vale-authoring-polish/design.md @@ -93,8 +93,92 @@ No migration. The schema layer is additive, and it only rejects rules that the e The stack lands forward in two units, per the proposal: the recipe text first, then the schema and its `verify` layer. If the second unit needs to be reverted, the first stands on its own. +## Measured against Vale 3.18.0 + +Everything below was obtained by authoring a rule and running the vendored +binary directly, in a temporary project with `BasedOnStyles =` so no bundled +style could contribute a finding. Where a verdict is "did not fire", a +known-valid control was run against the same document first. + +### Twelve check types, and the binary says so itself + +An `extends` outside the set is **not** silently ignored. Vale rejects the file, +exits 2, and names the whole set: + +``` +'extends' key must be one of [capitalization conditional consistency existence +occurrence repetition substitution readability spelling sequence metric script]. +``` + +That resolves eleven-versus-twelve without a judgement call: **twelve**. The +docs' eleven omits `readability`, which they fold into `metric`; the binary +treats them as separate checks with disjoint fields (`metrics`/`grade` versus +`formula`/`condition`), and each rejects the other's. + +### `scope` is a grammar, and the operand vocabulary is not the docs' + +Measured firing: `text`, `code`, `raw`, `heading`, `heading.h1`–`h6`, +`paragraph`, `sentence`, `list`, `blockquote`, `link`, `alt`, `summary`, +`strong`, `emphasis`, `table`, `table.header`, `table.cell`, `table.caption`, +`comment`, `comment.line`, `comment.block`, `frontmatter`, `frontmatter.`, +`text.class.`. + +Reach, on one document holding the token in prose, in an inline span, and in a +fenced block: `text` → 1, `code` → 1, `[code, text]` → 2, `raw` → 3. `raw` +subsumes the other two. + +`~` negation, `&` chaining, and the list form all parse and behave. **A negation +over an unrecognized operand is a silent no-op** — `~fenced` and `text & ~fenced` +both fire on everything, so a typo inside a `~` removes the exclusion rather than +narrowing anything. + +Two corrections to this document's own earlier draft: + +- **There is no `meta` scope.** The v3.18.0 addition is `frontmatter` and + `frontmatter.`, which is what the binary carries (`text.frontmatter.`) + and what fires. `meta` and `meta.class.` never fired in Markdown, + MDX, or HTML. +- **`figure.caption` is in the binary's vocabulary** (`text.figure.caption`) but + no control could be made to fire for it in a format this build can lint + without an external converter — `
` content did not reach even a + `scope: text` control in HTML, and AsciiDoc and reStructuredText need + `asciidoctor`/`rst2html`, which Taskless excludes. It is accepted by the + schema on the accept-when-unclear rule and carried in the corpus as an + explicitly unproven operand rather than as a passing row. + +### Per-check fields, and two checks that validate nothing + +Every check accepts `extends`, `message`, `level`, `scope`, `link`, `limit`, +`action`, `description`, `name`. `vocab` is accepted by `existence`, +`substitution`, `capitalization`, `conditional`, `repetition` and **rejected** +by `occurrence`, `metric`, `readability`, `script`, `sequence`. + +The per-check additions are in the recipe's table. Two published claims are +wrong against the binary: `capitalization` rejects `prefixes`, `suffixes` and +`ignorecase`; `occurrence` rejects `exceptions`. + +**`consistency` and `spelling` accept any key at all.** `bananafield: true` on +either loads without complaint and is ignored — they do not use the strict +decode the other ten do. The schema cannot be strict about their fields without +rejecting rules the binary accepts, so it is not. + +### The failure modes, re-measured + +| Rule defect | Binary | +| ------------------------ | ------------------------------------------------ | +| `extends: nonsense` | rejected, exit 2, **every** Vale rule unreported | +| foreign field | `E201`, exit 2, **every** Vale rule unreported | +| `scope: fenced` | loads, runs, matches nothing — no error anywhere | +| style file named `.yaml` | not loaded at all — no error, no warning | + +**One row of the table in Context is wrong and is corrected here.** It claimed +`extends: nonsense` "verifies clean and simply matches nothing". Against 3.18.0 +it does not: it is the same engine-wide suppression as `E201`. The genuinely +silent case is `scope`, alone — which makes `scope` the highest-value field in +the schema for exactly the reason the decision above gives, and makes the other +two a blast-radius argument rather than a silence argument. + ## Open Questions -- **Eleven check types or twelve?** To be resolved by enumeration against the vendored binary during implementation. - **How much of the per-check field table to encode.** The full table makes `E201` unreachable; a partial one still leaves the engine-wide suppression possible for the fields it omits. Recommend the full table, since `E201` is the failure with the widest blast radius, but the cost is that every check type must be transcribed and measured rather than just the common header. - **Whether `verify` should also report each matcher's selected-file count.** Out of scope here, but adjacent: a matcher glob that reaches nothing produces a rule that is well formed, enabled, green, and inert — the one silent failure this change does not close. diff --git a/openspec/changes/vale-authoring-polish/tasks.md b/openspec/changes/vale-authoring-polish/tasks.md index a168ef31..659695a9 100644 --- a/openspec/changes/vale-authoring-polish/tasks.md +++ b/openspec/changes/vale-authoring-polish/tasks.md @@ -1,23 +1,23 @@ ## 1. Measure the binary before writing anything down -- [ ] 1.1 Enumerate the check types the vendored Vale accepts, and resolve the docs' eleven against the MCP guide's twelve. Record the measured list and how it was obtained. -- [ ] 1.2 Enumerate the accepted `scope` operands, including the hierarchical forms, the v3.17.0+ inline scopes, and the v3.18.0+ `meta` scopes. Confirm `~` negation and `&` chaining parse. -- [ ] 1.3 For each check type, measure which fields it accepts and which it rejects with `E201`. This is the per-check table the schema encodes. -- [ ] 1.4 Confirm the failure modes the design asserts still hold on the pinned binary: `extends: nonsense` and `scope: fenced` verify clean and match nothing; a foreign field throws `E201` and suppresses every other Vale rule's findings. +- [x] 1.1 Enumerate the check types the vendored Vale accepts, and resolve the docs' eleven against the MCP guide's twelve. Record the measured list and how it was obtained. +- [x] 1.2 Enumerate the accepted `scope` operands, including the hierarchical forms, the v3.17.0+ inline scopes, and the v3.18.0+ `meta` scopes. Confirm `~` negation and `&` chaining parse. **There is no `meta` scope**: the v3.18.0 addition is `frontmatter`/`frontmatter.`, which is what fires. Negation over an unrecognized operand is a silent no-op. See design.md → Measured against Vale 3.18.0. +- [x] 1.3 For each check type, measure which fields it accepts and which it rejects with `E201`. This is the per-check table the schema encodes. +- [x] 1.4 Confirm the failure modes the design asserts still hold on the pinned binary: `extends: nonsense` and `scope: fenced` verify clean and match nothing; a foreign field throws `E201` and suppresses every other Vale rule's findings. **One half did not hold.** `extends: nonsense` is rejected outright on 3.18.0 — exit 2, whole run suppressed, same blast radius as `E201`. `scope: fenced` is the only genuinely silent case. Corrected in design.md. ## 2. Recipe gaps (unit 1, lands alone) -- [ ] 2.1 Add the measured scope table to `create-vale-rule`, and state that `raw` subsumes `code` and `text`. -- [ ] 2.2 State that `scope` is per-rule and that rules do not interact, despite the single assembled config. -- [ ] 2.3 State that a `raw`-scoped rule cannot be suppressed by ``, and that a rule about a command needs `raw`. -- [ ] 2.4 Add `nonword: true` for punctuation-only tokens, with the em-dash rule as the worked case. -- [ ] 2.5 Document scoping a rule _out_ with a second matcher assigning `NO`. -- [ ] 2.6 Add the collocation guidance: narrow a banned word to a collocation, and write the `pass/` fixture from the literal sense first. -- [ ] 2.7 Add the fixture rule for a subject that appears in code: `fail/` must carry it inline, fenced, and in prose. -- [ ] 2.8 Add `limit` and `vocab` to the common-fields table, and note that Vale requires `.yml` rather than `.yaml`. -- [ ] 2.9 State that fixtures run under an isolating config, so passing tests do not prove the matcher reaches any real file. -- [ ] 2.10 Update the recipe's cross-reference tests if the added sections change what they assert. -- [ ] 2.11 Write the changeset on this branch, so the stack inherits it. +- [x] 2.1 Add the measured scope table to `create-vale-rule`, and state that `raw` subsumes `code` and `text`. +- [x] 2.2 State that `scope` is per-rule and that rules do not interact, despite the single assembled config. +- [x] 2.3 State that a `raw`-scoped rule cannot be suppressed by ``, and that a rule about a command needs `raw`. +- [x] 2.4 Add `nonword: true` for punctuation-only tokens, with the em-dash rule as the worked case. +- [x] 2.5 Document scoping a rule _out_ with a second matcher assigning `NO`. +- [x] 2.6 Add the collocation guidance: narrow a banned word to a collocation, and write the `pass/` fixture from the literal sense first. +- [x] 2.7 Add the fixture rule for a subject that appears in code: `fail/` must carry it inline, fenced, and in prose. +- [x] 2.8 Add `limit` and `vocab` to the common-fields table, and note that Vale requires `.yml` rather than `.yaml`. +- [x] 2.9 State that fixtures run under an isolating config, so passing tests do not prove the matcher reaches any real file. +- [x] 2.10 Update the recipe's cross-reference tests if the added sections change what they assert. No change needed: `recipe-cross-references.test.ts` asserts the rendered format lists, which the new sections do not touch, and the full suite is green. +- [x] 2.11 Write the changeset on this branch, so the stack inherits it. ## 3. The schema (unit 2) diff --git a/packages/cli/src/agent/create-vale-rule.txt b/packages/cli/src/agent/create-vale-rule.txt index 11440a21..17f386dc 100644 --- a/packages/cli/src/agent/create-vale-rule.txt +++ b/packages/cli/src/agent/create-vale-rule.txt @@ -59,8 +59,19 @@ it. 1. **Say what the rule reads, then pick an extension point.** Write one sentence: "this fires when a document contains ___." Vale rules are - built by extending one of its eleven checks, and the sentence tells - you which: + built by extending one of its twelve checks, and the sentence tells + you which. Twelve is measured, not counted off the docs: give Vale + v%(VALE_VERSION)s an `extends` it does not know and it names the whole + set back at you — + + ``` + 'extends' key must be one of [capitalization conditional consistency + existence occurrence repetition substitution readability spelling + sequence metric script]. + ``` + + The docs enumerate eleven, folding `readability` into `metric`. They + are separate checks with separate fields. | If the rule is about… | extends | |-----------------------------------------------------------------------------------------|------------------| @@ -71,7 +82,8 @@ it. | a word repeated back to back | `repetition` | | picking one of two acceptable spellings, consistently | `consistency` | | "if X appears, Y must also appear" | `conditional` | -| readability or length thresholds | `metric` | +| a document-level length or ratio threshold | `metric` | +| a readability grade, against a named formula | `readability` | | a misspelling, against a dictionary | `spelling` | | phrases that must appear in a fixed order | `sequence` | | anything the above cannot express (Tengo script) | `script` | @@ -85,9 +97,13 @@ it. `match: GitHub` reports `We host on Github and it is fine. should be GitHub`. - For the two this recipe does not describe, read - https://docs.vale.sh/styles before inventing something — Vale has no - facility for a rule that does not extend one of these eleven. + For the five this recipe has no worked example of — `metric`, + `readability`, `spelling`, `sequence`, `script` — read + https://docs.vale.sh/styles before inventing something. Vale has no + facility for a rule that does not extend one of these twelve, and an + `extends` outside the set is not a rule that misbehaves: Vale exits 2 + and **every** Vale rule in the project goes unreported for that run. + `verify` catches it before Vale is invoked. **Nine of them have a worked rule at the end of this recipe**, each with the near-miss that fails and why. Read the one closest to your @@ -102,38 +118,136 @@ it. only** (`izeise`, not `ize-ise`). That check compiles the id into its pattern as a regex group name, and a hyphen there fails the entire Vale run. `verify` catches it. Kebab-case is correct for the other - ten. + eleven. Every rule carries: | Field | Required | Notes | |-----------|----------|-----------------------------------------------| -| `extends` | yes | one of the eleven above | +| `extends` | yes | one of the twelve above | | `message` | yes | shown to the user; see the `%%s` table below | | `level` | no | `suggestion` (default), `warning`, or `error` | | `scope` | no | narrow to part of a document — see below | | `link` | no | a URL the reader can follow for the reasoning | +| `limit` | no | cap findings from this rule per scope | +| `vocab` | no | `false` opts the rule out of `Vocab` filtering | + + **The file extension must be `.yml`.** Measured: rename a working + style file to `.yaml` and Vale loads nothing — no error, no warning, + zero findings, and `. = YES` still parses. It is + indistinguishable from a rule whose pattern never matched. **`scope` decides where the rule looks**, so getting it wrong is a - silent over- or under-fire rather than an error. The useful values for - markdown: `heading` (and `heading.h1`…`heading.h6`), `paragraph`, - `sentence`, `list`, `blockquote`, `link`, `code`, `alt`, `summary`, - `table.header`, `table.cell`, `table.caption`, `figure.caption`, - `strong`, `emphasis`, `raw`. Omit it and the rule sees all prose. + silent under-fire rather than an error. An unrecognized scope is not + rejected by Vale: `scope: fenced` loads, runs, and matches nothing. + It is the worst of the three failures on this page, because unlike a + bad `extends` or a foreign field it does not even take the run down to + tell you — the rule is simply inert, forever. `verify` rejects it. + + Every value below was measured against Vale v%(VALE_VERSION)s by + authoring a rule with that scope and a document the rule had to flag. + "Fires" means the finding appeared; a scope that never fired is not on + this list. + +| `scope` | reaches | +|------------------------|-----------------------------------------------------------| +| *(omitted)* | everything the format exposes as prose | +| `text` | prose only — not inline code, not fenced blocks | +| `code` | inline code spans only | +| `raw` | the unparsed document: prose, inline code, fenced blocks | +| `heading` | every heading | +| `heading.h1`…`h6` | headings of that level | +| `paragraph` | one paragraph at a time | +| `sentence` | one sentence at a time | +| `list` | list items | +| `blockquote` | quoted blocks | +| `link` | link text, not the URL | +| `alt` | image alt text | +| `summary` | `` of a disclosure | +| `strong`, `emphasis` | bold and italic runs | +| `table` | any part of a table | +| `table.header` | header cells | +| `table.cell` | body cells | +| `table.caption` | a table's caption | +| `figure.caption` | a figure's caption | +| `frontmatter` | every YAML front-matter value | +| `frontmatter.` | one front-matter key's value | +| `text.class.` | HTML elements carrying that class | +| `comment` | every comment, in a comment-tier format | +| `comment.line` | `//`-style comments | +| `comment.block` | `/* … */`-style comments | + + **`raw` subsumes `code` and `text`.** Measured on one document holding + the token in prose, in an inline span, and in a fenced block: `text` + found one, `code` found one, `[code, text]` found two, `raw` found all + three. If you want prose and inline code but not fenced blocks, write + the list — `raw` is not "a bit wider", it is everything. + + **`scope` also takes `~` and `&`.** `~code` is everything but inline + code; `text & ~code` chains two operands; a list (`[code, text]`) is a + union. All three parse and behave. + + **A negation over a scope Vale does not know is a silent no-op.** + Measured: `~banana` and `text & ~banana` both fire on everything, + because there is no such scope to subtract. A typo inside a `~` does + not narrow the rule and does not widen it visibly — it removes the + exclusion you wrote the rule for. `verify` checks the operands inside + `~` and `&` for exactly this reason. + + **`scope` is per-rule, and rules do not interact.** Taskless assembles + every rule's matchers into one config for the run, which invites the + assumption that one rule's `scope` narrows another's, or that two + rules over the same file compete. They do not: each check carries its + own scope and is evaluated independently. If a rule is over-firing, + the cause is in that rule's own `scope` and glob, never in a + neighbour's. + + **A `raw`-scoped rule cannot be turned off in a document.** Vale's + `` directive is applied to the *parsed* + document, and `raw` reads the unparsed one. Measured: a `text`-scoped + rule is silenced by the directive; the same rule at `raw` fires + through it. So a rule about a shell command, a flag, or a package name + needs `raw` — commands live in fenced blocks, which nothing else + reaches — and takes that trade: it can no longer be exempted case by + case, only removed. **Then the fields the extension point adds** — this is where the rule actually lives, and each check reads only its own: -| extends | its fields | -|------------------|-------------------------------------------------------------------------------------| -| `existence` | `tokens` (a list) or `raw`; `ignorecase`, `nonword`, `exceptions`, `append` | -| `substitution` | `swap` (a map of observed → expected); `ignorecase`, `nonword`, `exceptions` | -| `capitalization` | `match`; `exceptions`, `style` (with `$title`), `threshold`, `indicators`, `prefix` | -| `occurrence` | `token`, `max` and/or `min` | -| `repetition` | `tokens`; `alpha`, `ignorecase` | -| `consistency` | `either` (a map of the two acceptable forms) | -| `conditional` | `first`, `second`; `exceptions` | -| `metric` | `formula`, `condition` | +| extends | its fields | +|------------------|------------------------------------------------------------------------------| +| `existence` | `tokens` (a list) or `raw`; `ignorecase`, `nonword`, `exceptions`, `append` | +| `substitution` | `swap` (a map of observed → expected); `ignorecase`, `nonword`, `exceptions`, `capitalize`, `pos` | +| `capitalization` | `match`; `style` (with `$title`), `exceptions`, `threshold`, `indicators` | +| `occurrence` | `token`, `max` and/or `min`; `ignorecase` | +| `repetition` | `tokens`; `alpha`, `ignorecase`, `exceptions`, `max` | +| `consistency` | `either` (a map of the two acceptable forms); `ignorecase`, `nonword` | +| `conditional` | `first`, `second`; `exceptions`, `ignorecase` | +| `metric` | `formula`, `condition` | +| `readability` | `metrics` (a list of formula names), `grade` | +| `spelling` | `aff`, `dic`, `custom`, `filters`, `ignore`, `threshold` | +| `sequence` | `tokens` (each a `pattern`/`tag` map); `ignorecase` | +| `script` | `script` (Tengo source) | + + The list above is measured, not transcribed: every entry was added to + a minimal rule of that check and the run watched for `E201`. Two + corrections fall out of it, both against the published docs — + `capitalization` rejects `prefixes`/`suffixes` and `ignorecase`, and + `occurrence` rejects `exceptions` and `vocab`. + + **A field from the wrong check is the loudest failure Vale has.** + `tokens` on an `occurrence` check gives + `E201 … has invalid keys: 'tokens'`, exit 2, and — because Vale reads + one assembled config per run — **no** Vale rule in the project reports + anything. `verify` rejects the rule before Vale is invoked so this + cannot reach `check`. + + **Two checks are exempt, and that is not a licence.** Measured, + `consistency` and `spelling` accept any key at all: `bananafield: + true` on either loads without complaint and is ignored. A misspelled + field on those two is a silent no-op instead of a loud one, so the + schema cannot catch a typo there and neither can Vale. Read the field + list twice when writing those two. **What `%%s` fills with depends on the extension point.** Getting this wrong is the one mistake in this recipe that passes every check below @@ -239,6 +353,42 @@ it. `ignorecase: false` to protect the correct spelling. - `raw` takes a full regex when `tokens` is too restrictive; `nonword` removes the implicit boundaries. + - **A token made only of punctuation can never match without + `nonword: true`.** The boundaries above are `\b`, which needs a word + character on the inside. An em dash has none, on either side. + Measured against `This is a sentence — with an em dash.`: + + ```yaml + # fires on nothing, ever — and reports no error + extends: existence + message: "Use a comma, not an em dash" + tokens: + - '—' + ``` + + ```yaml + # fires + extends: existence + message: "Use a comma, not an em dash" + nonword: true + tokens: + - '—' + ``` + + The first rule verifies, tests green if its `fail/` fixture is + missing the dash, and reports nothing forever. Any token whose + pattern contains no `\w` — punctuation, an emoji, a bare symbol — + needs `nonword: true`. + + - **A bare word finds senses you did not mean.** `landed on` in a rule + about jargon also matches "the plane landed on time"; `simply` in a + rule about hedging also matches "simply connected" in a maths doc. + Narrow the token to the **collocation** you actually object to + (`landed on a decision`, not `landed on`), and check that you got it + right by writing the `pass/` fixture from the literal sense *first*: + put the innocent sentence in `pass/` before you write the guilty one + in `fail/`. A rule whose `pass/` bucket was written afterwards tends + to contain only sentences the author already knew were safe. Vale also understands the markup, which decides what counts as text before your pattern ever runs. Measured in markdown: @@ -288,6 +438,29 @@ it. the style, which is this rule's directory; the second is the check inside it, which is the file. Both are the id. + **Scope a rule *out* with a second matcher, not a cleverer glob.** A + glob says which files a rule sees; it has no way to say "these but not + those". The exclusion is a second matcher that assigns `NO`, and + because precedence here is positional — a later matcher wins — the + exclusion goes **after** the inclusion: + + ```ini + # Every markdown file… + [*.md] + tskl) rule = no-simply + BasedOnStyles = + no-simply.no-simply = YES + + # …except the changelog, which quotes release notes verbatim. + [CHANGELOG.md] + tskl) rule = no-simply + no-simply.no-simply = NO + ``` + + Write the breadcrumb in the second matcher too, or assembly attributes + it to nobody. Reversing the two blocks silently re-enables the rule on + the file you meant to exempt, and nothing reports that. + **Do NOT write `StylesPath` or `MinAlertLevel` here.** Those describe the run rather than a rule, the assembler supplies them, and a copy in a rule's config is dropped on the way in. @@ -383,6 +556,34 @@ it. That is the half of the fixture set that has to work for you. + **When the rule's subject normally appears in code, the `fail/` + fixture must carry it three ways** — inline in a code span, inside a + fenced block, and in ordinary prose — in that one document. A rule + about a command, a flag, a package name or an env var has a subject + that lives in fenced blocks in every real README, and the default + scope cannot see fenced blocks at all. A `fail/` fixture written only + in prose therefore fires, goes green, and the rule then catches none + of the real violations. Measured on one document holding the token in + all three places: the default scope found one of three, `raw` found + three. If the fixture fires on the prose line and not on the other + two, the answer is `scope: raw` — see step 2 for what that costs. + + **Fixtures run under a config that isolates this rule, so a green + `test` is not evidence the rule reaches any real file.** `test` + generates its own `.vale.ini` pointing at the fixture directory and + enabling only `.`; your rule's own matcher globs are not + consulted. So a glob of `packages/cli/src/**/*.ts` that matches + nothing in the repository still produces a rule that verifies, tests + green, and reports forever. The only check for that is a real + `check` over a real file: + + ``` + %(TASKLESS_CLI)s check --json + ``` + + Do that once, on a file you have deliberately made violate the rule, + before you believe the rule works. + 6. **Verify, then test.** Two commands, both taking the rule's directory as their argument, both run from the project root: From ce731de44b06a8ebab74726f5d9dabc341b3abee Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 21:46:00 -0700 Subject: [PATCH 03/17] docs(cli): record that Vale drops everything inside a
MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scope: figure.caption` looked inert because the obvious control — a `
` nested in `
` — is invisible to every scope, `text` and `raw` included. A bare `
` is linted and the scope fires on it. The operand is measured after all; the first reading would have dropped it from the vocabulary. This is the case a known-valid control alongside each measurement exists to catch, so it is recorded in the design as well as the recipe. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .../changes/vale-authoring-polish/design.md | 17 ++++++++++------- packages/cli/src/agent/create-vale-rule.txt | 10 +++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/openspec/changes/vale-authoring-polish/design.md b/openspec/changes/vale-authoring-polish/design.md index 3931bd7e..65b5792f 100644 --- a/openspec/changes/vale-authoring-polish/design.md +++ b/openspec/changes/vale-authoring-polish/design.md @@ -138,13 +138,16 @@ Two corrections to this document's own earlier draft: `frontmatter.`, which is what the binary carries (`text.frontmatter.`) and what fires. `meta` and `meta.class.` never fired in Markdown, MDX, or HTML. -- **`figure.caption` is in the binary's vocabulary** (`text.figure.caption`) but - no control could be made to fire for it in a format this build can lint - without an external converter — `
` content did not reach even a - `scope: text` control in HTML, and AsciiDoc and reStructuredText need - `asciidoctor`/`rst2html`, which Taskless excludes. It is accepted by the - schema on the accept-when-unclear rule and carried in the corpus as an - explicitly unproven operand rather than as a passing row. +- **`figure.caption` fires, but only once you find a control it can reach.** + The obvious control — a `
` nested inside `
` — never fires, + and neither does a `scope: text` control over the same document: Vale drops + everything inside a `
` element before any check runs. A bare + `
` is linted normally, and `scope: figure.caption` fires on it in + both HTML and Markdown. This is the case task 5.4 exists for: without the + known-valid control alongside it, the first result would have been recorded as + "the binary ignores this scope" and the operand wrongly dropped from the enum. + (AsciiDoc and reStructuredText, the other formats with figure captions, need + `asciidoctor`/`rst2html`, which Taskless excludes.) ### Per-check fields, and two checks that validate nothing diff --git a/packages/cli/src/agent/create-vale-rule.txt b/packages/cli/src/agent/create-vale-rule.txt index 17f386dc..4180a4de 100644 --- a/packages/cli/src/agent/create-vale-rule.txt +++ b/packages/cli/src/agent/create-vale-rule.txt @@ -169,7 +169,7 @@ it. | `table.header` | header cells | | `table.cell` | body cells | | `table.caption` | a table's caption | -| `figure.caption` | a figure's caption | +| `figure.caption` | a figure's caption — but see below | | `frontmatter` | every YAML front-matter value | | `frontmatter.` | one front-matter key's value | | `text.class.` | HTML elements carrying that class | @@ -183,6 +183,14 @@ it. three. If you want prose and inline code but not fenced blocks, write the list — `raw` is not "a bit wider", it is everything. + **Vale drops everything inside a `
` element.** Measured: a + `
` nested in `
` is invisible to *every* scope, + `text` and `raw` included, so a `scope: figure.caption` rule over a + normally-marked-up figure reports nothing and looks like a bad scope. + A bare `
` is linted, and `figure.caption` fires on it. If a + fixture is not firing, check whether its subject is inside a `
` + before you touch the pattern. + **`scope` also takes `~` and `&`.** `~code` is everything but inline code; `text & ~code` chains two operands; a list (`[code, text]`) is a union. All three parse and behave. From 89db9239d0c428c4cc1b0e67d4c6cb5771c5cd27 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 21:49:07 -0700 Subject: [PATCH 04/17] docs(cli): measure Vale's field-name casing, and restore capitalization's prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more measurements against the pinned binary. `capitalization` takes `prefix` — singular — and rejects both `prefixes` and `suffixes`; the first draft of this table dropped the field entirely on the strength of the plural's rejection. Field keys decode case-insensitively (`Tokens:`, `ignoreCase:` both work) while `extends`, `message` and `level` are read literally and their values are case-sensitive, so `level: WARNING` and `extends: Existence` are both rejected. Recorded because the schema in the next unit has to match that split exactly or it rejects rules the binary accepts. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .../changes/vale-authoring-polish/design.md | 16 +++++++++++++--- packages/cli/src/agent/create-vale-rule.txt | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/openspec/changes/vale-authoring-polish/design.md b/openspec/changes/vale-authoring-polish/design.md index 65b5792f..a1031baf 100644 --- a/openspec/changes/vale-authoring-polish/design.md +++ b/openspec/changes/vale-authoring-polish/design.md @@ -156,9 +156,19 @@ Every check accepts `extends`, `message`, `level`, `scope`, `link`, `limit`, `substitution`, `capitalization`, `conditional`, `repetition` and **rejected** by `occurrence`, `metric`, `readability`, `script`, `sequence`. -The per-check additions are in the recipe's table. Two published claims are -wrong against the binary: `capitalization` rejects `prefixes`, `suffixes` and -`ignorecase`; `occurrence` rejects `exceptions`. +The per-check additions are in the recipe's table. Three published claims are +wrong against the binary: `capitalization` takes `prefix` (singular) and rejects +`prefixes` and `suffixes`; `capitalization` rejects `ignorecase`; `occurrence` +rejects `exceptions` and `vocab`. + +**Field keys are matched case-insensitively — but `extends`, `message` and +`level` are not.** Vale decodes a check's own fields through a case-insensitive +map, so `Tokens:` and `ignoreCase:` are read exactly as their lowercase +spellings. The three header keys are read separately and literally: `EXTENDS:` +fails with "Missing the required 'extends' key". Their _values_ are +case-sensitive too — `level: WARNING` and `extends: Existence` are both +rejected. The schema therefore compares field names case-insensitively and +`extends`/`level` values exactly, which is what the binary does. **`consistency` and `spelling` accept any key at all.** `bananafield: true` on either loads without complaint and is ignored — they do not use the strict diff --git a/packages/cli/src/agent/create-vale-rule.txt b/packages/cli/src/agent/create-vale-rule.txt index 4180a4de..8c1bbe7d 100644 --- a/packages/cli/src/agent/create-vale-rule.txt +++ b/packages/cli/src/agent/create-vale-rule.txt @@ -226,7 +226,7 @@ it. |------------------|------------------------------------------------------------------------------| | `existence` | `tokens` (a list) or `raw`; `ignorecase`, `nonword`, `exceptions`, `append` | | `substitution` | `swap` (a map of observed → expected); `ignorecase`, `nonword`, `exceptions`, `capitalize`, `pos` | -| `capitalization` | `match`; `style` (with `$title`), `exceptions`, `threshold`, `indicators` | +| `capitalization` | `match`; `style` (with `$title`), `exceptions`, `threshold`, `indicators`, `prefix` | | `occurrence` | `token`, `max` and/or `min`; `ignorecase` | | `repetition` | `tokens`; `alpha`, `ignorecase`, `exceptions`, `max` | | `consistency` | `either` (a map of the two acceptable forms); `ignorecase`, `nonword` | @@ -238,10 +238,18 @@ it. | `script` | `script` (Tengo source) | The list above is measured, not transcribed: every entry was added to - a minimal rule of that check and the run watched for `E201`. Two - corrections fall out of it, both against the published docs — - `capitalization` rejects `prefixes`/`suffixes` and `ignorecase`, and - `occurrence` rejects `exceptions` and `vocab`. + a minimal rule of that check and the run watched for `E201`. Three + corrections fall out of it, all against the published docs — + `capitalization` takes `prefix` (singular) and rejects both `prefixes` + and `suffixes`, it rejects `ignorecase`, and `occurrence` rejects + `exceptions` and `vocab`. + + **Field names are matched case-insensitively, but `extends`, `message` + and `level` are not.** Measured: `Tokens:` and `ignoreCase:` are read + exactly as their lowercase spellings, while `EXTENDS:` fails with + "Missing the required 'extends' key". Their *values* are case-sensitive + too — `level: WARNING` and `extends: Existence` are both rejected. + Write everything lowercase and none of this can bite you. **A field from the wrong check is the loudest failure Vale has.** `tokens` on an `occurrence` check gives From e98fbdf1bdb13a83999cfdbec131120a0bde1bdf Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 21:59:36 -0700 Subject: [PATCH 05/17] docs(cli): keep the recipe's claims true for the unit that ships it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The revised sections told the author that `verify` rejects a bad `extends`, an unrecognized `scope`, and a foreign field. None of that is true until the schema layer lands, and this stack merges forward — so this unit alone would ship a recipe promising a check that does not exist, which is the exact failure the recipe is about. Each promise is replaced by the instruction it was standing in for, and comes back in the unit that makes it true. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- packages/cli/src/agent/create-vale-rule.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/agent/create-vale-rule.txt b/packages/cli/src/agent/create-vale-rule.txt index 8c1bbe7d..5b1ced95 100644 --- a/packages/cli/src/agent/create-vale-rule.txt +++ b/packages/cli/src/agent/create-vale-rule.txt @@ -103,7 +103,6 @@ it. facility for a rule that does not extend one of these twelve, and an `extends` outside the set is not a rule that misbehaves: Vale exits 2 and **every** Vale rule in the project goes unreported for that run. - `verify` catches it before Vale is invoked. **Nine of them have a worked rule at the end of this recipe**, each with the near-miss that fails and why. Read the one closest to your @@ -142,7 +141,8 @@ it. rejected by Vale: `scope: fenced` loads, runs, and matches nothing. It is the worst of the three failures on this page, because unlike a bad `extends` or a foreign field it does not even take the run down to - tell you — the rule is simply inert, forever. `verify` rejects it. + tell you — the rule is simply inert, forever. Copy the value from the + table below rather than typing it. Every value below was measured against Vale v%(VALE_VERSION)s by authoring a rule with that scope and a document the rule had to flag. @@ -199,8 +199,8 @@ it. Measured: `~banana` and `text & ~banana` both fire on everything, because there is no such scope to subtract. A typo inside a `~` does not narrow the rule and does not widen it visibly — it removes the - exclusion you wrote the rule for. `verify` checks the operands inside - `~` and `&` for exactly this reason. + exclusion you wrote the rule for. Check the spelling inside a `~` + against the table as carefully as a bare one. **`scope` is per-rule, and rules do not interact.** Taskless assembles every rule's matchers into one config for the run, which invites the @@ -255,8 +255,8 @@ it. `tokens` on an `occurrence` check gives `E201 … has invalid keys: 'tokens'`, exit 2, and — because Vale reads one assembled config per run — **no** Vale rule in the project reports - anything. `verify` rejects the rule before Vale is invoked so this - cannot reach `check`. + anything. Take the field names from the table above rather than from + memory: this is the mistake with the widest blast radius. **Two checks are exempt, and that is not a licence.** Measured, `consistency` and `spelling` accept any key at all: `bananafield: From a4bf01f3a123b35078451d373b4763bcf64aca26 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Tue, 25 Aug 2026 14:34:43 -0700 Subject: [PATCH 06/17] docs(cli): reconcile the recipe's derived counts and the vocab field Six same-PR internal consistency defects, all cases of this change's own later text disagreeing with its earlier text: - `vocab` left the "every rule carries" table. Measured, five of the twelve checks reject it and raise E201, which suppresses every other Vale rule in the project. It now sits with the per-check fields. - "Nine of them have a worked rule" was arithmetic from the old eleven check total. Seven checks have one, across nine worked rules. - "For the other eight" was the same stale derivation; nine checks sit outside the %s table. - design.md's Decisions list and proposal.md both still named the v3.18.0+ `meta` scopes, which design.md's own measurement disproves. The real addition is `frontmatter`/`frontmatter.`. - design.md's measured-firing enumeration omitted `figure.caption`. - The changeset said the measurement corrects two published claims. It corrects three. The sweep for the same class of bug turned up two more `vocab`-as-a- common-field claims, in the cli-agent-authoring delta spec and in tasks 2.8 and 3.2, where task 3.2 would have carried the error into the unit-2 schema. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .changeset/vale-authoring-polish.md | 12 ++++---- .../changes/vale-authoring-polish/design.md | 6 ++-- .../changes/vale-authoring-polish/proposal.md | 2 +- .../specs/cli-agent-authoring/spec.md | 2 +- .../changes/vale-authoring-polish/tasks.md | 4 +-- packages/cli/src/agent/create-vale-rule.txt | 30 ++++++++++++------- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.changeset/vale-authoring-polish.md b/.changeset/vale-authoring-polish.md index d09e76b6..9e580a84 100644 --- a/.changeset/vale-authoring-polish.md +++ b/.changeset/vale-authoring-polish.md @@ -26,13 +26,15 @@ pinned Vale binary rather than as a caution in principle: fixture from the literal sense first. - **Fixture design for a subject that appears in code**: `fail/` must carry it inline, fenced, and in prose. -- **`limit` and `vocab`** in the common-fields table, and that Vale loads only - `.yml` — a style file renamed to `.yaml` is silently not loaded at all. +- **`limit`** in the common-fields table, **`vocab`** with the per-check fields + it actually belongs to, and that Vale loads only `.yml` — a style file renamed + to `.yaml` is silently not loaded at all. - **Fixtures run under an isolating config**, so a green `test` is not evidence the rule's matcher glob reaches any real file. Vale's check types are now enumerated from the binary rather than the docs. There are **twelve**, not eleven: the docs fold `readability` into `metric`. -The per-check field tables are measured the same way, which corrects two -published claims — `capitalization` rejects `prefixes`/`suffixes`, and -`occurrence` rejects `exceptions`. +The per-check field tables are measured the same way, which corrects three +published claims: `capitalization` takes `prefix` (singular) and rejects +`prefixes` and `suffixes`, `capitalization` rejects `ignorecase`, and +`occurrence` rejects `exceptions` and `vocab`. diff --git a/openspec/changes/vale-authoring-polish/design.md b/openspec/changes/vale-authoring-polish/design.md index a1031baf..2d1c1920 100644 --- a/openspec/changes/vale-authoring-polish/design.md +++ b/openspec/changes/vale-authoring-polish/design.md @@ -50,7 +50,7 @@ This is the same relationship `capabilities.ts` already has with the format tier The recipe presents `scope` as a single value. Measured, the binary accepts a bare string, a list, `~` negation, and `&` chaining — `~code`, `[~code]`, and `text & ~code` all parse and behave. A schema modeling `scope` as a flat enum would reject valid rules, which is worse than the gap it closes, so the enum applies to the _operands_ and the schema accepts the operators around them. -The vocabulary is also larger than the recipe's list: `heading` with levels, the `table.*` and `figure.caption` forms, `list`, `paragraph`, `sentence`, `blockquote`, `alt`, `summary`, `raw`, the v3.17.0+ inline scopes (`link`, `code`, `strong`, `emphasis`), `text.class.`, the v3.18.0+ `meta` and `meta.class.`, and `comment.line` / `comment.block`. +The vocabulary is also larger than the recipe's list: `heading` with levels, the `table.*` and `figure.caption` forms, `list`, `paragraph`, `sentence`, `blockquote`, `alt`, `summary`, `raw`, the v3.17.0+ inline scopes (`link`, `code`, `strong`, `emphasis`), `text.class.`, the v3.18.0+ `frontmatter` and `frontmatter.`, and `comment.line` / `comment.block`. `scope` is the highest-value field in the schema, because it is the one field nothing downstream ever validates. An invalid scope is not an error anywhere in the stack — it is a rule that matches nothing. @@ -120,8 +120,8 @@ treats them as separate checks with disjoint fields (`metrics`/`grade` versus Measured firing: `text`, `code`, `raw`, `heading`, `heading.h1`–`h6`, `paragraph`, `sentence`, `list`, `blockquote`, `link`, `alt`, `summary`, `strong`, `emphasis`, `table`, `table.header`, `table.cell`, `table.caption`, -`comment`, `comment.line`, `comment.block`, `frontmatter`, `frontmatter.`, -`text.class.`. +`figure.caption`, `comment`, `comment.line`, `comment.block`, `frontmatter`, +`frontmatter.`, `text.class.`. Reach, on one document holding the token in prose, in an inline span, and in a fenced block: `text` → 1, `code` → 1, `[code, text]` → 2, `raw` → 3. `raw` diff --git a/openspec/changes/vale-authoring-polish/proposal.md b/openspec/changes/vale-authoring-polish/proposal.md index 8ddca3b6..182cb0c1 100644 --- a/openspec/changes/vale-authoring-polish/proposal.md +++ b/openspec/changes/vale-authoring-polish/proposal.md @@ -12,7 +12,7 @@ Measured against the pinned Vale 3.18.0, the engine does not close these. `exten - **`verify` validates a Vale rule file structurally**, before Vale sees it, against a schema that models the check types and their fields. Today it validates `level` and the presence of the rule's `.vale.ini`, and nothing else — `extends: nonsense` verifies clean. - **The schema is hand-authored and pinned to `VALE_VERSION`**, because there is nothing to generate it from. Vale publishes no JSON Schema; the machine-readable field knowledge exists only behind its paid cloud MCP at `api.vale.sh/mcp`, which `verify` cannot depend on. The vendor-contract test is what stops the schema drifting from the binary. -- **`scope` gains a real enum**, including the hierarchical forms, the v3.17.0+ inline scopes, and the v3.18.0+ `meta` scopes — and accepts the `~` negation and `&` chaining syntax that the recipe never documents and that a naive enum would reject. +- **`scope` gains a real enum**, including the hierarchical forms, the v3.17.0+ inline scopes, and the v3.18.0+ `frontmatter` scopes — and accepts the `~` negation and `&` chaining syntax that the recipe never documents and that a naive enum would reject. - **The `create-vale-rule` recipe closes the gaps that produce silent rules**: the measured scope table, `nonword` for punctuation tokens, how to scope a rule _out_ (the recipe only explains scoping in), fixture design for a subject that normally appears in code, and the `limit`/`vocab` fields it omits. - **No breaking change.** Every rule that verifies today continues to verify; the schema only rejects rules the engine was already going to ignore or fail on. diff --git a/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md b/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md index 9818a953..82ab7273 100644 --- a/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md +++ b/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md @@ -23,7 +23,7 @@ The recipe SHALL additionally carry the guidance below. Each item is a failure o - **How to scope a rule out**, not only in: a second matcher assigning `. = NO`. - **That a bare word finds senses you did not mean**, and that narrowing to a collocation is checked by writing the `pass/` fixture from the literal sense first. - **That fixture design follows the rule's subject**: when the subject normally appears in code, the `fail/` fixture SHALL contain it inline, fenced, and in prose. -- **The `limit` and `vocab` common fields**, which the recipe's field table omits. +- **The `limit` common field and the per-check `vocab` field**, which the recipe's field table omits. `vocab` SHALL NOT be presented as a field every check accepts: measured, five of the twelve reject it and raise `E201`. #### Scenario: Authoring produces all three artifacts diff --git a/openspec/changes/vale-authoring-polish/tasks.md b/openspec/changes/vale-authoring-polish/tasks.md index 659695a9..98a2e009 100644 --- a/openspec/changes/vale-authoring-polish/tasks.md +++ b/openspec/changes/vale-authoring-polish/tasks.md @@ -14,7 +14,7 @@ - [x] 2.5 Document scoping a rule _out_ with a second matcher assigning `NO`. - [x] 2.6 Add the collocation guidance: narrow a banned word to a collocation, and write the `pass/` fixture from the literal sense first. - [x] 2.7 Add the fixture rule for a subject that appears in code: `fail/` must carry it inline, fenced, and in prose. -- [x] 2.8 Add `limit` and `vocab` to the common-fields table, and note that Vale requires `.yml` rather than `.yaml`. +- [x] 2.8 Add `limit` to the common-fields table and `vocab` to the per-check table, and note that Vale requires `.yml` rather than `.yaml`. `vocab` is not a common field: it is rejected by `occurrence`, `metric`, `readability`, `script` and `sequence`. - [x] 2.9 State that fixtures run under an isolating config, so passing tests do not prove the matcher reaches any real file. - [x] 2.10 Update the recipe's cross-reference tests if the added sections change what they assert. No change needed: `recipe-cross-references.test.ts` asserts the rendered format lists, which the new sections do not touch, and the full suite is green. - [x] 2.11 Write the changeset on this branch, so the stack inherits it. @@ -22,7 +22,7 @@ ## 3. The schema (unit 2) - [ ] 3.1 Author the schema from the measurements in group 1, alongside `packages/cli/src/generated/ast-grep-rule-schema.json`. Pin it to `VALE_VERSION`. -- [ ] 3.2 Model the common header fields: `extends`, `message`, `level`, `scope`, `link`, `limit`, `vocab`. +- [ ] 3.2 Model the common header fields: `extends`, `message`, `level`, `scope`, `link`, `limit`, `action`, `description`, `name`. `vocab` is per-check, not common, and belongs in 3.4. - [ ] 3.3 Model `scope` as a grammar over an enum of operands, accepting a string, a list, `~`, and `&`. - [ ] 3.4 Model the per-check field tables, so a field belonging to another check type is rejected before `E201` can suppress the engine. - [ ] 3.5 Decide and record how strictly to treat a value whose status the measurements left unclear. The design's rule is to accept it. diff --git a/packages/cli/src/agent/create-vale-rule.txt b/packages/cli/src/agent/create-vale-rule.txt index 5b1ced95..b6085762 100644 --- a/packages/cli/src/agent/create-vale-rule.txt +++ b/packages/cli/src/agent/create-vale-rule.txt @@ -104,10 +104,11 @@ it. `extends` outside the set is not a rule that misbehaves: Vale exits 2 and **every** Vale rule in the project goes unreported for that run. - **Nine of them have a worked rule at the end of this recipe**, each - with the near-miss that fails and why. Read the one closest to your - intent before writing anything — the mistakes documented there are - observed, and most of them fail silently. + **The other seven have a worked rule at the end of this recipe**, + nine rules between them, each with the near-miss that fails and why. + Read the one closest to your intent before writing anything — the + mistakes documented there are observed, and most of them fail + silently. 2. **Write the style file** to `.taskless/rules/vale//.yml`, where `` is kebab-case and @@ -129,7 +130,14 @@ it. | `scope` | no | narrow to part of a document — see below | | `link` | no | a URL the reader can follow for the reasoning | | `limit` | no | cap findings from this rule per scope | -| `vocab` | no | `false` opts the rule out of `Vocab` filtering | + + **`vocab` is not one of these.** It reads as though it were, but it is + a per-check field: measured, only `existence`, `substitution`, + `capitalization`, `conditional` and `repetition` accept it, and + `occurrence`, `metric`, `readability`, `script` and `sequence` reject + it. On one of those five it raises `E201` and takes every other Vale + rule in the project down with it. It is listed with the per-check + fields below. **The file extension must be `.yml`.** Measured: rename a working style file to `.yaml` and Vale loads nothing — no error, no warning, @@ -224,13 +232,13 @@ it. | extends | its fields | |------------------|------------------------------------------------------------------------------| -| `existence` | `tokens` (a list) or `raw`; `ignorecase`, `nonword`, `exceptions`, `append` | -| `substitution` | `swap` (a map of observed → expected); `ignorecase`, `nonword`, `exceptions`, `capitalize`, `pos` | -| `capitalization` | `match`; `style` (with `$title`), `exceptions`, `threshold`, `indicators`, `prefix` | +| `existence` | `tokens` (a list) or `raw`; `ignorecase`, `nonword`, `exceptions`, `append`, `vocab` | +| `substitution` | `swap` (a map of observed → expected); `ignorecase`, `nonword`, `exceptions`, `capitalize`, `pos`, `vocab` | +| `capitalization` | `match`; `style` (with `$title`), `exceptions`, `threshold`, `indicators`, `prefix`, `vocab` | | `occurrence` | `token`, `max` and/or `min`; `ignorecase` | -| `repetition` | `tokens`; `alpha`, `ignorecase`, `exceptions`, `max` | +| `repetition` | `tokens`; `alpha`, `ignorecase`, `exceptions`, `max`, `vocab` | | `consistency` | `either` (a map of the two acceptable forms); `ignorecase`, `nonword` | -| `conditional` | `first`, `second`; `exceptions`, `ignorecase` | +| `conditional` | `first`, `second`; `exceptions`, `ignorecase`, `vocab` | | `metric` | `formula`, `condition` | | `readability` | `metrics` (a list of formula names), `grade` | | `spelling` | `aff`, `dic`, `custom`, `filters`, `ignore`, `threshold` | @@ -280,7 +288,7 @@ it. *replacement*, not the match, so `"Use GitHub not %%s"` against the text `Github` renders `Use GitHub not GitHub`. - For the other eight, do not guess. Write the message, run step 6, and + For the other nine, do not guess. Write the message, run step 6, and read it back off the finding — no test you can write catches a wrong `%%s`, so your own eyes on the rendered message are the check. From 792d49bcc9957723c97066e5170417e6e39bc79b Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 22:00:09 -0700 Subject: [PATCH 07/17] feat(cli): schema-check a Vale rule before Vale ever sees it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `verify` validated `level` and the presence of the rule's `.vale.ini`, so `extends: nonsense` and `scope: fenced` both verified clean. It now also checks `extends` against the twelve check types, every `scope` operand against the vocabulary the binary honors, and every field against the table for the check the rule extends. Ordering is the point for two of the three. Vale reads one assembled config per run, so an unknown `extends` or a foreign field reaching the binary takes down every other Vale rule's findings — not just the offending rule's. The layer is a pure read of the rule's own files, and `test` runs it first and stops, so neither can get that far. `scope` is modelled as a grammar rather than an enum, because the binary accepts a bare operand, a list, `~` negation and `&` chaining, and a flat enum would reject all but the first. It is deliberately stricter than Vale in one place: `~fenced` fires on everything, having silently lost the exclusion it was written for, which is exactly the class of failure this closes. The schema is hand-authored — Vale publishes no JSON Schema and its machine-readable field knowledge is behind a paid hosted MCP — so what makes it true is `test/vale-corpus.ts`: 82 minimal rules, each with a document it must flag, run through both the vendored binary and the schema, asserting the two agree in both directions. A verdict cannot be read off an exit code, since an unrecognized scope parses clean and matches nothing, so the corpus records three outcomes and carries two guards against a row that passes while asserting nothing. Both guards earned their place: `figure.caption` first measured as ignored, because Vale drops everything inside a `
` element. Fixes #171 Refs #170 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .changeset/vale-authoring-polish.md | 26 + .../.openspec.yaml | 0 .../design.md | 8 +- .../proposal.md | 0 .../specs/cli-agent-authoring/spec.md | 0 .../specs/cli-rule-validation/spec.md | 0 .../tasks.md | 56 +- openspec/specs/cli-agent-authoring/spec.md | 42 ++ openspec/specs/cli-rule-validation/spec.md | 48 +- packages/cli/src/agent/create-vale-rule.txt | 29 +- packages/cli/src/rules/inspect.ts | 33 +- packages/cli/src/schemas/vale-rule.ts | 358 ++++++++++ packages/cli/test/vale-corpus.ts | 628 ++++++++++++++++++ .../cli/test/vale-schema-contract.test.ts | 299 +++++++++ .../cli/test/vale-vendor-contract.test.ts | 66 ++ .../cli/test/verify-test-commands.test.ts | 177 +++++ 16 files changed, 1705 insertions(+), 65 deletions(-) rename openspec/changes/{vale-authoring-polish => archive/2026-08-25-vale-authoring-polish}/.openspec.yaml (100%) rename openspec/changes/{vale-authoring-polish => archive/2026-08-25-vale-authoring-polish}/design.md (91%) rename openspec/changes/{vale-authoring-polish => archive/2026-08-25-vale-authoring-polish}/proposal.md (100%) rename openspec/changes/{vale-authoring-polish => archive/2026-08-25-vale-authoring-polish}/specs/cli-agent-authoring/spec.md (100%) rename openspec/changes/{vale-authoring-polish => archive/2026-08-25-vale-authoring-polish}/specs/cli-rule-validation/spec.md (100%) rename openspec/changes/{vale-authoring-polish => archive/2026-08-25-vale-authoring-polish}/tasks.md (63%) create mode 100644 packages/cli/src/schemas/vale-rule.ts create mode 100644 packages/cli/test/vale-corpus.ts create mode 100644 packages/cli/test/vale-schema-contract.test.ts diff --git a/.changeset/vale-authoring-polish.md b/.changeset/vale-authoring-polish.md index 9e580a84..7d2bf027 100644 --- a/.changeset/vale-authoring-polish.md +++ b/.changeset/vale-authoring-polish.md @@ -38,3 +38,29 @@ The per-check field tables are measured the same way, which corrects three published claims: `capitalization` takes `prefix` (singular) and rejects `prefixes` and `suffixes`, `capitalization` rejects `ignorecase`, and `occurrence` rejects `exceptions` and `vocab`. + +`verify` now schema-checks a Vale rule structurally, before Vale is invoked. +It previously validated `level` and the presence of the rule's `.vale.ini`, so +`extends: nonsense` and `scope: fenced` both verified clean. It now also checks: + +- **`extends`** against the twelve check types, naming the accepted set. +- **`scope`** as a grammar over measured operands — a bare value, a list, `~` + negation, `&` chaining — rather than a flat enum, which would have rejected + working rules. It is deliberately stricter than Vale in one place: a negation + over an operand Vale does not know (`~fenced`) fires on everything, having + silently lost its exclusion, and is rejected. +- **Per-check fields**, so a field belonging to another check type is caught + before Vale reports `E201`. `consistency` and `spelling` are exempt because + the binary accepts any key on those two. + +The ordering is the point for two of the three: Vale reads one assembled config +per run, so an unknown `extends` or a foreign field reaching the binary takes +down **every** Vale rule's findings, not just the offending rule's. + +The schema is hand-authored, because Vale publishes no JSON Schema and its +machine-readable field knowledge is behind a paid hosted MCP. What holds it to +the binary is a corpus of 82 minimal rules, each with a document it must flag, +run through both the vendored Vale and the schema, asserting the two agree — +with guards so that a rule which "did not fire" because its fixture was +unreachable cannot pass as a measurement. A Vale upgrade that changes the +vocabulary fails a test that names the value. diff --git a/openspec/changes/vale-authoring-polish/.openspec.yaml b/openspec/changes/archive/2026-08-25-vale-authoring-polish/.openspec.yaml similarity index 100% rename from openspec/changes/vale-authoring-polish/.openspec.yaml rename to openspec/changes/archive/2026-08-25-vale-authoring-polish/.openspec.yaml diff --git a/openspec/changes/vale-authoring-polish/design.md b/openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md similarity index 91% rename from openspec/changes/vale-authoring-polish/design.md rename to openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md index 2d1c1920..fac4da30 100644 --- a/openspec/changes/vale-authoring-polish/design.md +++ b/openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md @@ -191,7 +191,13 @@ silent case is `scope`, alone — which makes `scope` the highest-value field in the schema for exactly the reason the decision above gives, and makes the other two a blast-radius argument rather than a silence argument. +## Resolved during implementation + +- **Eleven check types or twelve?** Twelve, and the binary settles it rather than a judgement call — see _Measured against Vale 3.18.0_. +- **How much of the per-check field table to encode.** The full table, as recommended, for ten of the twelve. `consistency` and `spelling` are encoded as **permissive**: measured, they accept any key at all, so there is no `E201` to make unreachable and a strict table there would only reject rules that work. That is the accept-when-unclear rule applied to a case where the measurement was in fact clear and went the other way. +- **How strictly to treat a value the measurements left unclear.** Nothing was left unclear in the end. `figure.caption` looked ambiguous until the reach guard showed the control was at fault, and `meta` looked ambiguous until `frontmatter` turned out to be the real name — both resolved by measurement rather than by the fallback. The rule stands unused, which is the outcome to prefer. +- **One place the schema is deliberately stricter than the binary.** `scope: ~fenced` fires on everything, because there is no `fenced` to subtract, so the binary "accepts" it in the only sense an exit code can express. What the author gets is a rule whose exclusion was silently deleted, which is precisely the class of failure this change exists to close, so `verify` rejects it. It is recorded as a `divergence` row in the corpus, asserted as a disagreement rather than skipped, so the exception is countable. + ## Open Questions -- **How much of the per-check field table to encode.** The full table makes `E201` unreachable; a partial one still leaves the engine-wide suppression possible for the fields it omits. Recommend the full table, since `E201` is the failure with the widest blast radius, but the cost is that every check type must be transcribed and measured rather than just the common header. - **Whether `verify` should also report each matcher's selected-file count.** Out of scope here, but adjacent: a matcher glob that reaches nothing produces a rule that is well formed, enabled, green, and inert — the one silent failure this change does not close. diff --git a/openspec/changes/vale-authoring-polish/proposal.md b/openspec/changes/archive/2026-08-25-vale-authoring-polish/proposal.md similarity index 100% rename from openspec/changes/vale-authoring-polish/proposal.md rename to openspec/changes/archive/2026-08-25-vale-authoring-polish/proposal.md diff --git a/openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md b/openspec/changes/archive/2026-08-25-vale-authoring-polish/specs/cli-agent-authoring/spec.md similarity index 100% rename from openspec/changes/vale-authoring-polish/specs/cli-agent-authoring/spec.md rename to openspec/changes/archive/2026-08-25-vale-authoring-polish/specs/cli-agent-authoring/spec.md diff --git a/openspec/changes/vale-authoring-polish/specs/cli-rule-validation/spec.md b/openspec/changes/archive/2026-08-25-vale-authoring-polish/specs/cli-rule-validation/spec.md similarity index 100% rename from openspec/changes/vale-authoring-polish/specs/cli-rule-validation/spec.md rename to openspec/changes/archive/2026-08-25-vale-authoring-polish/specs/cli-rule-validation/spec.md diff --git a/openspec/changes/vale-authoring-polish/tasks.md b/openspec/changes/archive/2026-08-25-vale-authoring-polish/tasks.md similarity index 63% rename from openspec/changes/vale-authoring-polish/tasks.md rename to openspec/changes/archive/2026-08-25-vale-authoring-polish/tasks.md index 98a2e009..300e0400 100644 --- a/openspec/changes/vale-authoring-polish/tasks.md +++ b/openspec/changes/archive/2026-08-25-vale-authoring-polish/tasks.md @@ -21,45 +21,45 @@ ## 3. The schema (unit 2) -- [ ] 3.1 Author the schema from the measurements in group 1, alongside `packages/cli/src/generated/ast-grep-rule-schema.json`. Pin it to `VALE_VERSION`. -- [ ] 3.2 Model the common header fields: `extends`, `message`, `level`, `scope`, `link`, `limit`, `action`, `description`, `name`. `vocab` is per-check, not common, and belongs in 3.4. -- [ ] 3.3 Model `scope` as a grammar over an enum of operands, accepting a string, a list, `~`, and `&`. -- [ ] 3.4 Model the per-check field tables, so a field belonging to another check type is rejected before `E201` can suppress the engine. -- [ ] 3.5 Decide and record how strictly to treat a value whose status the measurements left unclear. The design's rule is to accept it. +- [x] 3.1 Author the schema from the measurements in group 1, alongside `packages/cli/src/generated/ast-grep-rule-schema.json`. Pin it to `VALE_VERSION`. **Placed** at `packages/cli/src/schemas/vale-rule.ts` rather than under `generated/`: that directory is for artifacts a script fetches, and this is hand-authored. It sits beside `schemas/ast-grep-rule.ts`, which is where the ast-grep JSON schema is consumed. +- [x] 3.2 Model the common header fields: `extends`, `message`, `level`, `scope`, `link`, `limit`, `action`, `description`, `name`. `vocab` is per-check, not common, and belongs in 3.4. +- [x] 3.3 Model `scope` as a grammar over an enum of operands, accepting a string, a list, `~`, and `&`. +- [x] 3.4 Model the per-check field tables, so a field belonging to another check type is rejected before `E201` can suppress the engine. +- [x] 3.5 Decide and record how strictly to treat a value whose status the measurements left unclear. The design's rule is to accept it. ## 4. Wire it into verify -- [ ] 4.1 Add the schema layer to the Vale verify path, reporting through the existing `LayerResult` and `VerifyResult` shapes so both engines fail the same way. -- [ ] 4.2 Ensure the error names the field and the accepted values, rather than reporting a raw schema path. -- [ ] 4.3 Confirm the layer runs before Vale is invoked, so a rule that would throw `E201` never reaches the binary. -- [ ] 4.4 Confirm `test` still runs `verify` first and stops on its failure. +- [x] 4.1 Add the schema layer to the Vale verify path, reporting through the existing `LayerResult` and `VerifyResult` shapes so both engines fail the same way. +- [x] 4.2 Ensure the error names the field and the accepted values, rather than reporting a raw schema path. +- [x] 4.3 Confirm the layer runs before Vale is invoked, so a rule that would throw `E201` never reaches the binary. +- [x] 4.4 Confirm `test` still runs `verify` first and stops on its failure. ## 5. The corpus, and the differential test that makes the schema true Build this before the schema is finalized — the schema is derived from it. This is the largest group and the one that will be under-built if rushed. -- [ ] 5.1 Design the corpus entry shape: a minimal rule YAML, the fixture that acts as its positive control, and the measured verdict. Keep it a declarative table so a Vale upgrade is a re-run rather than a re-authoring. -- [ ] 5.2 Establish the three-outcome verdict, since an exit code cannot express it: **fired** (accepted), **did not fire** (the binary ignored the construct), **`E201`** (rejected outright). An unrecognized `extends` or `scope` parses clean, so "did not fire" is the signal that the construct is invalid. -- [ ] 5.3 Write a positive control per entry. It depends on what the rule matches — a `scope: heading` entry needs a heading, `scope: table.cell` needs a table — so it cannot be generated from the check type. This is the bulk of the work. -- [ ] 5.4 Guard against a vacuous entry: assert each control fires for at least one known-valid variant, so a control that can never fire is a test failure rather than a quiet pass. -- [ ] 5.5 Cover every check type from 1.1, every scope operand from 1.2, and the per-check field tables from 1.3, including the `~` and `&` forms. -- [ ] 5.6 Write the differential test: for every entry, assert `schemaAccepts === binaryAccepts`. Report both directions distinctly — schema-too-lax is the gap being closed, schema-too-strict blocks valid work and is the worse failure. -- [ ] 5.7 Wire the corpus into `packages/cli/test/vale-vendor-contract.test.ts`, or a sibling beside it, following the existing convention of invoking the vendored binary directly rather than through `runVale`. -- [ ] 5.8 Add the version-bump case: raising `VALE_VERSION` past a change in accepted check types or scopes fails a test that names the field. -- [ ] 5.9 Verify the suite is non-vacuous by reverting the schema and watching exactly the expected entries fail. +- [x] 5.1 Design the corpus entry shape: a minimal rule YAML, the fixture that acts as its positive control, and the measured verdict. Keep it a declarative table so a Vale upgrade is a re-run rather than a re-authoring. +- [x] 5.2 Establish the three-outcome verdict, since an exit code cannot express it: **fired** (accepted), **did not fire** (the binary ignored the construct), **`E201`** (rejected outright). An unrecognized `extends` or `scope` parses clean, so "did not fire" is the signal that the construct is invalid. +- [x] 5.3 Write a positive control per entry. It depends on what the rule matches — a `scope: heading` entry needs a heading, `scope: table.cell` needs a table — so it cannot be generated from the check type. This is the bulk of the work. +- [x] 5.4 Guard against a vacuous entry: assert each control fires for at least one known-valid variant, so a control that can never fire is a test failure rather than a quiet pass. +- [x] 5.5 Cover every check type from 1.1, every scope operand from 1.2, and the per-check field tables from 1.3, including the `~` and `&` forms. +- [x] 5.6 Write the differential test: for every entry, assert `schemaAccepts === binaryAccepts`. Report both directions distinctly — schema-too-lax is the gap being closed, schema-too-strict blocks valid work and is the worse failure. +- [x] 5.7 Wire the corpus into `packages/cli/test/vale-vendor-contract.test.ts`, or a sibling beside it, following the existing convention of invoking the vendored binary directly rather than through `runVale`. +- [x] 5.8 Add the version-bump case: raising `VALE_VERSION` past a change in accepted check types or scopes fails a test that names the field. Two tripwires: the corpus differential names any construct whose verdict changed, and `vale-vendor-contract.test.ts` asks the binary to enumerate its own check types and compares that to `VALE_CHECK_TYPES`. +- [x] 5.9 Verify the suite is non-vacuous by reverting the schema and watching exactly the expected entries fail. Five mutations run: dropping `figure.caption` and dropping `readability` each failed _too strict_ naming the value; adding `meta` and allowing `tokens` on `occurrence` each failed _too lax_; breaking a control document failed both vacuity guards. ## 6. Regression coverage -- [ ] 6.1 Test that `extends: nonsense` fails `verify`, naming the field and the accepted values. -- [ ] 6.2 Test that an unrecognized `scope` fails `verify`, and that `~` and `&` over recognized operands pass. -- [ ] 6.3 Test that a foreign field for the declared check type fails `verify`. -- [ ] 6.4 Test that every rule under `.taskless/rules/vale/` still verifies, so no valid rule regressed. -- [ ] 6.5 Test that a rule with no fixtures still verifies, preserving the existing requirement. +- [x] 6.1 Test that `extends: nonsense` fails `verify`, naming the field and the accepted values. +- [x] 6.2 Test that an unrecognized `scope` fails `verify`, and that `~` and `&` over recognized operands pass. +- [x] 6.3 Test that a foreign field for the declared check type fails `verify`. +- [x] 6.4 Test that every rule under `.taskless/rules/vale/` still verifies, so no valid rule regressed. **Adapted:** this repository carries no committed Vale rules, so the test would be vacuous. It verifies the nine worked rules from `create-vale-rule` instead, which is the population the requirement is actually about — a rule an author was told to write. +- [x] 6.5 Test that a rule with no fixtures still verifies, preserving the existing requirement. ## 7. Land it -- [ ] 7.1 `pnpm typecheck`, `pnpm lint`, `pnpm test` pass. -- [ ] 7.2 `pnpm build`, then author a deliberately-broken rule and confirm the real CLI reports what the specs require. Record the actual output. -- [ ] 7.3 Extend the changeset on the bottom branch with unit 2's scope. -- [ ] 7.4 Open the stack as two PRs merging forward, the recipe unit first. -- [ ] 7.5 Archive the change on the tip PR. +- [x] 7.1 `pnpm typecheck`, `pnpm lint`, `pnpm test` pass. +- [x] 7.2 `pnpm build`, then author a deliberately-broken rule and confirm the real CLI reports what the specs require. Record the actual output. +- [x] 7.3 Extend the changeset on the bottom branch with unit 2's scope. +- [x] 7.4 Open the stack as two PRs merging forward, the recipe unit first. +- [x] 7.5 Archive the change on the tip PR. diff --git a/openspec/specs/cli-agent-authoring/spec.md b/openspec/specs/cli-agent-authoring/spec.md index 00b8603a..7247d3b8 100644 --- a/openspec/specs/cli-agent-authoring/spec.md +++ b/openspec/specs/cli-agent-authoring/spec.md @@ -1,8 +1,11 @@ # cli-agent-authoring Specification ## Purpose + TBD - created by archiving change agent-command-and-vale-authoring. Update Purpose after archive. + ## Requirements + ### Requirement: Every engine a rule can be routed to has an authoring recipe The CLI SHALL provide an authoring recipe for each engine `route` can name: `create-sg-rule`, `create-vale-rule`, and `create-runtime-rule`, alongside `create-legacy-rule` for a linter the repository already uses. @@ -33,6 +36,17 @@ It SHALL direct the agent to check its work by running `verify` and then `test` The previous version of this requirement taught the agent to add a matcher to a single project-wide `.vale.ini`. Executing that recipe against sandboxed agents found every one of its silent failures in that step and nowhere else — an assignment above the first matcher, a glob that missed the fixture extension, three names that had to agree with nothing reporting when they didn't. The layout change removes the step rather than documenting it further. +The recipe SHALL additionally carry the guidance below. Each item is a failure observed while authoring rules against this repository, and each produced a rule that passed `verify` and `test` while reporting nothing: + +- **What each `scope` reaches**, as measured: `text` sees prose, `code` sees inline code spans, `[code, text]` sees both, and `raw` sees prose, inline code, and fenced blocks. `raw` subsumes the other two. +- **That `scope` is per-rule.** Vale assembles one config per run, which invites the assumption that scopes interact. They do not. +- **That a rule scoped to `raw` cannot be suppressed** by Vale's `` directive, because it reads the unparsed document. A rule about a command needs `raw`, so it trades away per-case exemption. +- **That a token made only of punctuation needs `nonword: true`**, because Vale wraps every token in word boundaries. +- **How to scope a rule out**, not only in: a second matcher assigning `. = NO`. +- **That a bare word finds senses you did not mean**, and that narrowing to a collocation is checked by writing the `pass/` fixture from the literal sense first. +- **That fixture design follows the rule's subject**: when the subject normally appears in code, the `fail/` fixture SHALL contain it inline, fenced, and in prose. +- **The `limit` and `vocab` common fields**, which the recipe's field table omits. + #### Scenario: Authoring produces all three artifacts - **WHEN** the agent follows `create-vale-rule` @@ -55,6 +69,17 @@ The previous version of this requirement taught the agent to add a matcher to a - **THEN** the recipe SHALL identify this as incomplete - **AND** `verify` SHALL report it +#### Scenario: A rule about a command reaches fenced blocks + +- **WHEN** the agent authors a rule whose subject is a command, flag, or package name +- **THEN** the recipe SHALL direct it to a scope that reaches fenced blocks +- **AND** the `fail/` fixture SHALL carry the subject inline, fenced, and in prose + +#### Scenario: A punctuation token is not left unable to match + +- **WHEN** the agent authors a rule whose token contains no word characters +- **THEN** the recipe SHALL direct it to set `nonword: true` + ### Requirement: Authoring recipes write files rather than invoking a writer The `create-*-rule` recipes SHALL instruct the agent to write the rule, its configuration, and its fixtures directly. The CLI SHALL NOT provide a command that generates a Vale style file or authors a rule's matchers on the agent's behalf. @@ -118,3 +143,20 @@ The line SHALL orient, not classify: it states this recipe's own scope and SHALL - **WHEN** the orientation line is read - **THEN** it SHALL describe only this recipe's scope, not the comparison between engines +### Requirement: An authoring recipe states the failure a rule cannot report itself + +An authoring recipe SHALL document the ways a rule of its engine can be well formed, enabled, green on its fixtures, and still report nothing. + +A malformed rule is caught by `verify` and needs no recipe. A rule that is merely _wrong_ is caught by nothing, ships green, and is discovered only when someone notices it has never fired — so the recipe is the only place that failure can be prevented. + +#### Scenario: The recipe names the silent failures for its engine + +- **WHEN** an authoring recipe is written or revised +- **THEN** it SHALL name the failures that pass every local gate for that engine +- **AND** each SHALL be stated as an observed behavior of the pinned engine rather than as a caution in principle + +#### Scenario: Passing fixtures are not presented as proof of reach + +- **WHEN** the recipe directs the agent to run `test` +- **THEN** it SHALL state that fixtures run under an isolating config +- **AND** that a clean `check` SHALL be confirmed against a real file before the rule is believed to be working diff --git a/openspec/specs/cli-rule-validation/spec.md b/openspec/specs/cli-rule-validation/spec.md index dc3c62cb..91849ef0 100644 --- a/openspec/specs/cli-rule-validation/spec.md +++ b/openspec/specs/cli-rule-validation/spec.md @@ -42,11 +42,13 @@ The two commands split because they have different preconditions. An agent part- Per engine, `verify` SHALL check: -| Engine | Components | -| --------- | ----------------------------------------------------------------------- | -| `sg` | `.yml` against the ast-grep schema and the Taskless required fields | -| `vale` | `.yml` against Vale's own validation, and the rule's `.vale.ini` | -| `runtime` | `check.ts` present, and at least one capture rule under `captures/` | +| Engine | Components | +| --------- | ---------------------------------------------------------------------------------------------------- | +| `sg` | `.yml` against the ast-grep schema and the Taskless required fields | +| `vale` | `.yml` against the Vale rule schema and the Taskless required fields, and the rule's `.vale.ini` | +| `runtime` | `check.ts` present, and at least one capture rule under `captures/` | + +The `vale` row previously read "against Vale's own validation." Measured against the pinned 3.18.0 binary, that covers less than it claims: `level: bananas` is reported, while `extends: nonsense` and `scope: fenced` both verify clean and produce a rule that matches nothing. Vale validates a rule when it _runs_ one, and it runs one field at a time — so a name it does not recognize is not an error, it is a check that never fires. Schema validation is therefore its own layer for `vale`, as it already is for `sg`. #### Scenario: A rule with no fixtures still verifies @@ -59,6 +61,25 @@ Per engine, `verify` SHALL check: - **WHEN** a Vale style declares a `level` outside `suggestion`/`warning`/`error` - **THEN** `verify` SHALL report that error, naming the field +#### Scenario: An unrecognized extension point is rejected + +- **WHEN** a Vale style declares an `extends` that is not one of Vale's check types +- **THEN** `verify` SHALL report it, naming the field and the accepted values +- **AND** it SHALL NOT report the rule as valid + +#### Scenario: An unrecognized scope is rejected + +- **WHEN** a Vale style declares a `scope` that is not one of Vale's scope values +- **THEN** `verify` SHALL report it, naming the field +- **AND** a scope using the `~` negation or `&` chaining syntax over recognized values SHALL be accepted + +#### Scenario: A field belonging to another check type is rejected + +- **WHEN** a Vale style declares a field its `extends` does not accept, such as `tokens` on an `occurrence` check +- **THEN** `verify` SHALL report it before Vale is invoked + +The failure it prevents is not a local one: Vale reports this as `E201: has invalid keys` and reads one assembled config per run, so a single rule with a stray field suppresses every other Vale rule's findings. + ### Requirement: Test runs a rule's fixtures and runs verify first `test` SHALL execute a rule against its test material — ast-grep test cases, Vale `pass`/`fail` fixture buckets, or the runtime harness — and SHALL run `verify` first, stopping on a verify failure without running the fixtures. @@ -97,3 +118,20 @@ The rule generation loop SHALL run `verify` and then `test` against a newly auth - **WHEN** a rule is authored locally or written by the service - **THEN** the loop SHALL run `verify` and `test` against its path - **AND** SHALL surface a failure rather than reporting the rule as written + +### Requirement: The Vale rule schema is pinned to the vendored binary + +The Vale rule schema SHALL be authored in this repository and pinned to `VALE_VERSION`, and a vendor-contract test SHALL hold its claims against the vendored binary. + +Vale publishes no JSON Schema for its check types. The machine-readable field knowledge exists only behind its hosted MCP server, which is a paid product and unavailable to `verify`. The schema is therefore a transcription, and a transcription drifts — so the binary, not the documentation, SHALL be the authority for what the schema asserts. + +#### Scenario: A Vale upgrade that invalidates the schema fails loudly + +- **WHEN** `VALE_VERSION` is raised to a version whose accepted check types or scopes differ from the schema +- **THEN** the vendor-contract test SHALL fail +- **AND** the failure SHALL name the field whose accepted values changed + +#### Scenario: The schema is not derived from documentation alone + +- **WHEN** a value is added to the schema's `extends` or `scope` enumerations +- **THEN** it SHALL be one the vendored binary was measured accepting diff --git a/packages/cli/src/agent/create-vale-rule.txt b/packages/cli/src/agent/create-vale-rule.txt index b6085762..1e9208a0 100644 --- a/packages/cli/src/agent/create-vale-rule.txt +++ b/packages/cli/src/agent/create-vale-rule.txt @@ -103,6 +103,7 @@ it. facility for a rule that does not extend one of these twelve, and an `extends` outside the set is not a rule that misbehaves: Vale exits 2 and **every** Vale rule in the project goes unreported for that run. + `verify` rejects it before Vale is invoked, and names the twelve. **The other seven have a worked rule at the end of this recipe**, nine rules between them, each with the near-miss that fails and why. @@ -149,8 +150,8 @@ it. rejected by Vale: `scope: fenced` loads, runs, and matches nothing. It is the worst of the three failures on this page, because unlike a bad `extends` or a foreign field it does not even take the run down to - tell you — the rule is simply inert, forever. Copy the value from the - table below rather than typing it. + tell you — the rule is simply inert, forever. `verify` rejects a scope + outside the table below, which is the only layer that ever will. Every value below was measured against Vale v%(VALE_VERSION)s by authoring a rule with that scope and a document the rule had to flag. @@ -207,8 +208,9 @@ it. Measured: `~banana` and `text & ~banana` both fire on everything, because there is no such scope to subtract. A typo inside a `~` does not narrow the rule and does not widen it visibly — it removes the - exclusion you wrote the rule for. Check the spelling inside a `~` - against the table as carefully as a bare one. + exclusion you wrote the rule for. `verify` checks the operands inside + `~` and `&` as strictly as a bare one, for exactly this reason — the + one place it is deliberately stricter than Vale itself. **`scope` is per-rule, and rules do not interact.** Taskless assembles every rule's matchers into one config for the run, which invites the @@ -263,8 +265,8 @@ it. `tokens` on an `occurrence` check gives `E201 … has invalid keys: 'tokens'`, exit 2, and — because Vale reads one assembled config per run — **no** Vale rule in the project reports - anything. Take the field names from the table above rather than from - memory: this is the mistake with the widest blast radius. + anything. `verify` rejects the rule before Vale is invoked, so this + cannot reach `check`. **Two checks are exempt, and that is not a licence.** Measured, `consistency` and `spelling` accept any key at all: `bananafield: @@ -617,9 +619,18 @@ it. ``` `verify` asks whether the rule is well-formed: the style file parses, - `extends` and `message` are present, `level` is one Vale accepts, and - the config declares a matcher that enables `.`. It does **not** - need fixtures, so run it as soon as the style file exists. + `extends` names one of the twelve checks, `message` is present, `level` + is one Vale accepts, every `scope` operand is one Vale honors, every + field belongs to the check the rule extends, and the config declares a + matcher that enables `.`. It does **not** need fixtures, so run + it as soon as the style file exists. + + Those checks are measured against Vale v%(VALE_VERSION)s rather than + transcribed from its docs, and they run **before** Vale is invoked. + That ordering matters for two of them: an unknown `extends` and a + foreign field each fail the whole Vale run rather than just this rule, + so letting either reach the binary would take every other Vale rule's + findings down with it. `test` runs the rule against both buckets. It runs `verify` first and stops if that fails, so a malformed rule tells you what is malformed diff --git a/packages/cli/src/rules/inspect.ts b/packages/cli/src/rules/inspect.ts index 28b042fe..4c2ad54d 100644 --- a/packages/cli/src/rules/inspect.ts +++ b/packages/cli/src/rules/inspect.ts @@ -9,6 +9,7 @@ import { ruleFilePath, type EngineName, } from "./engines"; +import { validateValeRule } from "../schemas/vale-rule"; import { verifyRule, type VerifyResult } from "./verify"; import { verifyValeRule } from "./vale/verify"; import type { ResolvedRule } from "./resolve-path"; @@ -97,19 +98,17 @@ export async function verifyOneRule( const stylePath = ruleFilePath(cwd, engine, ruleId); try { const style = await readYaml(stylePath); - if (typeof style !== "object" || style === null) { - errors.push(`${ruleId}.yml is not a YAML mapping.`); - } else { - // `extends` and `message` are Vale's own required keys. Checking them - // here means an author hears about a missing one from `verify` rather - // than as an E201 buried in an engine failure at check time. + // Layer 1 for `vale`, the counterpart of the ast-grep schema layer: + // `extends`, `message`, `level`, the `scope` grammar, and the per-check + // field tables, all measured against the pinned binary. It runs here, + // before Vale is ever invoked, because two of the three defects it + // catches are not local — Vale reads one assembled config per run, so an + // unknown `extends` or a foreign field takes down every other Vale + // rule's findings rather than just this one's. + errors.push(...validateValeRule(ruleId, style).errors); + + if (typeof style === "object" && style !== null) { const record = style as Record; - if (typeof record.extends !== "string") { - errors.push(`${ruleId}.yml is missing the required 'extends' key.`); - } - if (typeof record.message !== "string") { - errors.push(`${ruleId}.yml is missing the required 'message' key.`); - } // `consistency` is the one extension point that compiles the rule's // own name into the pattern: Vale emits `(?P<N>…)` named capture // groups, and Go RE2 requires a group name to be word characters @@ -124,16 +123,6 @@ export async function verifyOneRule( `${ruleId} extends consistency, so its id becomes a regex group name and must be word characters only. Rename it without '-' (Vale would fail the whole run with E201).` ); } - const level = record.level; - if ( - level !== undefined && - (typeof level !== "string" || - !["suggestion", "warning", "error"].includes(level)) - ) { - errors.push( - `${ruleId}.yml has an invalid level; it must be suggestion, warning, or error.` - ); - } } } catch { errors.push(`Style file not found or unreadable: ${stylePath}`); diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts new file mode 100644 index 00000000..066924ce --- /dev/null +++ b/packages/cli/src/schemas/vale-rule.ts @@ -0,0 +1,358 @@ +/** + * The structural schema for a Vale style file, pinned to {@link VALE_VERSION}. + * + * Vale publishes no JSON Schema. Its repository has no `schemas/` directory, + * and the machine-readable field knowledge exists only behind the hosted MCP + * server at `api.vale.sh/mcp`, which is a paid product and unavailable to + * `verify`. So unlike `ast-grep-rule.ts` — which wraps an upstream artifact + * fetched per tag — everything here is a **transcription**, and a transcription + * drifts. + * + * What makes it trustworthy is not care in the writing. It is + * `test/vale-corpus.ts`: a table of minimal rules, each with a document it must + * flag, run through the vendored binary and through this module, asserting the + * two agree. Every value enumerated below was measured accepted by the binary + * that way. A Vale bump that changes the vocabulary fails that test naming the + * field, rather than leaving this file quietly wrong. + * + * Two directions of error, and they are not symmetric: + * + * - **Too lax** — a rule the binary will not honor verifies clean. That is the + * gap this module exists to close. + * - **Too strict** — a rule the binary accepts is rejected. That is worse: it + * blocks work that would have functioned. Where a measurement is ambiguous, + * accept. + */ + +import { VALE_VERSION } from "../rules/capabilities"; + +/** + * Vale's check types, quoted from the binary rather than the documentation. + * + * Obtained by giving Vale an `extends` it does not know, which is not silently + * ignored — it fails the file, exits 2, and enumerates the set: + * + * ``` + * 'extends' key must be one of [capitalization conditional consistency + * existence occurrence repetition substitution readability spelling sequence + * metric script]. + * ``` + * + * That settles the eleven-versus-twelve question the docs and Vale's own MCP + * guide disagree on: **twelve**. The docs' eleven folds `readability` into + * `metric`; the binary treats them as separate checks whose fields are + * disjoint, and each rejects the other's. + * + * Sorted here rather than left in the binary's order, so the value `verify` + * prints to an author reads predictably. + */ +export const VALE_CHECK_TYPES = [ + "capitalization", + "conditional", + "consistency", + "existence", + "metric", + "occurrence", + "readability", + "repetition", + "script", + "sequence", + "spelling", + "substitution", +] as const; + +export type ValeCheckType = (typeof VALE_CHECK_TYPES)[number]; + +/** Vale's three severities. The value is case-sensitive: `WARNING` is rejected. */ +export const VALE_LEVELS = ["suggestion", "warning", "error"] as const; + +/** + * Fields every check accepts, whatever it extends. + * + * `vocab` is deliberately **not** here: five of the twelve reject it. It is + * listed per check instead. + */ +const COMMON_FIELDS = [ + "extends", + "message", + "level", + "scope", + "link", + "limit", + "action", + "description", + "name", +] as const; + +/** + * The fields each check accepts beyond {@link COMMON_FIELDS}, measured by + * adding each candidate to a minimal rule of that check and watching for + * `E201: has invalid keys`. + * + * Three entries contradict Vale's published docs, and the binary wins: + * `capitalization` takes `prefix` **singular** and rejects `prefixes` and + * `suffixes`; `capitalization` rejects `ignorecase`; `occurrence` rejects + * `exceptions` and `vocab`. + * + * `consistency` and `spelling` are absent on purpose — see + * {@link VALE_PERMISSIVE_CHECKS}. + */ +const CHECK_FIELDS: Partial> = { + existence: [ + "tokens", + "raw", + "ignorecase", + "nonword", + "exceptions", + "append", + "vocab", + ], + substitution: [ + "swap", + "ignorecase", + "nonword", + "exceptions", + "capitalize", + "pos", + "vocab", + ], + capitalization: [ + "match", + "style", + "exceptions", + "threshold", + "indicators", + "prefix", + "vocab", + ], + occurrence: ["token", "max", "min", "ignorecase"], + repetition: ["tokens", "alpha", "ignorecase", "exceptions", "max", "vocab"], + conditional: ["first", "second", "exceptions", "ignorecase", "vocab"], + metric: ["formula", "condition"], + readability: ["metrics", "grade"], + sequence: ["tokens", "ignorecase"], + script: ["script"], +}; + +/** + * The two checks that validate nothing. + * + * Measured: `bananafield: true` on a `consistency` or a `spelling` rule loads + * without complaint and is ignored. They do not use the strict decode the other + * ten do, so the binary raises no `E201` for a foreign field there. + * + * The schema follows the binary rather than the docs. Being strict here would + * reject rules that work — the "too strict" direction — to catch a typo that + * costs nothing but a rule quietly missing one of its own fields. The recipe + * says so in prose, which is the only place it can be said. + */ +export const VALE_PERMISSIVE_CHECKS: readonly ValeCheckType[] = [ + "consistency", + "spelling", +]; + +/** + * The `scope` operands the binary was measured honoring. + * + * Each was established by authoring a rule with that scope over a document the + * rule had to flag, and confirming the finding appeared. A scope that never + * fired for any control is not here. + * + * This is the highest-value list in the module, because `scope` is the one + * field nothing downstream ever validates. An unrecognized `extends` fails the + * run loudly; an unrecognized field raises `E201`. An unrecognized **scope** + * loads, runs, and matches nothing — no error, anywhere, ever. + * + * Note what is *not* here: `meta` and `meta.class.`. The v3.18.0 addition + * is `frontmatter` / `frontmatter.`, which is the vocabulary the binary + * carries and the one that fires. + */ +const SCOPE_OPERANDS = new Set([ + "text", + "code", + "raw", + "heading", + "heading.h1", + "heading.h2", + "heading.h3", + "heading.h4", + "heading.h5", + "heading.h6", + "paragraph", + "sentence", + "list", + "blockquote", + "link", + "alt", + "summary", + "strong", + "emphasis", + "table", + "table.header", + "table.cell", + "table.caption", + "figure.caption", + "frontmatter", + "comment", + "comment.line", + "comment.block", +]); + +/** + * Scope families whose tail is author-supplied and cannot be enumerated. + * + * `frontmatter.title` names a key in the document's own front matter; + * `text.class.callout` names an HTML class. Both were measured firing, and + * neither has a closed set — rejecting an unfamiliar tail would be the + * "too strict" failure against a value the binary honors. + */ +const SCOPE_PREFIXES = ["frontmatter.", "text.class."]; + +/** Operands, for the message `verify` shows an author. */ +export const VALE_SCOPE_OPERANDS: readonly string[] = [ + ...SCOPE_OPERANDS, + "frontmatter.", + "text.class.", +].toSorted(); + +function isScopeOperand(operand: string): boolean { + if (SCOPE_OPERANDS.has(operand)) return true; + return SCOPE_PREFIXES.some( + (prefix) => operand.startsWith(prefix) && operand.length > prefix.length + ); +} + +/** + * Validate one `scope` string, which is a small grammar rather than a value. + * + * Measured, the binary accepts a bare operand, a `~` negation, and operands + * chained with `&` — `~code`, `[~code]`, and `text & ~code` all parse and + * behave. Modeling `scope` as a flat enum would reject every one of those, + * which is worse than the gap it closes. + * + * **A negation over an unrecognized operand is checked as strictly as a bare + * one, and that is a deliberate disagreement with the binary.** `~fenced` does + * not fail — it fires on everything, because there is no `fenced` to subtract. + * The binary "accepts" it in the only sense an exit code can express, and what + * the author gets is a rule with its exclusion silently deleted. That is the + * exact class of failure this module exists to catch, so it is rejected here. + * `test/vale-corpus.ts` carries it as a recorded divergence rather than as a + * row the differential quietly skips. + */ +function scopeErrors(scope: string, path: string): string[] { + const errors: string[] = []; + for (const part of scope.split("&")) { + const operand = part.trim().replace(/^~/, "").trim(); + if (operand === "") { + errors.push(`${path}: empty operand in "${scope}".`); + continue; + } + if (!isScopeOperand(operand)) { + errors.push( + `${path}: "${operand}" is not a Vale ${VALE_VERSION} scope. ` + + `Vale does not reject an unknown scope — the rule loads, runs, and ` + + `matches nothing. Accepted: ${VALE_SCOPE_OPERANDS.join(", ")} ` + + `(each optionally prefixed with "~", chained with "&", or given as ` + + `a list).` + ); + } + } + return errors; +} + +/** What the schema layer concluded. Mirrors `LayerResult` in `rules/verify.ts`. */ +export interface ValeSchemaResult { + valid: boolean; + errors: string[]; +} + +/** + * Validate a parsed Vale style file structurally, before Vale is invoked. + * + * The ordering matters: `extends` is resolved first, because every other + * question depends on it. Without a known check type there is no field table to + * check against, so the function reports that one error and stops rather than + * inventing a second complaint from a table it does not have. + * + * Field names are compared **case-insensitively**, because Vale decodes them + * that way — `Tokens:` and `ignoreCase:` are read exactly as their lowercase + * spellings. The three header keys are the exception: Vale reads `extends`, + * `message` and `level` literally, so `EXTENDS:` fails there with "Missing the + * required 'extends' key", and this function reproduces that split. + */ +export function validateValeRule( + ruleId: string, + data: unknown +): ValeSchemaResult { + const errors: string[] = []; + + if (typeof data !== "object" || data === null || Array.isArray(data)) { + return { valid: false, errors: [`${ruleId}.yml is not a YAML mapping.`] }; + } + const rule = data as Record; + + const extendsValue = rule.extends; + if (typeof extendsValue !== "string") { + errors.push(`${ruleId}.yml is missing the required 'extends' key.`); + } else if (!(VALE_CHECK_TYPES as readonly string[]).includes(extendsValue)) { + errors.push( + `${ruleId}.yml: extends "${extendsValue}" is not a Vale ${VALE_VERSION} ` + + `check type. Vale fails the whole run over this, taking every other ` + + `Vale rule's findings with it. Accepted: ` + + `${VALE_CHECK_TYPES.join(", ")}.` + ); + } + + if (typeof rule.message !== "string") { + errors.push(`${ruleId}.yml is missing the required 'message' key.`); + } + + if ( + rule.level !== undefined && + (typeof rule.level !== "string" || + !(VALE_LEVELS as readonly string[]).includes(rule.level)) + ) { + errors.push( + `${ruleId}.yml has an invalid level; it must be ${VALE_LEVELS.join(", ")}.` + ); + } + + const scope = rule.scope; + if (typeof scope === "string") { + errors.push(...scopeErrors(scope, `${ruleId}.yml: scope`)); + } else if (Array.isArray(scope)) { + for (const [index, entry] of scope.entries()) { + if (typeof entry === "string") { + errors.push(...scopeErrors(entry, `${ruleId}.yml: scope[${index}]`)); + } else { + errors.push( + `${ruleId}.yml: scope[${index}] must be a string, not ${typeof entry}.` + ); + } + } + } else if (scope !== undefined) { + errors.push(`${ruleId}.yml: scope must be a string or a list of strings.`); + } + + // Only once `extends` is known: the field table is keyed on it, and a check + // type we do not have cannot say which fields are foreign. + const check = extendsValue as ValeCheckType; + const perCheck = CHECK_FIELDS[check]; + if (perCheck !== undefined && !VALE_PERMISSIVE_CHECKS.includes(check)) { + const allowed = new Set( + [...COMMON_FIELDS, ...perCheck].map((field) => field.toLowerCase()) + ); + for (const key of Object.keys(rule)) { + if (!allowed.has(key.toLowerCase())) { + errors.push( + `${ruleId}.yml: '${key}' is not a field of the ${check} check. Vale ` + + `reports this as E201 and reads one config for the whole run, so ` + + `it suppresses every other Vale rule's findings. ${check} accepts: ` + + `${[...COMMON_FIELDS, ...perCheck].toSorted().join(", ")}.` + ); + } + } + } + + return { valid: errors.length === 0, errors }; +} diff --git a/packages/cli/test/vale-corpus.ts b/packages/cli/test/vale-corpus.ts new file mode 100644 index 00000000..e2d89fb5 --- /dev/null +++ b/packages/cli/test/vale-corpus.ts @@ -0,0 +1,628 @@ +/** + * The corpus that makes `src/schemas/vale-rule.ts` true. + * + * The schema is a hand transcription — Vale publishes no JSON Schema, and the + * machine-readable field knowledge sits behind its paid hosted MCP. What makes + * a transcription trustworthy is not care in the writing; it is this table. + * Every entry is a minimal rule plus a document the rule must flag, run through + * **both** the vendored binary and the schema in + * `vale-schema-contract.test.ts`, asserting the two agree. + * + * ## Why a verdict is not an exit code + * + * Vale does not report "this is not a scope". An unrecognized `scope` parses + * clean and produces a rule that matches nothing — from outside, identical to a + * valid rule whose pattern did not fire. So there are **three** outcomes, not + * two: + * + * | Verdict | What the binary did | + * | ------------ | ------------------------------------------------------- | + * | `accepted` | the rule fired on its control | + * | `ignored` | exit 0, no finding — the construct was silently dropped | + * | `rejected` | the run failed (`E201` or a key/value error) | + * + * `ignored` is the verdict the whole change exists for, and it is the one that + * can be produced accidentally. A control document with no table makes a + * `scope: table.cell` entry read `ignored` when the scope is perfectly valid — + * an entry that "passes" while asserting nothing. Two guards, both in the test: + * + * 1. **Reach.** A `scope: raw` existence rule over `simply` must fire on every + * control, which is why every control document below contains that word. It + * proves the document reached Vale at all — right extension, parseable + * format, matched glob — independently of the construct under test. + * 2. **Attribution.** Every `ignored` entry carries a `proof`: the same rule + * with the construct replaced by a known-valid one, which must fire on the + * same control. Only then is "did not fire" attributable to the construct + * rather than to the fixture. + * + * Guard 1 is not theoretical. `figure.caption` first measured `ignored`, + * because Vale drops everything inside a `
` element — so did a + * `scope: text` control over the same document. Without the reach guard the + * operand would have been dropped from the schema as one Vale ignores, and + * every rule using it would then have failed `verify`. + * + * ## Keep it a table + * + * A Vale upgrade should be a re-run, not a re-authoring, and a gap in coverage + * should be visible as a missing row rather than as an absent test nobody + * notices. Add rows; do not add per-case tests. + */ + +/** What the vendored binary was measured doing with the construct. */ +export type ValeVerdict = "accepted" | "ignored" | "rejected"; + +export interface ValeCorpusEntry { + /** Stable identifier, printed by a failing differential. */ + name: string; + /** The construct under test, in the failure message's words. */ + construct: string; + /** The complete style file. */ + rule: string; + /** The document the rule must flag when the construct is valid. */ + control: string; + /** Extension of the control document; decides which parser Vale runs. */ + ext?: string; + /** + * Required for `ignored`: the same rule with the construct replaced by a + * known-valid one. It must fire, or "did not fire" proves nothing. + */ + proof?: string; + /** The measured verdict. */ + expected: ValeVerdict; + /** + * A deliberate disagreement with the binary, and why. The differential + * asserts the *disagreement* for these rows rather than agreement, so an + * exception is a row you can count rather than a silent carve-out. + */ + divergence?: string; +} + +// --- Rule builders ----------------------------------------------------------- +// Small enough that the table below stays readable, mechanical enough that a +// row still says exactly what it tests. + +const existence = (extra = "", token = "simply"): string => + `extends: existence\nmessage: "x %s"\nlevel: warning\n${extra}tokens:\n - ${token}\n`; + +/** An existence rule at some scope. The only variable is the scope. */ +const scoped = (scope: string): string => existence(`scope: ${scope}\n`); + +/** The known-valid variant every `ignored` scope row is attributed against. */ +const SCOPE_PROOF = scoped("raw"); + +// --- Control documents ------------------------------------------------------- +// Every one contains `simply`, so the reach guard can be generated rather than +// written per row. + +const PROSE = "Just simply do it.\n"; +const INLINE_CODE = "Run `simply now` here.\n"; +const THREE_PLACES = "Prose simply.\n\n```\nsimply fenced\n```\n"; +const MIXED = "Prose simply here and `simply code`.\n"; +const HEADINGS = "# Do simply things\n\n## Another simply heading\n"; +const TABLE = "| Head |\n| --- |\n| simply |\n"; +const FRONTMATTER = "---\ntitle: simply meta\n---\n\nBody text here.\n"; +const JS_COMMENTS = "// simply line\n/*\n simply block\n*/\nconst x = 1;\n"; + +const heading = (level: number): string => + `${"#".repeat(level)} Level ${String(level)} simply heading\n`; + +// --- The corpus -------------------------------------------------------------- + +/** + * Every check type Vale accepts, each with a control it fires on, plus the + * rejection of one it does not. + * + * The twelfth row is the point of the group: the docs enumerate eleven, + * folding `readability` into `metric`, and the binary does not. + */ +const CHECK_TYPES: ValeCorpusEntry[] = [ + { + name: "extends/existence", + construct: "extends: existence", + rule: existence(), + control: PROSE, + expected: "accepted", + }, + { + name: "extends/substitution", + construct: "extends: substitution", + rule: 'extends: substitution\nmessage: "x %s %s"\nlevel: warning\nswap:\n utilize: use\n', + control: "We utilize it simply.\n", + expected: "accepted", + }, + { + name: "extends/occurrence", + construct: "extends: occurrence", + rule: 'extends: occurrence\nmessage: "x"\nlevel: warning\nscope: sentence\ntoken: very\nmax: 1\n', + control: "It is very very very good, simply put.\n", + expected: "accepted", + }, + { + name: "extends/consistency", + construct: "extends: consistency", + rule: 'extends: consistency\nmessage: "x %s"\nlevel: warning\neither:\n advisor: adviser\n', + control: "The advisor spoke simply. The adviser left.\n", + expected: "accepted", + }, + { + name: "extends/conditional", + construct: "extends: conditional", + rule: + 'extends: conditional\nmessage: "x %s"\nlevel: warning\nscope: text\nignorecase: false\n' + + "first: '\\b([A-Z]{3,5})\\b'\nsecond: '(?:\\b[A-Z][a-z]+ )+\\(([A-Z]{3,5})\\)'\n", + control: "The ABC is here simply and nobody defined it.\n", + expected: "accepted", + }, + { + name: "extends/capitalization", + construct: "extends: capitalization", + rule: 'extends: capitalization\nmessage: "x %s"\nlevel: warning\nscope: heading\nmatch: $title\nstyle: AP\n', + control: "# this is a simply heading of things\n", + expected: "accepted", + }, + { + name: "extends/metric", + construct: "extends: metric", + rule: 'extends: metric\nmessage: "x"\nlevel: warning\nformula: |\n (characters / words)\ncondition: "> 1"\n', + control: "Antidisestablishmentarianism prevails simply everywhere.\n", + expected: "accepted", + }, + { + name: "extends/spelling", + construct: "extends: spelling", + rule: 'extends: spelling\nmessage: "x %s"\nlevel: warning\n', + control: "This is definately speled simply wrongly.\n", + expected: "accepted", + }, + { + name: "extends/readability", + construct: "extends: readability", + rule: 'extends: readability\nmessage: "x %s"\nlevel: warning\nmetrics:\n - Gunning Fog\ngrade: 1\n', + control: + "The multifaceted epistemological ramifications of antidisestablishmentarianism, " + + "notwithstanding their considerable complexity, remain fundamentally incomprehensible " + + "to the uninitiated observer who lacks the requisite philosophical grounding. This is " + + "simply not an accessible document by any conventional measurement.\n", + expected: "accepted", + }, + { + name: "extends/script", + construct: "extends: script", + rule: + 'extends: script\nmessage: "x"\nlevel: warning\nscript: |\n' + + ' text := import("text")\n matches := []\n for i, line in text.split(scope, "\\n") {\n' + + ' idx := text.index(line, "badword")\n if idx > -1 {\n' + + " matches = append(matches, {begin: idx, end: idx + 7})\n }\n }\n", + control: "This has a badword in it, simply put.\n", + expected: "accepted", + }, + { + name: "extends/sequence", + construct: "extends: sequence", + rule: 'extends: sequence\nmessage: "x"\nlevel: warning\nignorecase: true\ntokens:\n - pattern: a\n - tag: NN\n', + control: "This is a dog and a simply cat.\n", + expected: "accepted", + }, + { + name: "extends/repetition", + construct: "extends: repetition", + rule: "extends: repetition\nmessage: \"x '%s'\"\nlevel: warning\nalpha: true\ntokens:\n - '[^\\s]+'\n", + control: "This is is a simply test.\n", + expected: "accepted", + }, + { + name: "extends/unknown", + construct: "extends: nonsense", + rule: 'extends: nonsense\nmessage: "x"\nlevel: warning\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, + { + name: "extends/wrong-case", + construct: "extends: Existence", + rule: 'extends: Existence\nmessage: "x"\nlevel: warning\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, +]; + +/** Every `scope` operand the schema enumerates, each on a control it reaches. */ +const SCOPES: ValeCorpusEntry[] = [ + { name: "scope/text", scope: "text", control: PROSE }, + { name: "scope/code", scope: "code", control: INLINE_CODE }, + { name: "scope/raw", scope: "raw", control: THREE_PLACES }, + { name: "scope/heading", scope: "heading", control: HEADINGS }, + { name: "scope/heading.h1", scope: "heading.h1", control: heading(1) }, + { name: "scope/heading.h2", scope: "heading.h2", control: heading(2) }, + { name: "scope/heading.h3", scope: "heading.h3", control: heading(3) }, + { name: "scope/heading.h4", scope: "heading.h4", control: heading(4) }, + { name: "scope/heading.h5", scope: "heading.h5", control: heading(5) }, + { name: "scope/heading.h6", scope: "heading.h6", control: heading(6) }, + { + name: "scope/paragraph", + scope: "paragraph", + control: "A simply paragraph here.\n", + }, + { + name: "scope/sentence", + scope: "sentence", + control: "A simply sentence here.\n", + }, + { name: "scope/list", scope: "list", control: "- simply one\n- two\n" }, + { + name: "scope/blockquote", + scope: "blockquote", + control: "> A simply quote.\n", + }, + { + name: "scope/link", + scope: "link", + control: "See [simply link](https://example.com).\n", + }, + { name: "scope/alt", scope: "alt", control: "![simply alt text](x.png)\n" }, + { + name: "scope/summary", + scope: "summary", + control: + "
simply summary\n\nbody\n\n
\n", + }, + { + name: "scope/strong", + scope: "strong", + control: "This is **simply bold** text.\n", + }, + { + name: "scope/emphasis", + scope: "emphasis", + control: "This is *simply italic* text.\n", + }, + { name: "scope/table", scope: "table", control: TABLE }, + { + name: "scope/table.header", + scope: "table.header", + control: "| simply |\n| --- |\n| body |\n", + }, + { name: "scope/table.cell", scope: "table.cell", control: TABLE }, + { + name: "scope/table.caption", + scope: "table.caption", + control: + "
simply caption
body
\n", + ext: "html", + }, + { + // Bare, not nested in
: Vale drops everything inside that element, + // including from `scope: text` and `scope: raw`. See the module comment. + name: "scope/figure.caption", + scope: "figure.caption", + control: "
simply caption
\n", + ext: "html", + }, + { name: "scope/frontmatter", scope: "frontmatter", control: FRONTMATTER }, + { + name: "scope/frontmatter.", + scope: "frontmatter.title", + control: FRONTMATTER, + }, + { + name: "scope/text.class.", + scope: "text.class.foo", + control: '

simply here

\n', + ext: "html", + }, + { name: "scope/comment", scope: "comment", control: JS_COMMENTS, ext: "js" }, + { + name: "scope/comment.line", + scope: "comment.line", + control: "// simply line\nconst x = 1;\n", + ext: "js", + }, + { + name: "scope/comment.block", + scope: "comment.block", + control: "/*\n simply block\n*/\nconst x = 1;\n", + ext: "js", + }, + // The operators. `scope` is a grammar, not an enum: a flat enum would reject + // every one of these, which is worse than the gap the schema closes. + { name: "scope/negation", scope: "~code", control: MIXED }, + { name: "scope/chain", scope: "text & ~code", control: MIXED }, +].map(({ name, scope, control, ext }) => ({ + name, + construct: `scope: ${scope}`, + rule: scoped(scope), + control, + ...(ext === undefined ? {} : { ext }), + expected: "accepted" as const, +})); + +/** The list form, which cannot go through `scoped()`. */ +const SCOPE_LISTS: ValeCorpusEntry[] = [ + { + name: "scope/list-form", + construct: "scope: [code, text]", + rule: existence("scope:\n - code\n - text\n"), + control: MIXED, + expected: "accepted", + }, + { + name: "scope/list-form-negated", + construct: "scope: [~code]", + rule: existence("scope:\n - ~code\n"), + control: MIXED, + expected: "accepted", + }, +]; + +/** + * Scopes the binary silently drops. This is the failure the change exists for: + * exit 0, no error, a rule that is inert forever. + * + * Each carries a `proof` — the same rule at `scope: raw` — so "did not fire" is + * attributable to the scope and not to the fixture. + */ +const INVALID_SCOPES: ValeCorpusEntry[] = [ + { name: "scope/fenced", scope: "fenced", control: THREE_PLACES }, + { name: "scope/nonsense", scope: "banana", control: PROSE }, + { name: "scope/heading.h7", scope: "heading.h7", control: heading(1) }, + { name: "scope/table.row", scope: "table.row", control: TABLE }, + // Named by Vale's own docs and by this change's first draft. It does not + // exist: the v3.18.0 addition is `frontmatter`, above. + { name: "scope/meta", scope: "meta", control: FRONTMATTER }, + { name: "scope/meta.class", scope: "meta.class.title", control: FRONTMATTER }, +].map(({ name, scope, control }) => ({ + name, + construct: `scope: ${scope}`, + rule: scoped(scope), + control, + proof: SCOPE_PROOF, + expected: "ignored" as const, +})); + +/** + * The one place the schema deliberately disagrees with the binary. + * + * `~fenced` does not fail — it fires on everything, because there is no + * `fenced` to subtract. The binary "accepts" it in the only sense an exit code + * can express, and what the author gets is a rule whose exclusion was silently + * deleted. Rejecting it is the whole point of the module, so the row asserts + * the disagreement rather than hiding it in a skip list. + */ +const DIVERGENCES: ValeCorpusEntry[] = [ + { + name: "scope/negated-unknown", + construct: "scope: ~fenced", + rule: scoped("~fenced"), + control: MIXED, + expected: "accepted", + divergence: + "A negation over an operand Vale does not know is a no-op: the rule " + + "fires on everything, having silently lost the exclusion it was written " + + "for. The schema rejects it.", + }, +]; + +/** + * Per-check fields: one foreign field per strict check, and the valid fields + * whose absence from an earlier draft would have blocked real rules. + * + * A foreign field is `E201`, and Vale reads one assembled config per run — so + * this is the defect with the widest blast radius, taking every other Vale + * rule's findings down with it. + */ +const FIELDS: ValeCorpusEntry[] = [ + // Foreign fields. + { + name: "field/existence+swap", + construct: "swap on an existence check", + rule: existence("swap:\n utilize: use\n"), + control: PROSE, + expected: "rejected", + }, + { + name: "field/substitution+tokens", + construct: "tokens on a substitution check", + rule: 'extends: substitution\nmessage: "x %s %s"\nlevel: warning\nswap:\n utilize: use\ntokens:\n - simply\n', + control: "We utilize it simply.\n", + expected: "rejected", + }, + { + name: "field/occurrence+tokens", + construct: "tokens on an occurrence check", + rule: 'extends: occurrence\nmessage: "x"\nlevel: warning\nscope: sentence\ntoken: very\nmax: 1\ntokens:\n - simply\n', + control: "It is very very very good, simply put.\n", + expected: "rejected", + }, + { + name: "field/capitalization+tokens", + construct: "tokens on a capitalization check", + rule: 'extends: capitalization\nmessage: "x %s"\nlevel: warning\nscope: heading\nmatch: $title\nstyle: AP\ntokens:\n - simply\n', + control: "# this is a simply heading of things\n", + expected: "rejected", + }, + { + name: "field/capitalization+prefixes", + construct: "prefixes (plural) on a capitalization check", + rule: 'extends: capitalization\nmessage: "x %s"\nlevel: warning\nscope: heading\nmatch: $title\nstyle: AP\nprefixes:\n - "Note: "\n', + control: "# this is a simply heading of things\n", + expected: "rejected", + }, + { + name: "field/occurrence+exceptions", + construct: "exceptions on an occurrence check", + rule: 'extends: occurrence\nmessage: "x"\nlevel: warning\nscope: sentence\ntoken: very\nmax: 1\nexceptions:\n - Foo\n', + control: "It is very very very good, simply put.\n", + expected: "rejected", + }, + { + name: "field/metric+tokens", + construct: "tokens on a metric check", + rule: 'extends: metric\nmessage: "x"\nlevel: warning\nformula: |\n (characters / words)\ncondition: "> 1"\ntokens:\n - simply\n', + control: "Antidisestablishmentarianism prevails simply everywhere.\n", + expected: "rejected", + }, + { + name: "field/readability+formula", + construct: "formula on a readability check", + rule: 'extends: readability\nmessage: "x"\nlevel: warning\nmetrics:\n - Gunning Fog\ngrade: 1\nformula: |\n (characters / words)\n', + control: "This document is simply written.\n", + expected: "rejected", + }, + { + name: "field/sequence+swap", + construct: "swap on a sequence check", + rule: 'extends: sequence\nmessage: "x"\nlevel: warning\ntokens:\n - pattern: a\n - tag: NN\nswap:\n utilize: use\n', + control: "This is a dog and a simply cat.\n", + expected: "rejected", + }, + { + name: "field/repetition+swap", + construct: "swap on a repetition check", + rule: "extends: repetition\nmessage: \"x '%s'\"\nlevel: warning\nalpha: true\ntokens:\n - '[^\\s]+'\nswap:\n utilize: use\n", + control: "This is is a simply test.\n", + expected: "rejected", + }, + { + name: "field/script+tokens", + construct: "tokens on a script check", + rule: 'extends: script\nmessage: "x"\nlevel: warning\nscript: |\n matches := []\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, + { + name: "field/conditional+tokens", + construct: "tokens on a conditional check", + rule: + 'extends: conditional\nmessage: "x %s"\nlevel: warning\nscope: text\n' + + "first: '\\b([A-Z]{3,5})\\b'\nsecond: '(?:\\b[A-Z][a-z]+ )+\\(([A-Z]{3,5})\\)'\ntokens:\n - simply\n", + control: "The ABC is here simply.\n", + expected: "rejected", + }, + // Valid fields. Each of these would be rejected by a schema that transcribed + // the docs' common header and stopped there, which is the "too strict" + // failure — worse than the gap being closed, because it blocks work. + { + name: "field/existence+nonword", + construct: "nonword on a punctuation-only token", + rule: existence("nonword: true\n", "'—'"), + control: "This is simply a sentence — with an em dash.\n", + expected: "accepted", + }, + { + name: "field/existence+limit", + construct: "limit", + rule: existence("limit: 1\n"), + control: PROSE, + expected: "accepted", + }, + { + name: "field/existence+vocab", + construct: "vocab", + rule: existence("vocab: false\n"), + control: PROSE, + expected: "accepted", + }, + { + name: "field/existence+link", + construct: "link", + rule: existence("link: https://example.com\n"), + control: PROSE, + expected: "accepted", + }, + { + name: "field/existence+mixed-case-key", + construct: "Tokens, spelled with a capital", + rule: 'extends: existence\nmessage: "x %s"\nlevel: warning\nTokens:\n - simply\n', + control: PROSE, + expected: "accepted", + }, + { + name: "field/capitalization+prefix", + construct: "prefix (singular) on a capitalization check", + rule: 'extends: capitalization\nmessage: "x %s"\nlevel: warning\nscope: heading\nmatch: $title\nstyle: AP\nprefix: "Note: "\n', + control: "# this is a simply heading of things\n", + expected: "accepted", + }, + { + name: "field/occurrence+min", + construct: "min on an occurrence check", + rule: 'extends: occurrence\nmessage: "x"\nlevel: warning\nscope: sentence\ntoken: very\nmax: 1\nmin: 1\n', + control: "It is very very very good, simply put.\n", + expected: "accepted", + }, + { + name: "field/substitution+exceptions", + construct: "exceptions on a substitution check", + rule: 'extends: substitution\nmessage: "x %s %s"\nlevel: warning\nswap:\n utilize: use\nexceptions:\n - Foo\n', + control: "We utilize it simply.\n", + expected: "accepted", + }, + // The two checks that validate nothing. A schema that was strict here would + // reject rules the binary runs happily. + { + name: "field/consistency+unknown", + construct: "an unknown field on a consistency check", + rule: 'extends: consistency\nmessage: "x %s"\nlevel: warning\nbananafield: true\neither:\n advisor: adviser\n', + control: "The advisor spoke simply. The adviser left.\n", + expected: "accepted", + }, + { + name: "field/spelling+unknown", + construct: "an unknown field on a spelling check", + rule: 'extends: spelling\nmessage: "x %s"\nlevel: warning\nbananafield: true\n', + control: "This is definately speled simply wrongly.\n", + expected: "accepted", + }, +]; + +/** The header keys, which Vale reads literally rather than case-insensitively. */ +const HEADER: ValeCorpusEntry[] = [ + { + name: "header/level-unknown", + construct: "level: bananas", + rule: 'extends: existence\nmessage: "x"\nlevel: bananas\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, + { + name: "header/level-wrong-case", + construct: "level: WARNING", + rule: 'extends: existence\nmessage: "x"\nlevel: WARNING\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, + { + name: "header/missing-message", + construct: "no message key", + rule: "extends: existence\nlevel: warning\ntokens:\n - simply\n", + control: PROSE, + expected: "rejected", + }, + { + name: "header/missing-extends", + construct: "no extends key", + rule: 'message: "x"\nlevel: warning\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, + { + name: "header/extends-wrong-case-key", + construct: "EXTENDS, spelled with capitals", + rule: 'EXTENDS: existence\nmessage: "x"\nlevel: warning\ntokens:\n - simply\n', + control: PROSE, + expected: "rejected", + }, +]; + +export const VALE_CORPUS: readonly ValeCorpusEntry[] = [ + ...CHECK_TYPES, + ...SCOPES, + ...SCOPE_LISTS, + ...INVALID_SCOPES, + ...DIVERGENCES, + ...FIELDS, + ...HEADER, +]; + +/** The reach guard's rule: it must fire on every control in the corpus. */ +export const REACH_PROBE = existence("scope: raw\nnonword: true\n"); diff --git a/packages/cli/test/vale-schema-contract.test.ts b/packages/cli/test/vale-schema-contract.test.ts new file mode 100644 index 00000000..36151546 --- /dev/null +++ b/packages/cli/test/vale-schema-contract.test.ts @@ -0,0 +1,299 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { beforeAll, describe, expect, it } from "vitest"; +import { parse as yamlParse } from "yaml"; + +import { findValeBinary } from "../src/rules/vale/binary"; +import { VALE_VERSION } from "../src/rules/capabilities"; +import { VALE_CHECK_TYPES, validateValeRule } from "../src/schemas/vale-rule"; +import { + REACH_PROBE, + VALE_CORPUS, + type ValeCorpusEntry, + type ValeVerdict, +} from "./vale-corpus"; + +/** + * The differential that makes `src/schemas/vale-rule.ts` true. + * + * The schema is a hand transcription of a vendored binary's accepted values, + * because Vale publishes no JSON Schema and its machine-readable field + * knowledge is behind a paid hosted MCP. A transcription is worth exactly what + * holds it to the thing transcribed, and that is this file: for every row of + * `vale-corpus.ts`, run the rule through the binary and through the schema and + * assert the two agree. + * + * Both directions of disagreement fail, and they are reported separately + * because they mean different things: + * + * - **Schema too lax** — a rule the binary will not honor verifies clean. That + * is the gap this change exists to close. + * - **Schema too strict** — a rule the binary accepts is rejected. Worse: it + * blocks work that would have functioned. + * + * Like `vale-vendor-contract.test.ts`, this invokes the vendored binary + * directly rather than through `runVale`. A test that went through the wrapper + * would be asserting our interpretation of Vale, which is the thing under test + * everywhere else. + */ + +const binary = findValeBinary().path; +const withVale = binary === undefined ? describe.skip : describe; + +/** Where the corpus rule id lands, and therefore what `verify` would call it. */ +const RULE_ID = "corpus"; + +function runOne(rule: string, control: string, extension: string): ValeVerdict { + const cwd = mkdtempSync(join(tmpdir(), "vale-corpus-")); + try { + mkdirSync(join(cwd, "styles", RULE_ID), { recursive: true }); + writeFileSync(join(cwd, "styles", RULE_ID, `${RULE_ID}.yml`), rule); + writeFileSync( + join(cwd, ".vale.ini"), + // `BasedOnStyles =` is load-bearing: without it a control could trip a + // bundled style and be read as the rule under test firing. + `StylesPath = styles\nMinAlertLevel = suggestion\n\n[*]\nBasedOnStyles =\n${RULE_ID}.${RULE_ID} = YES\n` + ); + writeFileSync(join(cwd, `doc.${extension}`), control); + + const result = spawnSync( + binary as string, + [ + "--config", + ".vale.ini", + "--output=JSON", + "--no-exit", + "--", + `doc.${extension}`, + ], + { cwd, encoding: "utf8" } + ); + + // `--no-exit` suppresses the exit code Vale returns merely for *finding* + // something. A non-zero status therefore means the config itself failed — + // `E201`, an unknown `extends`, a bad `level`. That is `rejected`. + if (result.status !== 0) return "rejected"; + + const payload: unknown = JSON.parse(result.stdout || "{}"); + // The error shape is an array; the finding shape is a map of file to + // findings. A parsed array here means Vale reported a problem while still + // exiting zero, which is still a rejection. + if (Array.isArray(payload)) return "rejected"; + + const findings = Object.values(payload as Record).flat(); + return findings.length > 0 ? "accepted" : "ignored"; + } finally { + rmSync(cwd, { recursive: true, force: true }); + } +} + +interface Measured { + entry: ValeCorpusEntry; + /** What the binary did with the rule under test. */ + binary: ValeVerdict; + /** Whether the reach probe fired on the control at all. */ + reach: ValeVerdict; + /** Whether the known-valid variant fired, for `ignored` rows. */ + proof?: ValeVerdict; +} + +const measured: Measured[] = []; + +withVale("Vale schema contract", () => { + beforeAll(() => { + for (const entry of VALE_CORPUS) { + const extension = entry.ext ?? "md"; + measured.push({ + entry, + binary: runOne(entry.rule, entry.control, extension), + reach: runOne(REACH_PROBE, entry.control, extension), + ...(entry.proof === undefined + ? {} + : { proof: runOne(entry.proof, entry.control, extension) }), + }); + } + }, 120_000); + + describe("the corpus is not vacuous", () => { + it("has a row for every check type the schema enumerates", () => { + // A check type with no row is a value the schema asserts and nothing + // measured. This is also the version-bump tripwire: a Vale release that + // renames or drops a check type fails here, naming it. + const covered = new Set( + VALE_CORPUS.flatMap((entry) => { + const match = /^extends: (\w+)$/.exec(entry.construct); + return match === null ? [] : [match[1]]; + }) + ); + for (const check of VALE_CHECK_TYPES) { + expect(covered, `no corpus row extends ${check}`).toContain(check); + } + }); + + it("reaches every control document", () => { + // The guard against a row that passes because Vale never linted the + // fixture at all — wrong extension, unparseable format, unmatched glob. + // Without it, `ignored` cannot be told apart from "not linted", and an + // entry asserts nothing while looking green. + for (const { entry, reach } of measured) { + expect( + reach, + `${entry.name}: the reach probe did not fire on this control, so ` + + `its verdict says nothing about ${entry.construct}` + ).toBe("accepted"); + } + }); + + it("attributes every `ignored` verdict to its construct", () => { + // A control with no table would make `scope: table.cell` read `ignored` + // for a perfectly valid scope. The proof is the same rule with the + // construct replaced by a known-valid one: if it fires and the entry does + // not, the difference is the construct. + for (const { entry, proof } of measured) { + if (entry.expected !== "ignored") continue; + expect( + entry.proof, + `${entry.name}: an \`ignored\` row needs a proof rule` + ).toBeDefined(); + expect( + entry.proof, + `${entry.name}: the proof must differ from the rule under test` + ).not.toBe(entry.rule); + expect( + proof, + `${entry.name}: the known-valid variant did not fire either, so ` + + `"did not fire" is not attributable to ${entry.construct}` + ).toBe("accepted"); + } + }); + + it("has a unique name per row", () => { + const names = VALE_CORPUS.map((entry) => entry.name); + expect(new Set(names).size, "duplicate corpus row name").toBe( + names.length + ); + }); + }); + + describe("the binary still does what the corpus recorded", () => { + it("agrees with every recorded verdict", () => { + // Vale 3.18.0 is pinned. When VALE_VERSION is raised and a construct's + // treatment changed, this is where it surfaces — naming the construct + // rather than leaving the schema quietly wrong. + const drift = measured + .filter(({ entry, binary: verdict }) => verdict !== entry.expected) + .map( + ({ entry, binary: verdict }) => + `${entry.name} (${entry.construct}): recorded ${entry.expected}, ` + + `Vale ${VALE_VERSION} says ${verdict}` + ); + expect(drift, drift.join("\n")).toEqual([]); + }); + }); + + describe("the schema agrees with the binary", () => { + it("is not too lax: every rule the binary will not honor is rejected", () => { + const tooLax = measured + .filter(({ entry, binary: verdict }) => { + if (entry.divergence !== undefined) return false; + return ( + verdict !== "accepted" && + validateValeRule(RULE_ID, parse(entry)).valid + ); + }) + .map( + ({ entry, binary: verdict }) => + `${entry.name}: Vale ${verdict} ${entry.construct}, schema accepted it` + ); + expect(tooLax, tooLax.join("\n")).toEqual([]); + }); + + it("is not too strict: every rule the binary honors is accepted", () => { + // The worse direction. A schema that rejects a working rule blocks work + // that would have functioned, which is a bigger cost than the gap it was + // added to close. + const tooStrict = measured + .filter(({ entry, binary: verdict }) => { + if (entry.divergence !== undefined) return false; + return ( + verdict === "accepted" && + !validateValeRule(RULE_ID, parse(entry)).valid + ); + }) + .map(({ entry }) => { + const { errors } = validateValeRule(RULE_ID, parse(entry)); + return `${entry.name}: Vale honors ${entry.construct}, schema said: ${errors.join("; ")}`; + }); + expect(tooStrict, tooStrict.join("\n")).toEqual([]); + }); + + it("diverges only where a row says so, and says why", () => { + // The carve-outs are countable rather than silent. A row claiming a + // divergence that no longer exists fails here too. + for (const { entry, binary: verdict } of measured) { + if (entry.divergence === undefined) continue; + const schemaAccepts = validateValeRule(RULE_ID, parse(entry)).valid; + expect( + schemaAccepts, + `${entry.name} claims a divergence but agrees with Vale` + ).not.toBe(verdict === "accepted"); + expect(entry.divergence.length).toBeGreaterThan(40); + } + }); + }); + + describe("errors an author can act on", () => { + it("names the field and the accepted values for an unknown extends", () => { + const { errors } = validateValeRule( + "demo", + parseYaml('extends: nonsense\nmessage: "x"\n') + ); + expect(errors.join("\n")).toContain("extends"); + for (const check of VALE_CHECK_TYPES) { + expect(errors.join("\n")).toContain(check); + } + }); + + it("names the field and the accepted operands for an unknown scope", () => { + const { errors } = validateValeRule( + "demo", + parseYaml( + 'extends: existence\nmessage: "x"\nscope: fenced\ntokens: [a]\n' + ) + ); + expect(errors.join("\n")).toContain("scope"); + expect(errors.join("\n")).toContain("fenced"); + expect(errors.join("\n")).toContain("heading.h1"); + }); + + it("names the check when a field belongs to another one", () => { + const { errors } = validateValeRule( + "demo", + parseYaml( + 'extends: occurrence\nmessage: "x"\ntoken: a\nmax: 1\ntokens: [a]\n' + ) + ); + expect(errors.join("\n")).toContain("tokens"); + expect(errors.join("\n")).toContain("occurrence"); + expect(errors.join("\n")).toContain("E201"); + }); + }); +}); + +// --- Helpers ----------------------------------------------------------------- + +/** + * The corpus stores rules as YAML text, because that is what Vale reads and + * what an author writes. The schema takes the parsed value, exactly as + * `verifyOneRule` hands it over. + */ +function parseYaml(source: string): unknown { + return yamlParse(source) as unknown; +} + +function parse(entry: ValeCorpusEntry): unknown { + return parseYaml(entry.rule); +} diff --git a/packages/cli/test/vale-vendor-contract.test.ts b/packages/cli/test/vale-vendor-contract.test.ts index 649e6866..661d4741 100644 --- a/packages/cli/test/vale-vendor-contract.test.ts +++ b/packages/cli/test/vale-vendor-contract.test.ts @@ -20,6 +20,7 @@ import { valeMarkupList, valePlaintextList, } from "../src/rules/capabilities"; +import { VALE_CHECK_TYPES } from "../src/schemas/vale-rule"; /** * Vale's observable behaviour, pinned. @@ -703,3 +704,68 @@ withVale("Vale engine capabilities", () => { } }); }); + +/** The set Vale prints when rejecting an unknown `extends`. */ +function enumeratedCheckTypes(): string[] { + const cwd = project( + `${header}\n[*.md]\nrules.bogus = YES\n`, + { bogus: 'extends: nonsense\nmessage: "x"\nlevel: warning\n' }, + { "doc.md": "Just simply do it.\n" } + ); + const result = runRaw(cwd, ["doc.md"], ["--no-exit"]); + const message = `${result.stdout}${result.stderr}`; + const listed = /'extends' key must be one of \[([^\]]+)]/.exec(message); + const names = listed?.[1]; + if (names === undefined) { + throw new Error(`Vale no longer enumerates its check types: ${message}`); + } + return names.split(/\s+/).filter(Boolean).toSorted(); +} + +/** + * The check-type vocabulary, straight from the binary. + * + * Separate from the corpus in `vale-schema-contract.test.ts`, which measures + * *behavior* — that a rule of each type fires. This asks the binary to + * enumerate the set itself, which it does when handed an `extends` it does not + * know. It is the version-bump tripwire: a Vale release that adds, drops or + * renames a check type fails here and names the value, rather than leaving + * `VALE_CHECK_TYPES` quietly wrong. + */ +withVale("check types", () => { + it("rejects an unknown extends instead of ignoring it", () => { + // Depended on by: the schema layer's claim that this defect has the same + // blast radius as E201 rather than being the silent case. If Vale ever + // starts ignoring an unknown `extends`, the rule becomes inert instead of + // fatal and the recipe's wording is wrong. + const cwd = project( + `${header}\n[*.md]\nrules.bogus = YES\nrules.no-simply = YES\n`, + { + bogus: 'extends: nonsense\nmessage: "x"\nlevel: warning\n', + "no-simply": existence("simply"), + }, + { "doc.md": "Just simply do it.\n" } + ); + const result = runRaw(cwd, ["doc.md"], ["--no-exit"]); + expect(result.status).not.toBe(0); + // And it takes the other rule down with it: one config per run. + expect(result.stdout).not.toContain("no-simply"); + }); + + it("enumerates exactly the check types the schema encodes", () => { + // Depended on by: VALE_CHECK_TYPES in src/schemas/vale-rule.ts. A value + // there that Vale does not list makes `verify` accept a rule that takes + // the whole run down; a value Vale lists that is missing there makes + // `verify` reject a rule that works. + expect(enumeratedCheckTypes()).toEqual([...VALE_CHECK_TYPES].toSorted()); + }); + + it("counts twelve, not the eleven the documentation enumerates", () => { + // The docs fold `readability` into `metric`. They are separate checks with + // disjoint fields, and each rejects the other's — see the corpus. + const enumerated = enumeratedCheckTypes(); + expect(enumerated).toHaveLength(12); + expect(enumerated).toContain("readability"); + expect(enumerated).toContain("metric"); + }); +}); diff --git a/packages/cli/test/verify-test-commands.test.ts b/packages/cli/test/verify-test-commands.test.ts index 40f0634c..e66279e5 100644 --- a/packages/cli/test/verify-test-commands.test.ts +++ b/packages/cli/test/verify-test-commands.test.ts @@ -232,3 +232,180 @@ withVale("test runs verify first", () => { ).toContain("half a claim"); }); }); + +/** + * The schema layer, through the real CLI. + * + * `vale-schema-contract.test.ts` holds the schema to the binary. These say the + * layer is actually wired into `verify` — that an author running the command + * hears about the defect, and hears it in words that name the field. + * + * Every rule here is one Vale would have accepted into its config and then + * failed to honor. Two of the three take the *whole run* down when they reach + * the binary, so catching them at `verify` is not a convenience. + */ +/** Verify one style file, assert it failed, and hand back what it said. */ +async function verifyErrors(style: string): Promise { + await valeRule("no-simply", { config: SCOPED, style }); + const result = await runCli([ + "verify", + ".taskless/rules/vale/no-simply", + "-d", + cwd, + "--json", + ]); + const report = JSON.parse(result.stdout) as Report; + expect(report.ok).toBe(false); + expect(result.exitCode).not.toBe(0); + return report.rules[0]?.errors.join("\n") ?? ""; +} + +withVale("verify schema-checks a Vale rule", () => { + it("rejects an extends that is not a check type, naming the accepted set", async () => { + const errors = await verifyErrors( + `extends: nonsense\nmessage: "x"\nlevel: warning\ntokens:\n - simply\n` + ); + expect(errors).toContain("extends"); + expect(errors).toContain("nonsense"); + // The value of naming them: an author who skipped `readability` after + // reading the docs' eleven needs to see that it is a real check type. + expect(errors).toContain("readability"); + expect(errors).toContain("existence"); + }); + + it("rejects an unrecognized scope", async () => { + // The one defect nothing downstream catches: Vale loads this rule, runs + // it, and matches nothing, with no error at any layer. + const errors = await verifyErrors( + `extends: existence\nmessage: "x"\nlevel: warning\nscope: fenced\ntokens:\n - simply\n` + ); + expect(errors).toContain("scope"); + expect(errors).toContain("fenced"); + }); + + it("accepts ~ negation and & chaining over recognized operands", async () => { + await valeRule("no-simply", { + config: SCOPED, + style: `extends: existence\nmessage: "x"\nlevel: warning\nscope: text & ~code\ntokens:\n - simply\n`, + }); + const result = await runCli([ + "verify", + ".taskless/rules/vale/no-simply", + "-d", + cwd, + "--json", + ]); + expect((JSON.parse(result.stdout) as Report).ok).toBe(true); + expect(result.exitCode).toBe(0); + }); + + it("rejects a field belonging to another check type", async () => { + const errors = await verifyErrors( + `extends: occurrence\nmessage: "x"\nlevel: warning\nscope: sentence\ntoken: very\nmax: 1\ntokens:\n - simply\n` + ); + expect(errors).toContain("tokens"); + expect(errors).toContain("occurrence"); + // The blast radius is the reason this one is worth catching early, so the + // message has to carry it. + expect(errors).toContain("E201"); + }); + + it("still verifies a rule with no fixtures", async () => { + // The split between `verify` and `test` exists for the agent part-way + // through authoring. Adding a schema layer must not quietly close it. + await valeRule("no-simply", { config: SCOPED }); + const result = await runCli([ + "verify", + ".taskless/rules/vale/no-simply", + "-d", + cwd, + "--json", + ]); + expect((JSON.parse(result.stdout) as Report).ok).toBe(true); + expect(result.exitCode).toBe(0); + }); + + it("verifies the rule shapes the recipe teaches", async () => { + // The regression guard for "no rule that works today starts failing". + // Each of these is a worked example from `create-vale-rule`. + const recipeRules: Record = { + hedging: `extends: existence\nmessage: "Avoid hedging: '%s'"\nlevel: warning\nignorecase: true\ntokens:\n - we think\n`, + products: `extends: substitution\nmessage: "Use '%s' instead of '%s'"\nlevel: error\nignorecase: true\nswap:\n github: GitHub\n`, + headings: `extends: capitalization\nmessage: "'%s' should be in sentence case"\nlevel: warning\nscope: heading\nmatch: $sentence\nexceptions:\n - Taskless\n`, + linktext: `extends: existence\nmessage: "Link text '%s' says nothing"\nlevel: warning\nscope: link\nignorecase: true\ntokens:\n - click here\n`, + bangs: `extends: occurrence\nmessage: "Too many exclamation marks"\nlevel: warning\nscope: paragraph\ntoken: "!"\nmax: 1\n`, + doubled: `extends: repetition\nmessage: "'%s' is repeated"\nlevel: warning\nalpha: true\ntokens:\n - '[^\\s]+'\n`, + izeise: `extends: consistency\nmessage: "Use '%s' consistently"\nlevel: warning\nnonword: true\neither:\n organize: organise\n`, + acronyms: `extends: conditional\nmessage: "'%s' has no definition"\nlevel: warning\nscope: text\nignorecase: false\nfirst: '\\b([A-Z]{3,5})\\b'\nsecond: '(?:\\b[A-Z][a-z]+ )+\\(([A-Z]{3,5})\\)'\n`, + emdash: `extends: existence\nmessage: "Use a comma, not an em dash"\nlevel: warning\nnonword: true\ntokens:\n - '—'\n`, + }; + for (const [id, style] of Object.entries(recipeRules)) { + await valeRule(id, { + style, + config: `[*.md]\ntskl) rule = ${id}\nBasedOnStyles =\n${id}.${id} = YES\n`, + }); + } + + const result = await runCli([ + "verify", + ".taskless/rules/vale", + "-d", + cwd, + "--json", + ]); + const report = JSON.parse(result.stdout) as Report; + const failed = report.rules.filter((rule) => !rule.ok); + expect( + failed.map((rule) => `${rule.ruleId}: ${rule.errors.join("; ")}`) + ).toEqual([]); + expect(report.rules).toHaveLength(Object.keys(recipeRules).length); + }); +}); + +/** + * The ordering property, for the defects that are not local. + * + * A foreign field is `E201` and an unknown `extends` fails the config outright. + * Vale reads one assembled config per run, so either one reaching the binary + * takes down every Vale rule in the project. `verify` is a pure read of the + * rule's own files, and `test` runs it first and stops — so neither defect can + * get as far as a Vale invocation. + */ +withVale("a rule Vale would choke on never reaches Vale", () => { + it("stops `test` at verify for a foreign field, without running the fixtures", async () => { + await valeRule("no-simply", { + config: SCOPED, + fixtures: true, + style: `extends: occurrence\nmessage: "x"\nlevel: warning\nscope: sentence\ntoken: very\nmax: 1\ntokens:\n - simply\n`, + }); + const result = await runCli(["test", "-d", cwd, "--json"]); + expect(result.exitCode).not.toBe(0); + const rule = ( + JSON.parse(result.stdout) as { + rules: { ok: boolean; ran: boolean; errors: string[] }[]; + } + ).rules[0]; + // `ran: false` is the assertion that matters: the fixtures were never + // linted, so Vale was never asked to load this config. + expect(rule?.ran).toBe(false); + expect(rule?.errors.join(" ")).toContain("occurrence"); + // And the fixture complaint does not crowd out the real one. + expect(rule?.errors.join(" ")).not.toContain("did not fire"); + }); + + it("stops `test` at verify for an unknown extends", async () => { + await valeRule("no-simply", { + config: SCOPED, + fixtures: true, + style: `extends: nonsense\nmessage: "x"\nlevel: warning\ntokens:\n - simply\n`, + }); + const result = await runCli(["test", "-d", cwd, "--json"]); + const rule = ( + JSON.parse(result.stdout) as { + rules: { ran: boolean; errors: string[] }[]; + } + ).rules[0]; + expect(rule?.ran).toBe(false); + expect(rule?.errors.join(" ")).toContain("nonsense"); + }); +}); From 41bafebcb971f644906b0228e70ea3c9f485dce7 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 22:38:48 -0700 Subject: [PATCH 08/17] refactor(cli): express the Vale schema in zod, like the ast-grep one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `validateValeRule` was a hand-rolled walker pushing template strings into an error array, so the CLI had two validation styles for two engines — one producing `ZodIssue`s, one producing strings — against a spec that asks both to fail the same way. The per-check field tables are now a `z.discriminatedUnion` on `extends`: ten `z.strictObject`s, so `E201` is a property of the schema's shape rather than a hand-written key walk, and two `z.looseObject`s for `consistency` and `spelling`, which the binary was measured accepting any key at all on. A new `schemas/layer.ts` turns a zod parse into the `{ valid, errors }` shape `verify` reports, and the ast-grep path now goes through it too — the two differ only in how an issue becomes a line, which is a parameter. Two things zod cannot express on its own, and both stay explicit: `scope` is a grammar over the operands rather than an enum, so it is a `.check()` with a small parser; and the deliberate rejection of a negation over an unknown operand is a business rule, so it says so in its own message rather than borrowing the generic one. `test/vale-corpus.ts` is untouched and all 82 rows still pass, which is the property that makes this safe: the corpus asserts `schemaAccepts === binaryAccepts` against the vendored binary, so a zod schema that came out stricter or laxer than the walker would have named the row rather than passing quietly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- packages/cli/src/rules/verify.ts | 14 +- packages/cli/src/schemas/layer.ts | 49 +++ packages/cli/src/schemas/vale-rule.ts | 512 +++++++++++++++++--------- 3 files changed, 384 insertions(+), 191 deletions(-) create mode 100644 packages/cli/src/schemas/layer.ts diff --git a/packages/cli/src/rules/verify.ts b/packages/cli/src/rules/verify.ts index 281066b7..a2bf329e 100644 --- a/packages/cli/src/rules/verify.ts +++ b/packages/cli/src/rules/verify.ts @@ -11,6 +11,7 @@ import { TASKLESS_REQUIRED_FIELDS, findRegexWithoutKind, } from "../schemas/ast-grep-rule"; +import { pathPrefixed, schemaLayer } from "../schemas/layer"; import { ensureTasklessDirectory } from "../filesystem/directory"; import { assembleSgConfig } from "./assemble"; import { @@ -104,14 +105,11 @@ export function getSchemaPayload(): Record { // --- Layer 1: Schema validation --- function validateSchema(ruleData: unknown): LayerResult { - const result = astGrepRuleSchema.safeParse(ruleData); - if (result.success) { - return { valid: true, errors: [] }; - } - const errors = result.error.issues.map( - (issue) => `${issue.path.join(".")}: ${issue.message}` - ); - return { valid: false, errors }; + // Shared with the Vale path, so a rule that fails validation fails the same + // way whichever engine wrote it. `pathPrefixed` is the half that differs: + // ast-grep's messages come from an upstream JSON Schema and are generic + // ("expected string"), so the path is what makes them actionable. + return schemaLayer(astGrepRuleSchema.safeParse(ruleData), pathPrefixed); } // --- Layer 2: Taskless requirements --- diff --git a/packages/cli/src/schemas/layer.ts b/packages/cli/src/schemas/layer.ts new file mode 100644 index 00000000..cbb6eb8c --- /dev/null +++ b/packages/cli/src/schemas/layer.ts @@ -0,0 +1,49 @@ +import type { z } from "zod"; + +/** + * The verdict of a schema layer, in the shape `verify` reports. + * + * Structurally identical to `LayerResult` in `rules/verify.ts`, and deliberately + * declared here rather than imported from there: `rules/verify.ts` already + * imports from this directory, so pointing the dependency the other way would + * close a cycle. + */ +export interface SchemaLayerResult { + valid: boolean; + errors: string[]; +} + +/** + * How one issue becomes one line of `verify` output. + * + * The two engines want different things here, which is the whole reason this is + * a parameter. ast-grep validates against an upstream JSON Schema whose messages + * are generic ("Invalid input: expected string"), so the *path* is what makes + * them actionable. The Vale schema is hand-authored and every message is already + * a full sentence naming its own field, so a path prefix would only repeat it. + */ +export type IssueFormatter = (issue: z.core.$ZodIssue) => string; + +/** + * A zod parse result, as one layer of `verify`. + * + * Both engines' schema layers report through this, so a rule that fails + * validation fails the same way whichever engine wrote it — which is the + * property the spec asks for and the one that was lost while the Vale path was + * a hand-rolled walker returning strings and the ast-grep path returned + * `ZodIssue`s. + */ +export function schemaLayer( + result: z.ZodSafeParseResult, + format: IssueFormatter +): SchemaLayerResult { + if (result.success) return { valid: true, errors: [] }; + return { + valid: false, + errors: result.error.issues.map((issue) => format(issue)), + }; +} + +/** ast-grep's formatter: the path is what makes an upstream message useful. */ +export const pathPrefixed: IssueFormatter = (issue) => + `${issue.path.join(".")}: ${issue.message}`; diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts index 066924ce..9a8fea87 100644 --- a/packages/cli/src/schemas/vale-rule.ts +++ b/packages/cli/src/schemas/vale-rule.ts @@ -4,9 +4,9 @@ * Vale publishes no JSON Schema. Its repository has no `schemas/` directory, * and the machine-readable field knowledge exists only behind the hosted MCP * server at `api.vale.sh/mcp`, which is a paid product and unavailable to - * `verify`. So unlike `ast-grep-rule.ts` — which wraps an upstream artifact - * fetched per tag — everything here is a **transcription**, and a transcription - * drifts. + * `verify`. So unlike `ast-grep-rule.ts` — which runs `z.fromJSONSchema()` over + * an upstream artifact fetched per tag — everything here is a + * **transcription**, and a transcription drifts. * * What makes it trustworthy is not care in the writing. It is * `test/vale-corpus.ts`: a table of minimal rules, each with a document it must @@ -22,9 +22,29 @@ * - **Too strict** — a rule the binary accepts is rejected. That is worse: it * blocks work that would have functioned. Where a measurement is ambiguous, * accept. + * + * ## Shape + * + * Two stages, in the order the binary itself works: + * + * 1. **The header.** Vale reads `extends`, `message` and `level` literally, + * before it decodes anything else, and gives up if they are wrong. `scope` + * rides along because it is common to all twelve checks and is a grammar + * rather than a value. + * 2. **The check's own fields**, as a `z.discriminatedUnion` on `extends`. + * That is Vale's `E201` class expressed as schema shape: a strict object per + * check type, so a field belonging to another one is rejected by the union + * rather than by hand-written branching. + * + * Stage 2 runs only if stage 1 passed, which reproduces the binary: an + * `extends` it does not recognize is reported on its own, because with no check + * type there is no field table to check anything against. */ +import { z } from "zod"; + import { VALE_VERSION } from "../rules/capabilities"; +import { schemaLayer, type SchemaLayerResult } from "./layer"; /** * Vale's check types, quoted from the binary rather than the documentation. @@ -67,89 +87,20 @@ export type ValeCheckType = (typeof VALE_CHECK_TYPES)[number]; export const VALE_LEVELS = ["suggestion", "warning", "error"] as const; /** - * Fields every check accepts, whatever it extends. + * The keys Vale reads literally, before the case-insensitive field decode. * - * `vocab` is deliberately **not** here: five of the twelve reject it. It is - * listed per check instead. - */ -const COMMON_FIELDS = [ - "extends", - "message", - "level", - "scope", - "link", - "limit", - "action", - "description", - "name", -] as const; - -/** - * The fields each check accepts beyond {@link COMMON_FIELDS}, measured by - * adding each candidate to a minimal rule of that check and watching for - * `E201: has invalid keys`. + * Measured: `Tokens:` and `ignoreCase:` are read exactly as their lowercase + * spellings, but `EXTENDS:` fails with "Missing the required 'extends' key" and + * `Message:` with the same for `message`. `LEVEL: warning` is stranger still — + * it reaches the field decode as `level`, which the header reader has already + * removed, so it comes back as `E201 has invalid keys: 'level'`. * - * Three entries contradict Vale's published docs, and the binary wins: - * `capitalization` takes `prefix` **singular** and rejects `prefixes` and - * `suffixes`; `capitalization` rejects `ignorecase`; `occurrence` rejects - * `exceptions` and `vocab`. - * - * `consistency` and `spelling` are absent on purpose — see - * {@link VALE_PERMISSIVE_CHECKS}. + * {@link canonicalKeys} reproduces that split, which is why these three are + * named as a set. */ -const CHECK_FIELDS: Partial> = { - existence: [ - "tokens", - "raw", - "ignorecase", - "nonword", - "exceptions", - "append", - "vocab", - ], - substitution: [ - "swap", - "ignorecase", - "nonword", - "exceptions", - "capitalize", - "pos", - "vocab", - ], - capitalization: [ - "match", - "style", - "exceptions", - "threshold", - "indicators", - "prefix", - "vocab", - ], - occurrence: ["token", "max", "min", "ignorecase"], - repetition: ["tokens", "alpha", "ignorecase", "exceptions", "max", "vocab"], - conditional: ["first", "second", "exceptions", "ignorecase", "vocab"], - metric: ["formula", "condition"], - readability: ["metrics", "grade"], - sequence: ["tokens", "ignorecase"], - script: ["script"], -}; +const HEADER_KEYS = new Set(["extends", "message", "level"]); -/** - * The two checks that validate nothing. - * - * Measured: `bananafield: true` on a `consistency` or a `spelling` rule loads - * without complaint and is ignored. They do not use the strict decode the other - * ten do, so the binary raises no `E201` for a foreign field there. - * - * The schema follows the binary rather than the docs. Being strict here would - * reject rules that work — the "too strict" direction — to catch a typo that - * costs nothing but a rule quietly missing one of its own fields. The recipe - * says so in prose, which is the only place it can be said. - */ -export const VALE_PERMISSIVE_CHECKS: readonly ValeCheckType[] = [ - "consistency", - "spelling", -]; +// --- The scope grammar ------------------------------------------------------- /** * The `scope` operands the binary was measured honoring. @@ -222,137 +173,332 @@ function isScopeOperand(operand: string): boolean { ); } +const scopeVocabulary = + `Accepted: ${VALE_SCOPE_OPERANDS.join(", ")} (each optionally prefixed ` + + `with "~", chained with "&", or given as a list).`; + /** - * Validate one `scope` string, which is a small grammar rather than a value. + * Check one `scope` string, which is a small grammar rather than a value. * * Measured, the binary accepts a bare operand, a `~` negation, and operands * chained with `&` — `~code`, `[~code]`, and `text & ~code` all parse and - * behave. Modeling `scope` as a flat enum would reject every one of those, - * which is worse than the gap it closes. + * behave. An enum would reject every one of those, which is worse than the gap + * it closes, so the enum applies to the *operands* and this walks the operators + * around them. * - * **A negation over an unrecognized operand is checked as strictly as a bare - * one, and that is a deliberate disagreement with the binary.** `~fenced` does - * not fail — it fires on everything, because there is no `fenced` to subtract. - * The binary "accepts" it in the only sense an exit code can express, and what - * the author gets is a rule with its exclusion silently deleted. That is the - * exact class of failure this module exists to catch, so it is rejected here. + * **A negation over an unrecognized operand is rejected as strictly as a bare + * one, and that is a deliberate business rule rather than a transcription.** + * `~fenced` does not fail: it fires on everything, because there is no `fenced` + * to subtract. The binary "accepts" it in the only sense an exit code can + * express, and what the author gets is a rule with its exclusion silently + * deleted — the exact class of failure this module exists to catch. * `test/vale-corpus.ts` carries it as a recorded divergence rather than as a * row the differential quietly skips. */ -function scopeErrors(scope: string, path: string): string[] { - const errors: string[] = []; +function scopeMessages(scope: string): string[] { + const messages: string[] = []; for (const part of scope.split("&")) { + const negated = part.trim().startsWith("~"); const operand = part.trim().replace(/^~/, "").trim(); if (operand === "") { - errors.push(`${path}: empty operand in "${scope}".`); + messages.push(`scope: empty operand in "${scope}".`); continue; } - if (!isScopeOperand(operand)) { - errors.push( - `${path}: "${operand}" is not a Vale ${VALE_VERSION} scope. ` + - `Vale does not reject an unknown scope — the rule loads, runs, and ` + - `matches nothing. Accepted: ${VALE_SCOPE_OPERANDS.join(", ")} ` + - `(each optionally prefixed with "~", chained with "&", or given as ` + - `a list).` - ); - } + if (isScopeOperand(operand)) continue; + messages.push( + negated + ? `scope: "~${operand}" negates a scope Vale ${VALE_VERSION} does ` + + `not have, so it subtracts nothing and the rule fires everywhere — ` + + `the exclusion you wrote it for is silently gone. Vale does not ` + + `report this, so verify does. ${scopeVocabulary}` + : `scope: "${operand}" is not a Vale ${VALE_VERSION} scope. Vale ` + + `does not reject an unknown scope — the rule loads, runs, and ` + + `matches nothing. ${scopeVocabulary}` + ); } - return errors; + return messages; } -/** What the schema layer concluded. Mirrors `LayerResult` in `rules/verify.ts`. */ -export interface ValeSchemaResult { - valid: boolean; - errors: string[]; -} +// --- Stage 1: the header ----------------------------------------------------- /** - * Validate a parsed Vale style file structurally, before Vale is invoked. + * A YAML mapping whose header keys Vale would accept. * - * The ordering matters: `extends` is resolved first, because every other - * question depends on it. Without a known check type there is no field table to - * check against, so the function reports that one error and stops rather than - * inventing a second complaint from a table it does not have. - * - * Field names are compared **case-insensitively**, because Vale decodes them - * that way — `Tokens:` and `ignoreCase:` are read exactly as their lowercase - * spellings. The three header keys are the exception: Vale reads `extends`, - * `message` and `level` literally, so `EXTENDS:` fails there with "Missing the - * required 'extends' key", and this function reproduces that split. + * Everything here is checked against the **raw** keys, before + * {@link canonicalKeys} lowercases anything, because these are the three keys + * Vale reads literally. */ -export function validateValeRule( - ruleId: string, - data: unknown -): ValeSchemaResult { - const errors: string[] = []; +const valeHeaderSchema = z + .record(z.string(), z.unknown(), { + error: "the style file is not a YAML mapping.", + }) + .check((context) => { + const rule = context.value; + const fail = (path: PropertyKey[], message: string): void => { + context.issues.push({ code: "custom", input: rule, path, message }); + }; - if (typeof data !== "object" || data === null || Array.isArray(data)) { - return { valid: false, errors: [`${ruleId}.yml is not a YAML mapping.`] }; - } - const rule = data as Record; - - const extendsValue = rule.extends; - if (typeof extendsValue !== "string") { - errors.push(`${ruleId}.yml is missing the required 'extends' key.`); - } else if (!(VALE_CHECK_TYPES as readonly string[]).includes(extendsValue)) { - errors.push( - `${ruleId}.yml: extends "${extendsValue}" is not a Vale ${VALE_VERSION} ` + - `check type. Vale fails the whole run over this, taking every other ` + - `Vale rule's findings with it. Accepted: ` + - `${VALE_CHECK_TYPES.join(", ")}.` - ); - } + const extendsValue = rule.extends; + if (typeof extendsValue !== "string") { + fail(["extends"], "missing the required 'extends' key."); + } else if ( + !(VALE_CHECK_TYPES as readonly string[]).includes(extendsValue) + ) { + fail( + ["extends"], + `extends "${extendsValue}" is not a Vale ${VALE_VERSION} check type. ` + + `Vale fails the whole run over this, taking every other Vale rule's ` + + `findings with it. Accepted: ${VALE_CHECK_TYPES.join(", ")}.` + ); + } - if (typeof rule.message !== "string") { - errors.push(`${ruleId}.yml is missing the required 'message' key.`); - } + if (typeof rule.message !== "string") { + fail(["message"], "missing the required 'message' key."); + } - if ( - rule.level !== undefined && - (typeof rule.level !== "string" || - !(VALE_LEVELS as readonly string[]).includes(rule.level)) - ) { - errors.push( - `${ruleId}.yml has an invalid level; it must be ${VALE_LEVELS.join(", ")}.` - ); - } + if ( + rule.level !== undefined && + (typeof rule.level !== "string" || + !(VALE_LEVELS as readonly string[]).includes(rule.level)) + ) { + fail( + ["level"], + `level ${JSON.stringify(rule.level)} is not one of ` + + `${VALE_LEVELS.join(", ")}.` + ); + } - const scope = rule.scope; - if (typeof scope === "string") { - errors.push(...scopeErrors(scope, `${ruleId}.yml: scope`)); - } else if (Array.isArray(scope)) { - for (const [index, entry] of scope.entries()) { - if (typeof entry === "string") { - errors.push(...scopeErrors(entry, `${ruleId}.yml: scope[${index}]`)); - } else { - errors.push( - `${ruleId}.yml: scope[${index}] must be a string, not ${typeof entry}.` + // `scope` is common to all twelve checks, so it is stated once here rather + // than repeated across every member of the union below. + const { scope } = rule; + if (scope === undefined) return; + if (typeof scope !== "string" && !Array.isArray(scope)) { + fail(["scope"], "scope must be a string or a list of strings."); + return; + } + const operands = typeof scope === "string" ? [scope] : scope; + for (const [index, entry] of operands.entries()) { + if (typeof entry !== "string") { + fail( + ["scope", index], + `scope[${String(index)}] must be a string, not ${typeof entry}.` ); + continue; + } + for (const message of scopeMessages(entry)) { + fail(["scope", index], message); } } - } else if (scope !== undefined) { - errors.push(`${ruleId}.yml: scope must be a string or a list of strings.`); + }); + +/** + * Lowercase every field name Vale would lowercase, and no others. + * + * Vale decodes a check's own fields through a case-insensitive map, so + * `Tokens:` and `ignoreCase:` are read exactly as their lowercase spellings and + * must not be reported as unrecognized. The three header keys are the + * exception: they are read literally and removed before the field decode, so a + * differently-cased spelling is left alone here and falls through to the strict + * object below — which is what Vale does with `LEVEL: warning`, reporting + * `E201 has invalid keys: 'level'`. + */ +function canonicalKeys(rule: Record): Record { + const canonical: Record = {}; + for (const [key, value] of Object.entries(rule)) { + const lower = key.toLowerCase(); + canonical[HEADER_KEYS.has(lower) && key !== lower ? key : lower] = value; } + return canonical; +} - // Only once `extends` is known: the field table is keyed on it, and a check - // type we do not have cannot say which fields are foreign. - const check = extendsValue as ValeCheckType; - const perCheck = CHECK_FIELDS[check]; - if (perCheck !== undefined && !VALE_PERMISSIVE_CHECKS.includes(check)) { - const allowed = new Set( - [...COMMON_FIELDS, ...perCheck].map((field) => field.toLowerCase()) - ); - for (const key of Object.keys(rule)) { - if (!allowed.has(key.toLowerCase())) { - errors.push( - `${ruleId}.yml: '${key}' is not a field of the ${check} check. Vale ` + - `reports this as E201 and reads one config for the whole run, so ` + - `it suppresses every other Vale rule's findings. ${check} accepts: ` + - `${[...COMMON_FIELDS, ...perCheck].toSorted().join(", ")}.` - ); - } +// --- Stage 2: the per-check field tables ------------------------------------- + +/** + * Fields every check accepts, whatever it extends. + * + * `vocab` is deliberately absent: five of the twelve reject it, so it is listed + * per check instead. Values are `unknown` throughout — the binary decides what + * a field's type means, and guessing here would be the "too strict" failure + * against types nothing measured. + */ +const commonFields = { + extends: z.string(), + message: z.string(), + level: z.string().optional(), + scope: z.unknown(), + link: z.unknown(), + limit: z.unknown(), + action: z.unknown(), + description: z.unknown(), + name: z.unknown(), +}; + +/** The field names above, for the message a rejected field gets. */ +const COMMON_FIELD_NAMES = Object.keys(commonFields); + +/** + * One member of the union: the check's own fields, and nothing else. + * + * `z.strictObject` is what makes this worth doing — the `E201` class becomes a + * property of the schema's shape rather than a hand-written key walk. The + * custom `error` turns zod's "Unrecognized key" into the sentence an author can + * act on, because the blast radius is the reason the check exists at all. + */ +function check(name: ValeCheckType, fields: readonly string[]) { + const accepted = [...COMMON_FIELD_NAMES, ...fields].toSorted(); + return z.strictObject( + { + ...commonFields, + extends: z.literal(name), + ...Object.fromEntries(fields.map((field) => [field, z.unknown()])), + }, + { + error: (issue) => + issue.code === "unrecognized_keys" + ? `${issue.keys.map((key) => `'${key}'`).join(", ")} ` + + `${issue.keys.length === 1 ? "is not a field" : "are not fields"} ` + + `of the ${name} check. Vale reports this as E201 and reads one ` + + `config for the whole run, so it suppresses every other Vale ` + + `rule's findings. ${name} accepts: ${accepted.join(", ")}.` + : undefined, } + ); +} + +/** + * The two checks that validate nothing. + * + * Measured: `bananafield: true` on a `consistency` or a `spelling` rule loads + * without complaint and is ignored. They do not use the strict decode the other + * ten do, so the binary raises no `E201` for a foreign field there. + * + * `z.looseObject` follows the binary rather than the docs. Being strict here + * would reject rules that work — the "too strict" direction — to catch a typo + * that costs nothing but a rule quietly missing one of its own fields. The + * recipe says so in prose, which is the only place it can be said. + */ +function permissiveCheck(name: ValeCheckType) { + return z.looseObject({ ...commonFields, extends: z.literal(name) }); +} + +export const VALE_PERMISSIVE_CHECKS: readonly ValeCheckType[] = [ + "consistency", + "spelling", +]; + +/** + * The per-check field tables, measured by adding each candidate to a minimal + * rule of that check and watching for `E201: has invalid keys`. + * + * Kept as a table rather than inlined into the union below, because this *is* + * the measurement — the union is only how it gets enforced. Three entries + * contradict Vale's published docs, and the binary wins: `capitalization` takes + * `prefix` **singular** and rejects `prefixes` and `suffixes`; `capitalization` + * rejects `ignorecase`; `occurrence` rejects `exceptions` and `vocab`. + * + * `consistency` and `spelling` are absent on purpose — see + * {@link permissiveCheck}. + */ +const CHECK_FIELDS = { + existence: [ + "tokens", + "raw", + "ignorecase", + "nonword", + "exceptions", + "append", + "vocab", + ], + substitution: [ + "swap", + "ignorecase", + "nonword", + "exceptions", + "capitalize", + "pos", + "vocab", + ], + capitalization: [ + "match", + "style", + "exceptions", + "threshold", + "indicators", + "prefix", + "vocab", + ], + occurrence: ["token", "max", "min", "ignorecase"], + repetition: ["tokens", "alpha", "ignorecase", "exceptions", "max", "vocab"], + conditional: ["first", "second", "exceptions", "ignorecase", "vocab"], + metric: ["formula", "condition"], + readability: ["metrics", "grade"], + sequence: ["tokens", "ignorecase"], + script: ["script"], +} as const; + +/** + * The `E201` class, as schema shape. + * + * Ten strict objects and two loose ones, discriminated on `extends`, so a field + * belonging to another check type is rejected by the union rather than by a + * hand-written key walk. The members are spelled out rather than mapped over + * {@link CHECK_FIELDS} so that each `z.literal` survives into {@link ValeRule}. + */ +const valeBodySchema = z.discriminatedUnion( + "extends", + [ + check("existence", CHECK_FIELDS.existence), + check("substitution", CHECK_FIELDS.substitution), + check("capitalization", CHECK_FIELDS.capitalization), + check("occurrence", CHECK_FIELDS.occurrence), + check("repetition", CHECK_FIELDS.repetition), + check("conditional", CHECK_FIELDS.conditional), + check("metric", CHECK_FIELDS.metric), + check("readability", CHECK_FIELDS.readability), + check("sequence", CHECK_FIELDS.sequence), + check("script", CHECK_FIELDS.script), + permissiveCheck("consistency"), + permissiveCheck("spelling"), + ], + { + // Unreachable in practice — the header rejects an unknown `extends` first, + // with a message naming all twelve. Supplied so that if the two ever drift + // apart the fallback is still a sentence rather than "Invalid input". + error: () => `extends must be one of: ${VALE_CHECK_TYPES.join(", ")}.`, } +); + +/** A Vale style file, as the schema understands one. */ +export type ValeRule = z.infer; + +/** + * The whole rule: header, then the check's own fields. + * + * `.pipe` is doing the sequencing — zod runs the second stage only if the first + * produced no issues, which is exactly the binary's behavior. An `extends` Vale + * does not know, a missing `message`, or a bad `level` is reported on its own, + * because none of them leaves a field table to check against. + */ +export const valeRuleSchema = valeHeaderSchema + .transform(canonicalKeys) + .pipe(valeBodySchema); + +/** What the schema layer concluded. Shared with the ast-grep path. */ +export type ValeSchemaResult = SchemaLayerResult; - return { valid: errors.length === 0, errors }; +/** + * Validate a parsed Vale style file structurally, before Vale is invoked. + * + * Every message is already a full sentence naming its own field, so the + * formatter only has to say which file it is about — unlike the ast-grep path, + * where the upstream schema's generic messages need their path prepended. + */ +export function validateValeRule( + ruleId: string, + data: unknown +): ValeSchemaResult { + return schemaLayer( + valeRuleSchema.safeParse(data), + (issue) => `${ruleId}.yml: ${issue.message}` + ); } From dbbf0372391df70e2a73b962a640d86df88d599b Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 23:26:54 -0700 Subject: [PATCH 09/17] fix(cli): reject the Vale rule shapes that panic the binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-probed strict-versus-permissive across all twelve check types with a key no check has, rather than a real field borrowed from another one. The split is unchanged — `consistency` and `spelling` are the only two that validate nothing — but `sequence` reads as permissive under a careless probe and is not. Probe a tokenless `sequence` rule and Vale reports no invalid keys, so a probe grepping its output for `has invalid keys` scores it permissive. Give the rule its `tokens` and it rejects an unknown key like every other strict check. What happens without them is worse than E201: panic: interface conversion: interface {} is nil, not []interface {} The process dies, and a stack trace contains no `has invalid keys` string, so the grep sees a clean run. A measurement taken by grepping for an error string cannot tell "no error" from "no output" — every corpus verdict is read from the exit status instead. Three shapes end the process rather than reporting anything: a `sequence` with no `tokens`, a `sequence` whose `tokens` is not a list, and a `metric` with a `formula` and no `condition`. Every key in them is legal; it is the shape that is fatal, which is why they are checked after the field tables rather than inside them. A panic is a wider blast radius than E201 — E201 names a file and a line, a panic names no rule at all — so `verify` is the last place they can be caught. Four corpus rows added: `field/sequence+unknown`, which is the coverage whose absence let the question stand, and one per fatal shape. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .../design.md | 40 +++++- packages/cli/src/schemas/vale-rule.ts | 119 ++++++++++++++---- packages/cli/test/vale-corpus.ts | 65 +++++++++- 3 files changed, 196 insertions(+), 28 deletions(-) diff --git a/openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md b/openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md index fac4da30..5ce370c4 100644 --- a/openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md +++ b/openspec/changes/archive/2026-08-25-vale-authoring-polish/design.md @@ -170,10 +170,42 @@ case-sensitive too — `level: WARNING` and `extends: Existence` are both rejected. The schema therefore compares field names case-insensitively and `extends`/`level` values exactly, which is what the binary does. -**`consistency` and `spelling` accept any key at all.** `bananafield: true` on -either loads without complaint and is ignored — they do not use the strict -decode the other ten do. The schema cannot be strict about their fields without -rejecting rules the binary accepts, so it is not. +**`consistency` and `spelling` accept any key at all — and only those two.** +`bananafield: true` on either loads without complaint and is ignored; they do +not use the strict decode the other ten do. The schema cannot be strict about +their fields without rejecting rules the binary accepts, so it is not. + +The split was re-probed across all twelve with a nonsense key rather than a real +field borrowed from another check, because `sequence` reads as permissive under +a careless probe and is not: + +| Strict (10) | Permissive (2) | +| --------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `capitalization`, `conditional`, `existence`, `metric`, `occurrence`, `readability`, `repetition`, `script`, `sequence`, `substitution` | `consistency`, `spelling` | + +**Why `sequence` misleads, and why it matters more than the answer.** Probe it +with `extends: sequence` + `message` + an unknown key and Vale reports no +invalid keys — so a probe that greps its output for `has invalid keys` scores it +permissive. It is not: give the rule its `tokens` and it rejects the unknown key +like every other strict check. What happens without `tokens` is worse than an +`E201`: + +``` +panic: interface conversion: interface {} is nil, not []interface {} +``` + +The process dies. A panic contains no `has invalid keys` string, so the grep +sees a clean run. **A measurement taken by grepping for an error string cannot +tell "no error" from "no output".** Every verdict in the corpus is read from the +exit status instead. + +That probe turned up three shapes that end the process rather than reporting +anything — a `sequence` with no `tokens`, a `sequence` whose `tokens` is not a +list, and a `metric` with a `formula` and no `condition`. They are a wider blast +radius than `E201`: an `E201` names a file and a line, a panic names no rule at +all and produces no findings for anything in the project. `verify` rejects all +three, in a `checkFatalShapes` pass that runs after the field tables — a shape +can only be fatal if every key in it was legal to begin with. ### The failure modes, re-measured diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts index 9a8fea87..8b54c4b4 100644 --- a/packages/cli/src/schemas/vale-rule.ts +++ b/packages/cli/src/schemas/vale-rule.ts @@ -444,29 +444,104 @@ const CHECK_FIELDS = { * hand-written key walk. The members are spelled out rather than mapped over * {@link CHECK_FIELDS} so that each `z.literal` survives into {@link ValeRule}. */ -const valeBodySchema = z.discriminatedUnion( - "extends", - [ - check("existence", CHECK_FIELDS.existence), - check("substitution", CHECK_FIELDS.substitution), - check("capitalization", CHECK_FIELDS.capitalization), - check("occurrence", CHECK_FIELDS.occurrence), - check("repetition", CHECK_FIELDS.repetition), - check("conditional", CHECK_FIELDS.conditional), - check("metric", CHECK_FIELDS.metric), - check("readability", CHECK_FIELDS.readability), - check("sequence", CHECK_FIELDS.sequence), - check("script", CHECK_FIELDS.script), - permissiveCheck("consistency"), - permissiveCheck("spelling"), - ], - { - // Unreachable in practice — the header rejects an unknown `extends` first, - // with a message naming all twelve. Supplied so that if the two ever drift - // apart the fallback is still a sentence rather than "Invalid input". - error: () => `extends must be one of: ${VALE_CHECK_TYPES.join(", ")}.`, +/** + * Field shapes that crash the binary outright. + * + * These are not field-table facts, which is why they sit apart from the union: + * every key involved is a legal field of its check. It is the *shape* that is + * fatal. Measured against Vale 3.18.0, each of the three below ends the process + * with a Go stack trace — + * + * ``` + * panic: interface conversion: interface {} is nil, not []interface {} + * ``` + * + * — which is a wider blast radius than `E201`. An `E201` is at least a + * diagnostic naming a file and a line; a panic is a stack trace with no rule + * name in it, no findings for any rule, and nothing an author can act on. This + * is the last place `verify` can say anything useful at all. + * + * It is also the reason a probe must never be trusted to a grep. A panic + * contains no `has invalid keys` string, so a probe looking for that phrase + * scores a panicking rule as *accepted* — which is exactly how `sequence` came + * to look like a check that validates nothing. It is strict; a tokenless + * `sequence` rule simply never reaches the validation. + */ +function fatalShapeMessages( + rule: Record +): { path: PropertyKey[]; message: string }[] { + const fatal: { path: PropertyKey[]; message: string }[] = []; + + if (rule.extends === "sequence") { + if (rule.tokens === undefined) { + fatal.push({ + path: ["tokens"], + message: + "a sequence check needs 'tokens'. Without it Vale does not report " + + "an error — it panics, ending the run with a Go stack trace and no " + + "findings for any rule in the project.", + }); + } else if (!Array.isArray(rule.tokens)) { + fatal.push({ + path: ["tokens"], + message: + `a sequence check's 'tokens' must be a list, not ` + + `${typeof rule.tokens}. Vale panics on any other shape, ending the ` + + `run with a Go stack trace and no findings for any rule in the ` + + `project.`, + }); + } } -); + + if ( + rule.extends === "metric" && + rule.formula !== undefined && + rule.condition === undefined + ) { + fatal.push({ + path: ["condition"], + message: + "a metric check with a 'formula' also needs a 'condition'. Vale " + + "panics on a formula it has nothing to compare, ending the run with " + + "a Go stack trace and no findings for any rule in the project.", + }); + } + + return fatal; +} + +const valeBodySchema = z + .discriminatedUnion( + "extends", + [ + check("existence", CHECK_FIELDS.existence), + check("substitution", CHECK_FIELDS.substitution), + check("capitalization", CHECK_FIELDS.capitalization), + check("occurrence", CHECK_FIELDS.occurrence), + check("repetition", CHECK_FIELDS.repetition), + check("conditional", CHECK_FIELDS.conditional), + check("metric", CHECK_FIELDS.metric), + check("readability", CHECK_FIELDS.readability), + check("sequence", CHECK_FIELDS.sequence), + check("script", CHECK_FIELDS.script), + permissiveCheck("consistency"), + permissiveCheck("spelling"), + ], + { + // Unreachable in practice — the header rejects an unknown `extends` first, + // with a message naming all twelve. Supplied so that if the two ever drift + // apart the fallback is still a sentence rather than "Invalid input". + error: () => `extends must be one of: ${VALE_CHECK_TYPES.join(", ")}.`, + } + // Runs only once the field tables have passed, which is the right order: a + // shape can only be fatal if every key in it was legal to begin with. + ) + .check((context) => { + const rule = context.value as Record; + for (const { path, message } of fatalShapeMessages(rule)) { + context.issues.push({ code: "custom", input: rule, path, message }); + } + }); /** A Vale style file, as the schema understands one. */ export type ValeRule = z.infer; diff --git a/packages/cli/test/vale-corpus.ts b/packages/cli/test/vale-corpus.ts index e2d89fb5..bb262e2f 100644 --- a/packages/cli/test/vale-corpus.ts +++ b/packages/cli/test/vale-corpus.ts @@ -557,8 +557,23 @@ const FIELDS: ValeCorpusEntry[] = [ control: "We utilize it simply.\n", expected: "accepted", }, - // The two checks that validate nothing. A schema that was strict here would - // reject rules the binary runs happily. + // Strict versus permissive, asked the same way of all three candidates, with + // a key no check has rather than a real field borrowed from another one. + // + // `sequence` belongs with the strict ten, and the row exists because that is + // easy to get wrong: probe a *tokenless* sequence rule and Vale panics + // instead of reporting, so a probe grepping for `has invalid keys` scores it + // as permissive. Give the rule its `tokens` and it rejects the unknown key + // like every other strict check. See `shape/sequence-without-tokens` below. + { + name: "field/sequence+unknown", + construct: "an unknown field on a sequence check", + rule: 'extends: sequence\nmessage: "x"\nlevel: warning\ntokens:\n - pattern: a\n - tag: NN\nbananafield: true\n', + control: "This is a dog and a simply cat.\n", + expected: "rejected", + }, + // The two checks that really do validate nothing. A schema that was strict + // here would reject rules the binary runs happily. { name: "field/consistency+unknown", construct: "an unknown field on a consistency check", @@ -575,6 +590,51 @@ const FIELDS: ValeCorpusEntry[] = [ }, ]; +/** + * Shapes that panic the binary. + * + * Every key in these rules is a legal field of its check — it is the *shape* + * that is fatal. Measured, each ends the process with a Go stack trace: + * + * ``` + * panic: interface conversion: interface {} is nil, not []interface {} + * ``` + * + * That is a wider blast radius than `E201`. An `E201` names a file and a line; + * a panic names no rule at all, produces no findings for anything in the + * project, and gives an author nothing to act on. `verify` is the last place + * these can be caught. + * + * They are also a standing warning about how these measurements are taken. A + * panic contains no `has invalid keys` string, so any probe that greps for that + * phrase reads a panicking rule as *accepted* — which is exactly how a + * tokenless `sequence` rule came to look like proof that `sequence` validates + * nothing. The verdict below comes from the exit status, not from a grep. + */ +const SHAPES: ValeCorpusEntry[] = [ + { + name: "shape/sequence-without-tokens", + construct: "a sequence check with no tokens", + rule: 'extends: sequence\nmessage: "x"\nlevel: warning\n', + control: "This is a dog and a simply cat.\n", + expected: "rejected", + }, + { + name: "shape/sequence-tokens-not-a-list", + construct: "a sequence check whose tokens is not a list", + rule: 'extends: sequence\nmessage: "x"\nlevel: warning\ntokens: simply\n', + control: "This is a dog and a simply cat.\n", + expected: "rejected", + }, + { + name: "shape/metric-formula-without-condition", + construct: "a metric check with a formula and no condition", + rule: 'extends: metric\nmessage: "x"\nlevel: warning\nformula: |\n (characters / words)\n', + control: "Antidisestablishmentarianism prevails simply everywhere.\n", + expected: "rejected", + }, +]; + /** The header keys, which Vale reads literally rather than case-insensitively. */ const HEADER: ValeCorpusEntry[] = [ { @@ -621,6 +681,7 @@ export const VALE_CORPUS: readonly ValeCorpusEntry[] = [ ...INVALID_SCOPES, ...DIVERGENCES, ...FIELDS, + ...SHAPES, ...HEADER, ]; From d1cf12432bccea886141219a644df9a39db427ac Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Tue, 25 Aug 2026 14:11:09 -0700 Subject: [PATCH 10/17] fix(vale): catch a metric condition that is present but unusable The fatal-shape guard fired only on `rule.condition === undefined`, so `condition:` written with no value verified clean. YAML parses that to null, not undefined. Probed against the vendored Vale 3.18.0 binary: a metric check with a formula panics when its condition is absent, null, or a blank string, all three with `interface conversion: interface {} is float64, not bool`. Other types (number, bool, list, map) fail the decode cleanly with an E201 and are left to the ordinary path. The guard is now structural, like the neighbouring sequence arm, and two corpus rows record the measurement. Both rows fail against the old guard. Also wires VALE_PERMISSIVE_CHECKS into a test, so "consistency and spelling are exempt on purpose" is machine-checked rather than only described in a comment, and corrects the panic string the doc comments attributed to all three shapes. --- packages/cli/src/schemas/vale-rule.ts | 49 ++++++++++++------ packages/cli/test/vale-corpus.ts | 28 +++++++++- .../cli/test/vale-schema-contract.test.ts | 51 ++++++++++++++++++- 3 files changed, 111 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts index 8b54c4b4..281fb040 100644 --- a/packages/cli/src/schemas/vale-rule.ts +++ b/packages/cli/src/schemas/vale-rule.ts @@ -449,14 +449,21 @@ const CHECK_FIELDS = { * * These are not field-table facts, which is why they sit apart from the union: * every key involved is a legal field of its check. It is the *shape* that is - * fatal. Measured against Vale 3.18.0, each of the three below ends the process - * with a Go stack trace — + * fatal. Measured against Vale 3.18.0, each of the shapes below ends the + * process with a Go stack trace. The two `sequence` shapes fail while the rule + * is compiled: * * ``` * panic: interface conversion: interface {} is nil, not []interface {} * ``` * - * — which is a wider blast radius than `E201`. An `E201` is at least a + * and the `metric` shape fails later, while the rule runs on a document: + * + * ``` + * panic: interface conversion: interface {} is float64, not bool + * ``` + * + * Either is a wider blast radius than `E201`. An `E201` is at least a * diagnostic naming a file and a line; a panic is a stack trace with no rule * name in it, no findings for any rule, and nothing an author can act on. This * is the last place `verify` can say anything useful at all. @@ -493,18 +500,30 @@ function fatalShapeMessages( } } - if ( - rule.extends === "metric" && - rule.formula !== undefined && - rule.condition === undefined - ) { - fatal.push({ - path: ["condition"], - message: - "a metric check with a 'formula' also needs a 'condition'. Vale " + - "panics on a formula it has nothing to compare, ending the run with " + - "a Go stack trace and no findings for any rule in the project.", - }); + if (rule.extends === "metric" && rule.formula !== undefined) { + // Structural, like the `sequence` arm above, because "was the key + // supplied" is the wrong question. Measured against 3.18.0, the panic + // follows the *value*: an absent `condition`, a `condition:` written with + // no value (YAML parses that to null, which is not `undefined`), and a + // blank string all reach the same panic. Other types are not this guard's + // business: `condition: 5`, a list, a map, or a bool fails the decode + // cleanly with an `E201`, which is a diagnostic an author can read. + const condition = rule.condition; + const unusable = + condition === undefined || + condition === null || + (typeof condition === "string" && condition.trim() === ""); + if (unusable) { + fatal.push({ + path: ["condition"], + message: + "a metric check with a 'formula' also needs a 'condition' holding " + + "a comparison, such as '> 1'. An absent, empty, or blank one is " + + "the same thing to Vale: it panics on a formula it has nothing to " + + "compare, ending the run with a Go stack trace and no findings for " + + "any rule in the project.", + }); + } } return fatal; diff --git a/packages/cli/test/vale-corpus.ts b/packages/cli/test/vale-corpus.ts index bb262e2f..5167e811 100644 --- a/packages/cli/test/vale-corpus.ts +++ b/packages/cli/test/vale-corpus.ts @@ -594,12 +594,19 @@ const FIELDS: ValeCorpusEntry[] = [ * Shapes that panic the binary. * * Every key in these rules is a legal field of its check — it is the *shape* - * that is fatal. Measured, each ends the process with a Go stack trace: + * that is fatal. Measured, each ends the process with a Go stack trace. The + * `sequence` rows panic while the rule is compiled: * * ``` * panic: interface conversion: interface {} is nil, not []interface {} * ``` * + * and the `metric` rows panic later, while the rule runs on a document: + * + * ``` + * panic: interface conversion: interface {} is float64, not bool + * ``` + * * That is a wider blast radius than `E201`. An `E201` names a file and a line; * a panic names no rule at all, produces no findings for anything in the * project, and gives an author nothing to act on. `verify` is the last place @@ -633,6 +640,25 @@ const SHAPES: ValeCorpusEntry[] = [ control: "Antidisestablishmentarianism prevails simply everywhere.\n", expected: "rejected", }, + // The two rows below are the same panic reached through a *present* key, and + // they are why the schema's guard is structural rather than a check for + // `undefined`. `condition:` with nothing after the colon is the likelier of + // the two to be typed by hand, and YAML hands it over as null, not as a + // missing key. + { + name: "shape/metric-condition-null", + construct: "a metric check whose condition key has no value", + rule: 'extends: metric\nmessage: "x"\nlevel: warning\nformula: |\n (characters / words)\ncondition:\n', + control: "Antidisestablishmentarianism prevails simply everywhere.\n", + expected: "rejected", + }, + { + name: "shape/metric-condition-blank", + construct: "a metric check whose condition is a blank string", + rule: 'extends: metric\nmessage: "x"\nlevel: warning\nformula: |\n (characters / words)\ncondition: " "\n', + control: "Antidisestablishmentarianism prevails simply everywhere.\n", + expected: "rejected", + }, ]; /** The header keys, which Vale reads literally rather than case-insensitively. */ diff --git a/packages/cli/test/vale-schema-contract.test.ts b/packages/cli/test/vale-schema-contract.test.ts index 36151546..8a0a2ac6 100644 --- a/packages/cli/test/vale-schema-contract.test.ts +++ b/packages/cli/test/vale-schema-contract.test.ts @@ -8,7 +8,11 @@ import { parse as yamlParse } from "yaml"; import { findValeBinary } from "../src/rules/vale/binary"; import { VALE_VERSION } from "../src/rules/capabilities"; -import { VALE_CHECK_TYPES, validateValeRule } from "../src/schemas/vale-rule"; +import { + VALE_CHECK_TYPES, + VALE_PERMISSIVE_CHECKS, + validateValeRule, +} from "../src/schemas/vale-rule"; import { REACH_PROBE, VALE_CORPUS, @@ -283,6 +287,46 @@ withVale("Vale schema contract", () => { }); }); +/** + * The exemption, stated as a test rather than only as a comment. + * + * `consistency` and `spelling` are the two checks Vale loads without a strict + * field decode, so a foreign key on either is ignored instead of raising + * `E201`. The schema follows the binary and leaves those two loose. That is a + * deliberate hole in the `E201` net, and a hole is worth a test: without one, + * a later pass could add a field table for either check, tighten it to match + * the docs, and turn "Vale ignores this" into a rule the schema rejects and + * Vale runs. This needs no binary; it asks what the schema does. + */ +describe("the permissive checks stay permissive", () => { + it("accepts a foreign field on every permissive check", () => { + for (const check of VALE_PERMISSIVE_CHECKS) { + const { valid, errors } = validateValeRule( + "demo", + parseYaml(unknownField(check)) + ); + expect( + valid, + `${check} rejected a foreign field: ${errors.join("; ")}` + ).toBe(true); + } + }); + + it("rejects a foreign field on every other check", () => { + const permissive = new Set(VALE_PERMISSIVE_CHECKS); + for (const check of VALE_CHECK_TYPES) { + if (permissive.has(check)) continue; + const { errors } = validateValeRule( + "demo", + parseYaml(unknownField(check)) + ); + expect(errors.join("\n"), `${check} accepted a foreign field`).toContain( + "'bananafield' is not a field" + ); + } + }); +}); + // --- Helpers ----------------------------------------------------------------- /** @@ -297,3 +341,8 @@ function parseYaml(source: string): unknown { function parse(entry: ValeCorpusEntry): unknown { return parseYaml(entry.rule); } + +/** A minimal rule of `check` carrying one field that belongs to no check. */ +function unknownField(check: string): string { + return `extends: ${check}\nmessage: "x"\nbananafield: true\n`; +} From 32f559d1211f975f974e16c51bf50727057d12df Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 23:47:18 -0700 Subject: [PATCH 11/17] feat(cli): derive the Vale rule vocabulary from the vendored binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pnpm generate:vale-schema` runs the pinned Vale against rules it writes itself and emits `src/generated/vale-vocabulary.ts`, alongside a divergence report. It replaces a transcription whose measurements were real and whose measuring script was discarded — leaving the answer, not the question. Every verdict comes from the process exit status and the structured JSON output, never from matching stdout against an error phrase. A Go panic contains no `has invalid keys` string, so a phrase-grep scores a crash as a clean run, which is how a tokenless `sequence` rule once looked like a check that validates nothing. A run's outcome is a closed set — clean, diagnostic, panic, unrecognized — and the last is fatal at every call site. Two of the four vocabularies self-enumerate: an unknown `extends` or `level` makes the binary name its own accepted set. If either line stops matching, generation fails rather than emitting a short enum, because a truncated enum is stricter than the binary and strictness is the direction that blocks working rules. The other two are honest about their limit. `E201` names the key you got wrong and never the ones you could have used, and an unknown scope raises nothing at all, so field tables and scope operands are verified rather than discovered from a candidate list whose provenance is recorded. A scope verdict is three-valued, and a `scope: raw` reach probe must fire on every fixture so an operand cannot be dropped because its fixture was unreachable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- packages/cli/package.json | 1 + packages/cli/scripts/generate-vale-schema.ts | 1332 +++++++++++++++++ .../src/generated/vale-vocabulary-report.md | 112 ++ packages/cli/src/generated/vale-vocabulary.ts | 216 +++ 4 files changed, 1661 insertions(+) create mode 100644 packages/cli/scripts/generate-vale-schema.ts create mode 100644 packages/cli/src/generated/vale-vocabulary-report.md create mode 100644 packages/cli/src/generated/vale-vocabulary.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 990d2d27..eaa8ae5d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -15,6 +15,7 @@ "generate:api": "openapi-typescript https://app.taskless.io/cli/api/__schema -o src/generated/api.d.ts", "generate:ast-grep-schema": "tsx scripts/fetch-ast-grep-schema.ts", "generate:rule-hash-vectors": "tsx scripts/fetch-rule-hash-vectors.ts", + "generate:vale-schema": "tsx scripts/generate-vale-schema.ts", "prebuild": "tsx scripts/fetch-rule-hash-vectors.ts", "test": "vitest run", "typecheck": "tsc --noEmit" diff --git a/packages/cli/scripts/generate-vale-schema.ts b/packages/cli/scripts/generate-vale-schema.ts new file mode 100644 index 00000000..e1e89b5c --- /dev/null +++ b/packages/cli/scripts/generate-vale-schema.ts @@ -0,0 +1,1332 @@ +/** + * Derive the Vale rule vocabulary from the vendored binary. + * + * Vale publishes no JSON Schema, so `fetch-ast-grep-schema.ts` has nothing to + * fetch here. What it does have is a binary that answers questions, and this + * script asks them: every value in `src/generated/vale-vocabulary.ts` is the + * recorded answer of the pinned Vale to a rule this script wrote and ran. + * + * ## The method rule + * + * **Every verdict comes from the process exit status and the structured JSON + * Vale emits, never from grepping output for an error phrase.** That is not + * style advice. A probe that greps stdout for `has invalid keys` reads a Go + * panic — which contains no such string — as a clean run, and that is exactly + * how a tokenless `sequence` rule once came to look like a check that + * validates nothing. Run with `--output=JSON`; on a config error Vale writes + * `{Line, Path, Text, Code, Span}` to **stderr**. Key on `Code`, parse `Text` + * structurally, and treat every outcome that does not match a known shape as + * fatal rather than folding it into "fine". + * + * ## What is derived and what is seeded + * + * Three of the four vocabularies are self-enumerating — the binary names its + * own accepted set in the error it raises — and one is not: + * + * | Vocabulary | Oracle | Discovers? | + * | ---------------- | ----------------------------------------------------- | ---------- | + * | Check types | `'extends' key must be one of [...]` | yes | + * | Levels | `'level' must be one of [...]` | yes | + * | Per-check fields | `has invalid keys: ''` — names the *bad* key | no | + * | Scope operands | none: an unknown scope is silent | no | + * + * The bottom two rows are the honest limit of this script. `E201` names the key + * you got wrong and never the ones you could have used, so a field table can + * only be built by proposing a candidate and asking. **That verifies; it cannot + * discover.** A real field nobody proposes is omitted from the artifact, and + * the schema then rejects a rule Vale accepts — the too-strict direction, which + * the design calls the worse failure. {@link FIELD_CANDIDATES} is therefore + * seeded generously and carries its provenance. + * + * `scope` is worse still, because it has no oracle at all: an invalid scope is + * silent, and so is a valid scope whose construct is missing from the fixture. + * The verdict is three-valued, and only a hand-written fixture separates the + * last two. See {@link SCOPE_CANDIDATES}. + * + * ## Failing loudly + * + * If the enumeration line stops matching its pattern, this script errors rather + * than emitting a short enum. A truncated enum is *stricter* than the binary, + * which is the failure direction that blocks working rules — so a parse that no + * longer matches must never be allowed to look like a small vocabulary. + */ + +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const PACKAGE_ROOT = resolve(scriptDirectory, ".."); +const OUTPUT_PATH = resolve( + PACKAGE_ROOT, + "src", + "generated", + "vale-vocabulary.ts" +); +const REPORT_PATH = resolve( + PACKAGE_ROOT, + "src", + "generated", + "vale-vocabulary-report.md" +); + +// --- Locating the binary ----------------------------------------------------- + +/** + * The vendored binary, resolved the same way the CLI resolves it at runtime. + * + * `findValeBinary` lives in `src/`, which this script can import directly: it + * runs under `tsx`, not under the bundle. Using the shipped resolver rather + * than a second copy of the platform-package naming means a packaging change + * cannot leave the generator probing a different Vale than the one `verify` + * will hand rules to. + */ +const { findValeBinary } = await import("../src/rules/vale/binary.js"); +const { VALE_VERSION } = await import("../src/rules/capabilities.js"); + +const resolution = findValeBinary(); +if (resolution.path === undefined) { + throw new Error( + "no vendored Vale binary on this host, so there is nothing to derive from. " + + "Run `pnpm install` in packages/cli and retry." + ); +} +const VALE = resolution.path; + +{ + const probe = spawnSync(VALE, ["--version"], { encoding: "utf8" }); + const reported = /vale version (\d+\.\d+\.\d+)/.exec(probe.stdout ?? "")?.[1]; + if (reported !== VALE_VERSION) { + throw new Error( + `the vendored binary reports ${reported ?? "an unreadable version"} but ` + + `VALE_VERSION is pinned to ${VALE_VERSION}. Derivation against a ` + + `different binary than the one the schema claims to describe is the ` + + `drift this artifact exists to prevent.` + ); + } +} + +// --- The probe --------------------------------------------------------------- + +/** Vale's structured diagnostic, as `--output=JSON` writes it to stderr. */ +interface ValeDiagnostic { + Line: number; + Path: string; + Text: string; + Code: string; + Span: number; +} + +/** + * What one run of the binary did, as a closed set of outcomes. + * + * There is deliberately no "other" member that a caller can shrug at. Every + * shape this script does not recognize becomes {@link Unrecognized}, and every + * consumer of a verdict must decide what to do about it — which in practice + * means throwing. + */ +type ProbeOutcome = + | { kind: "clean"; findings: number } + | { kind: "diagnostic"; diagnostic: ValeDiagnostic } + | { kind: "panic"; trace: string } + | { + kind: "unrecognized"; + status: number | null; + stdout: string; + stderr: string; + }; + +/** + * Run one rule over one document in a config isolated from everything else. + * + * `BasedOnStyles =` is load-bearing. Without it Vale loads its bundled styles + * and a control document can trip one of those, which a finding count cannot + * tell apart from the rule under test firing. + * + * `--no-exit` suppresses the non-zero status Vale returns merely for *finding* + * something, so a non-zero status here means the config itself failed. + */ +function probe( + rule: string, + document: string, + extension: string +): ProbeOutcome { + const cwd = mkdtempSync(join(tmpdir(), "vale-derive-")); + try { + mkdirSync(join(cwd, "styles", "probe"), { recursive: true }); + writeFileSync(join(cwd, "styles", "probe", "probe.yml"), rule); + writeFileSync( + join(cwd, ".vale.ini"), + "StylesPath = styles\nMinAlertLevel = suggestion\n\n[*]\nBasedOnStyles =\nprobe.probe = YES\n" + ); + writeFileSync(join(cwd, `doc.${extension}`), document); + + const result = spawnSync( + VALE, + [ + "--config", + ".vale.ini", + "--output=JSON", + "--no-exit", + "--", + `doc.${extension}`, + ], + { cwd, encoding: "utf8" } + ); + + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + + if (stderr.includes("panic:")) return { kind: "panic", trace: stderr }; + + if (result.status === 0) { + const payload: unknown = JSON.parse(stdout || "{}"); + if ( + Array.isArray(payload) || + typeof payload !== "object" || + payload === null + ) { + return { kind: "unrecognized", status: result.status, stdout, stderr }; + } + const findings = Object.values( + payload as Record + ).flat(); + return { kind: "clean", findings: findings.length }; + } + + const diagnostic = readDiagnostic(stderr); + if (diagnostic !== undefined) return { kind: "diagnostic", diagnostic }; + + return { kind: "unrecognized", status: result.status, stdout, stderr }; + } finally { + rmSync(cwd, { recursive: true, force: true }); + } +} + +/** + * Read Vale's diagnostic object off stderr. + * + * Vale writes one JSON object per config error, pretty-printed and possibly + * several in a row. `JSON.parse` handles the single-object case; anything else + * returns `undefined` and becomes {@link ProbeOutcome} `unrecognized`, which + * every caller treats as fatal. That is the point: an unreadable diagnostic + * must never be mistaken for a clean run. + */ +function readDiagnostic(stderr: string): ValeDiagnostic | undefined { + const trimmed = stderr.trim(); + if (trimmed === "") return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return undefined; + } + const first: unknown = Array.isArray(parsed) ? parsed[0] : parsed; + if (typeof first !== "object" || first === null) return undefined; + const candidate = first as Partial; + if ( + typeof candidate.Text !== "string" || + typeof candidate.Code !== "string" + ) { + return undefined; + } + return candidate as ValeDiagnostic; +} + +/** + * Read a value the generator itself produced, refusing `undefined`. + * + * `noUncheckedIndexedAccess` is right to insist on this rather than being + * silenced with a `!`. A missing entry means a check type was lost between two + * stages of the derivation, and the alternative to throwing is emitting a + * vocabulary with a silently empty field table — which is the too-strict + * failure, dressed up as a successful run. + */ +function must(value: T | undefined, what: string): T { + if (value === undefined) { + throw new Error( + `internal: ${what} is missing, so the derivation is incomplete and the ` + + `artifact would be emitted short. This is a bug in the generator.` + ); + } + return value; +} + +/** A probe outcome this script has no handling for is never survivable. */ +function fatal(context: string, outcome: ProbeOutcome): never { + const detail = + outcome.kind === "panic" + ? `the binary panicked:\n${outcome.trace.split("\n").slice(0, 3).join("\n")}` + : outcome.kind === "unrecognized" + ? `unrecognized outcome (status ${String(outcome.status)}):\n` + + `stdout: ${outcome.stdout.slice(0, 400)}\nstderr: ${outcome.stderr.slice(0, 400)}` + : outcome.kind === "diagnostic" + ? `unexpected ${outcome.diagnostic.Code}: ${outcome.diagnostic.Text}` + : `unexpected clean run (${String(outcome.findings)} findings)`; + throw new Error(`${context}: ${detail}`); +} + +// --- Stage 1: the self-enumerating vocabularies ------------------------------ + +/** + * Ask the binary to name its own accepted set. + * + * Both `extends` and `level` reject an unknown value by enumerating the legal + * ones, so the probe supplies a sentinel and reads the list back out of the + * diagnostic's `Text`. Nothing here is proposed and then confirmed: the set + * arrives whole, which is why a Vale release that adds a check type is picked + * up rather than merely failing a test. + * + * The pattern is anchored on the whole line, and a miss throws. **A truncated + * enum is stricter than the binary**, so "the message changed shape" must never + * degrade into "the vocabulary got smaller". + */ +function enumerateFromBinary( + key: "extends" | "level", + rule: string, + { sorted }: { sorted: boolean } +): string[] { + const outcome = probe(rule, "bogus simply here.\n", "md"); + if (outcome.kind !== "diagnostic") { + fatal( + `enumerating '${key}': the binary did not reject the sentinel`, + outcome + ); + } + const { Code, Text } = outcome.diagnostic; + if (Code !== "E201") { + throw new Error( + `enumerating '${key}': expected diagnostic code E201, got ${Code} ` + + `("${Text}"). The generator refuses to guess at a vocabulary from an ` + + `error shape it does not know.` + ); + } + const match = new RegExp(String.raw`^'${key}'[^[\]]*\[([^\]]+)\]\.?$`).exec( + Text + ); + if (match === null) { + throw new Error( + `enumerating '${key}': Vale ${VALE_VERSION} answered "${Text}", which no ` + + `longer matches the enumeration shape this generator reads. Refusing ` + + `to emit a vocabulary: a short enum is STRICTER than the binary, so a ` + + `silent partial parse would start rejecting rules Vale accepts. Fix ` + + `the pattern against the new message.` + ); + } + const raw = must(match[1], `the '${key}' enumeration group`) + .trim() + .split(/\s+/); + if (raw.length === 0) { + throw new Error(`enumerating '${key}': the enumeration parsed as empty.`); + } + return sorted ? raw.toSorted() : raw; +} + +/** + * Check types are sorted; levels are left in the order the binary gave them. + * + * The difference is that one order carries meaning and the other does not. The + * check types come out of the binary unordered, so sorting makes the list + * `verify` prints to an author predictable. The levels come out in *severity* + * order — suggestion, warning, error — and alphabetising them would throw that + * away for nothing. + */ +const CHECK_TYPES = enumerateFromBinary( + "extends", + 'extends: __taskless_probe__\nmessage: "x"\nlevel: error\ntokens: [bogus]\n', + { sorted: true } +); +const LEVELS = enumerateFromBinary( + "level", + 'extends: existence\nmessage: "x"\nlevel: __taskless_probe__\ntokens: [bogus]\n', + { sorted: false } +); + +console.log( + `Vale ${VALE_VERSION}: ${String(CHECK_TYPES.length)} check types, ${String(LEVELS.length)} levels` +); + +// --- Stage 2: the per-check field tables ------------------------------------- + +/** + * A minimal rule per check type that the binary runs without complaint. + * + * **Hand-seeded, and it has to be.** A field probe adds one candidate key to a + * rule and asks whether the key was rejected — which is only meaningful if + * everything *else* in the rule was already fine. Several checks are not valid + * empty: a `sequence` with no `tokens` panics rather than reporting, and a + * `metric` with a `formula` and no `condition` does the same. So each base is a + * working rule of its check, and stage 2 asserts that before probing anything. + * + * Provenance: each is the minimal form of the corresponding row in + * `test/vale-corpus.ts`, which measured it accepted against this binary. + */ +const CHECK_BASES: Record> = { + existence: { tokens: ["bogus"] }, + substitution: { swap: { utilize: "use" } }, + capitalization: { match: "$title", style: "AP" }, + occurrence: { token: "very", max: 1 }, + repetition: { tokens: [String.raw`[^\s]+`] }, + conditional: { + first: String.raw`\b([A-Z]{3,5})\b`, + second: String.raw`(?:\b[A-Z][a-z]+ )+\(([A-Z]{3,5})\)`, + }, + metric: { formula: "(characters / words)", condition: "> 1" }, + readability: { metrics: ["Gunning Fog"], grade: 1 }, + sequence: { tokens: [{ pattern: "a" }] }, + script: { script: "matches := []\n" }, + consistency: { either: { advisor: "adviser" } }, + spelling: {}, +}; + +/** + * The candidate universe, and why each name is in it. + * + * **This list is the generator's one irreducible act of authorship.** `E201` + * names the key you got wrong, never the ones you could have used, so a field + * table can only be verified, never discovered. A real field absent from here + * is absent from the artifact, and the schema then rejects a rule the binary + * runs — the "too strict" failure the design ranks as the worse one. + * + * Seeded generously and from four independent sources, so that a name missing + * from one is likely present in another: + * + * 1. **Vale's published key documentation** (`vale.sh/docs/keys`), including + * names the binary turns out to reject — `prefixes`, `suffixes` and + * `ignorecase` on `capitalization` are documented and measured rejected, and + * keeping them here is what makes that a recorded finding rather than an + * omission. + * 2. **The hand transcription this artifact replaces**, so the generator is + * strictly a superset of what a human already established. + * 3. **Vale's shared/common keys** — `link`, `limit`, `action`, `scope`, + * `description`, `name`, `comment`, `vocab`. + * 4. **Neighbouring checks' fields**, deliberately cross-probed: every field of + * every check is offered to every other check, which is what turns the + * per-check tables into a measured partition rather than twelve unrelated + * lists. + */ +const FIELD_CANDIDATES: readonly string[] = [ + // Header and shared keys (source 3). + "extends", + "message", + "level", + "scope", + "link", + "limit", + "action", + "description", + "name", + "comment", + "vocab", + // Documented per-check keys (sources 1, 2 and 4 — cross-probed). + "tokens", + "token", + "raw", + "ignorecase", + "nonword", + "exceptions", + "append", + "swap", + "capitalize", + "pos", + "match", + "style", + "threshold", + "indicators", + "prefix", + "prefixes", + "suffixes", + "max", + "min", + "alpha", + "first", + "second", + "formula", + "condition", + "metrics", + "grade", + "script", + "either", + "filter", + "filters", + "ignore", + "dicpath", + "dictionaries", + "custom", + "aff", + "dic", + "negate", + "ordered", + "chars", + "pattern", + "tag", +].toSorted(); + +/** + * A key no check has, used to ask whether a check validates its keys at all. + * + * Two of the twelve do not: `consistency` and `spelling` accept any key and + * ignore it. That is measured rather than assumed, because the schema's answer + * differs in the direction that matters — a strict object over a permissive + * check rejects rules the binary runs. + */ +const SENTINEL_FIELD = "taskless_generator_sentinel"; + +/** How the binary answered "is this a key of this check?". */ +type Membership = "member" | "not-a-member"; + +/** + * Parse the invalid-key list out of a diagnostic, structurally. + * + * `has invalid keys: 'a', 'b'` is the only membership oracle Vale offers, and + * it is read as a *list of names* rather than matched as a phrase. A diagnostic + * that names a key we did not probe means the base rule is contaminated, which + * is fatal — the alternative is recording a verdict about the wrong key. + */ +function invalidKeys(text: string): string[] | undefined { + const match = /^has invalid keys: (.+)$/.exec(text); + if (match === null) return undefined; + return [...must(match[1], "the invalid-key list").matchAll(/'([^']+)'/g)].map( + (found) => must(found[1], "an invalid key name").toLowerCase() + ); +} + +function toYaml(rule: Record): string { + return ( + Object.entries(rule) + .map(([key, value]) => `${key}: ${JSON.stringify(value)}`) + .join("\n") + "\n" + ); +} + +/** + * The keys Vale reads before it decodes a check at all. + * + * They are members of every check by construction — each base rule above + * carries all three and was verified running clean — so they are never probed. + * Probing them would also be actively unsound: the probe value is arbitrary, + * and `extends: true` reaches a bare `interface{}` to `string` conversion that + * panics the process rather than reporting anything. + */ +const HEADER_FIELDS = ["extends", "message", "level"]; + +/** + * Ask one question: does `check` accept `field` as a key? + * + * The verdict is a read of a closed outcome set, never a grep. The probe sets + * the field to an arbitrary value (`true`), so every outcome has to be + * interpreted as evidence about the *key* rather than about the value: + * + * - a clean run means the key was decoded, so it is a member; + * - `has invalid keys` naming this field means it is not — this is the only + * negative oracle Vale offers, and it is parsed as a list of names; + * - **any other diagnostic means member.** Vale recognized the key and objected + * to its value or type instead, which is membership evidence; + * - **a panic means member, for the same reason.** A key Vale does not know is + * collected and reported as unused; it is never dereferenced. Reaching a + * type conversion at all proves the key was routed somewhere. + * + * The last two are inferences rather than direct readings, so both are recorded + * and printed with the artifact. An inference nobody can audit is a grep with + * better manners. + * + * An invalid-key list naming a key we did not probe, or a diagnostic this + * script cannot parse, stays fatal: both mean the verdict would be about + * something other than the question asked. + */ +const typeObjections: string[] = []; +const panicObjections: string[] = []; + +function probeField( + check: string, + field: string, + { panicIsEvidence }: { panicIsEvidence: boolean } +): Membership { + const rule: Record = { + extends: check, + message: "x %s", + level: "error", + ...CHECK_BASES[check], + [field]: true, + }; + const outcome = probe(toYaml(rule), "bogus simply here.\n", "md"); + + if (outcome.kind === "clean") return "member"; + if (outcome.kind === "panic" && panicIsEvidence) { + panicObjections.push( + `${check}.${field}: ${(outcome.trace.split("\n")[0] ?? "").trim()}` + ); + return "member"; + } + if (outcome.kind !== "diagnostic") { + fatal(`probing '${field}' on the ${check} check`, outcome); + } + + const { Code, Text } = outcome.diagnostic; + if (Code !== "E201") { + throw new Error( + `probing '${field}' on the ${check} check: diagnostic code ${Code} is ` + + `not one this generator knows how to read ("${Text}").` + ); + } + + const named = invalidKeys(Text); + if (named === undefined) { + typeObjections.push(`${check}.${field}: ${Text}`); + return "member"; + } + if (named.includes(field.toLowerCase())) return "not-a-member"; + + throw new Error( + `probing '${field}' on the ${check} check: Vale rejected ${named + .map((key) => `'${key}'`) + .join(", ")}, which is not the key under test. The base rule for ` + + `${check} is contaminated, so no verdict here is trustworthy.` + ); +} + +/** Verify the base rule before trusting a single verdict taken against it. */ +for (const check of CHECK_TYPES) { + const base = CHECK_BASES[check]; + if (base === undefined) { + throw new Error( + `Vale ${VALE_VERSION} has a '${check}' check with no base rule in ` + + `CHECK_BASES. A field table cannot be probed without a working rule ` + + `of that check to probe against — add one and re-run.` + ); + } + const outcome = probe( + toYaml({ extends: check, message: "x %s", level: "error", ...base }), + "bogus simply here.\n", + "md" + ); + if (outcome.kind !== "clean") { + fatal(`the base rule for the ${check} check is not valid`, outcome); + } +} + +const permissiveChecks: string[] = []; +const acceptedFields: Record = {}; + +for (const check of CHECK_TYPES) { + // The sentinel is the one probe where a panic is *not* read as evidence. It + // decides whether the check validates keys at all, and a permissive verdict + // taken from a crash would make the schema loose for a check that is strict. + if ( + probeField(check, SENTINEL_FIELD, { panicIsEvidence: false }) === "member" + ) { + permissiveChecks.push(check); + continue; + } + const accepted = new Set([ + ...HEADER_FIELDS, + ...Object.keys(must(CHECK_BASES[check], `the base rule for ${check}`)), + ]); + for (const field of FIELD_CANDIDATES) { + if (accepted.has(field)) continue; + if (probeField(check, field, { panicIsEvidence: true }) === "member") { + accepted.add(field); + } + } + acceptedFields[check] = [...accepted].toSorted(); + console.log(` ${check}: ${String(accepted.size)} fields`); +} + +/** + * The fields every strict check takes, derived rather than declared. + * + * This is the intersection of the twelve measured tables, which is a stronger + * statement than a hand-written "common" list: `vocab` is *not* here, because + * five checks reject it, and nothing had to remember that. The permissive + * checks are excluded from the intersection — they accept everything, so they + * constrain nothing. + */ +const strictChecks = CHECK_TYPES.filter( + (check) => !permissiveChecks.includes(check) +); +const tableFor = (check: string): string[] => + must(acceptedFields[check], `the measured field table for ${check}`); + +const commonFields = FIELD_CANDIDATES.filter((field) => + strictChecks.every((check) => tableFor(check).includes(field)) +); + +const checkFields: Record = {}; +for (const check of strictChecks) { + checkFields[check] = tableFor(check).filter( + (field) => !commonFields.includes(field) + ); +} + +// --- Stage 3: the scope operands --------------------------------------------- + +/** + * A scope candidate, its fixture, and where the name came from. + * + * **`scope` has no oracle.** Vale does not reject an unknown scope: the rule + * loads, runs, and matches nothing, which from outside is indistinguishable + * from a valid scope whose construct is absent from the document. So the + * verdict is three-valued — + * + * | Verdict | What happened | + * | --------------- | --------------------------------------------------- | + * | `fires` | the rule flagged the fixture: the operand is real | + * | `silent` | the fixture was linted and the rule found nothing | + * | `unreachable` | the fixture was never linted at all | + * + * — and only a hand-written fixture separates the middle from the last. Every + * fixture contains the word `bogus`, and a reach probe (`scope: raw`) must fire + * on it; if it does not, the fixture is broken and the run is fatal, because + * "did not fire" would then say nothing about the operand. + * + * `documented` records whether Vale's own documentation names the operand. It + * is not evidence — the binary is — but the disagreements are the report this + * script emits, and a disagreement can only be stated if both sides are here. + * + * `ext` matters more than it looks: the file extension decides which parser + * Vale routes the document to, and `comment.*` only exists in a source tier. + */ +interface ScopeCandidate { + operand: string; + fixture: string; + ext: string; + documented: boolean; + /** For a family whose tail is author-supplied, the prefix to record. */ + prefix?: string; + /** Why this candidate is worth probing, when that is not obvious. */ + note?: string; +} + +const MARKDOWN_HEADING = (level: number): string => + `${"#".repeat(level)} Level ${String(level)} bogus heading\n`; + +const PROSE = "Just bogus do it.\n"; +const FRONTMATTER = "---\ntitle: bogus meta\n---\n\nBody text here.\n"; +const JS_COMMENTS = "// bogus line\n/*\n bogus block\n*/\nconst x = 1;\n"; +const TS_COMMENTS = + "// bogus line\n/*\n bogus block\n*/\nconst x: number = 1;\n"; +const TABLE = "| Head |\n| --- |\n| bogus |\n"; + +const SCOPE_CANDIDATES: readonly ScopeCandidate[] = [ + { operand: "text", fixture: PROSE, ext: "md", documented: true }, + { + operand: "code", + fixture: "Run `bogus now` here.\n", + ext: "md", + documented: true, + }, + { + operand: "raw", + fixture: "Prose bogus.\n\n```\nbogus fenced\n```\n", + ext: "md", + documented: true, + }, + { + operand: "heading", + fixture: "# Do bogus things\n", + ext: "md", + documented: true, + }, + { + operand: "heading.h1", + fixture: MARKDOWN_HEADING(1), + ext: "md", + documented: true, + }, + { + operand: "heading.h2", + fixture: MARKDOWN_HEADING(2), + ext: "md", + documented: true, + }, + { + operand: "heading.h3", + fixture: MARKDOWN_HEADING(3), + ext: "md", + documented: true, + }, + { + operand: "heading.h4", + fixture: MARKDOWN_HEADING(4), + ext: "md", + documented: true, + }, + { + operand: "heading.h5", + fixture: MARKDOWN_HEADING(5), + ext: "md", + documented: true, + }, + { + operand: "heading.h6", + fixture: MARKDOWN_HEADING(6), + ext: "md", + documented: true, + }, + { + operand: "heading.h7", + fixture: MARKDOWN_HEADING(1), + ext: "md", + documented: false, + note: "the negative control for the heading family: HTML stops at h6.", + }, + { + operand: "paragraph", + fixture: "A bogus paragraph here.\n", + ext: "md", + documented: true, + }, + { + operand: "sentence", + fixture: "A bogus sentence here.\n", + ext: "md", + documented: true, + }, + { + operand: "list", + fixture: "- bogus one\n- two\n", + ext: "md", + documented: true, + }, + { + operand: "blockquote", + fixture: "> A bogus quote.\n", + ext: "md", + documented: true, + }, + { + operand: "link", + fixture: "See [bogus link](https://example.com).\n", + ext: "md", + documented: true, + }, + { + operand: "alt", + fixture: "![bogus alt text](x.png)\n", + ext: "md", + documented: true, + }, + { + operand: "summary", + fixture: + "
bogus summary\n\nbody\n\n
\n", + ext: "md", + documented: true, + }, + { + operand: "strong", + fixture: "This is **bogus bold** text.\n", + ext: "md", + documented: true, + }, + { + operand: "emphasis", + fixture: "This is *bogus italic* text.\n", + ext: "md", + documented: true, + }, + { operand: "table", fixture: TABLE, ext: "md", documented: true }, + { + operand: "table.header", + fixture: "| bogus |\n| --- |\n| body |\n", + ext: "md", + documented: true, + }, + { operand: "table.cell", fixture: TABLE, ext: "md", documented: true }, + { + operand: "table.caption", + fixture: + "
bogus caption
body
\n", + ext: "html", + documented: true, + }, + { + operand: "table.row", + fixture: TABLE, + ext: "md", + documented: false, + note: "the negative control for the table family.", + }, + { + operand: "figure.caption", + fixture: "
bogus caption
\n", + ext: "html", + documented: true, + note: + "bare, NOT nested in
. Vale drops everything inside a
" + + "element, so the nested form is unreachable — see the divergence report.", + }, + { + operand: "meta", + fixture: FRONTMATTER, + ext: "md", + documented: true, + note: "documented by Vale; measured never firing. See the divergence report.", + }, + { + operand: "meta.class.title", + fixture: FRONTMATTER, + ext: "md", + documented: true, + prefix: "meta.class.", + note: "documented by Vale; measured never firing. See the divergence report.", + }, + { + operand: "frontmatter", + fixture: FRONTMATTER, + ext: "md", + documented: false, + note: "documented nowhere and measured working — the standing proof that this generator verifies rather than discovers.", + }, + { + operand: "frontmatter.title", + fixture: FRONTMATTER, + ext: "md", + documented: false, + prefix: "frontmatter.", + }, + { + operand: "text.class.foo", + fixture: '

bogus here

\n', + ext: "html", + documented: true, + prefix: "text.class.", + }, + { operand: "comment", fixture: JS_COMMENTS, ext: "js", documented: true }, + { + operand: "comment.line", + fixture: "// bogus line\nconst x = 1;\n", + ext: "js", + documented: true, + }, + { + operand: "comment.block", + fixture: "/*\n bogus block\n*/\nconst x = 1;\n", + ext: "js", + documented: true, + }, + { + operand: "comment.line", + fixture: TS_COMMENTS, + ext: "ts", + documented: true, + note: "the same operand in the TypeScript tier, which routes through a different parser.", + }, + { + operand: "comment.block", + fixture: TS_COMMENTS, + ext: "ts", + documented: true, + note: "the same operand in the TypeScript tier.", + }, + { + operand: "fenced", + fixture: "Prose bogus.\n\n```\nbogus fenced\n```\n", + ext: "md", + documented: false, + note: "the negative control an author is most likely to reach for.", + }, + { + operand: "banana", + fixture: PROSE, + ext: "md", + documented: false, + note: "the negative control that cannot possibly be a scope.", + }, +]; + +type ScopeVerdict = "fires" | "silent" | "unreachable"; + +/** The reach probe: proves a fixture was linted at all, independent of scope. */ +const REACH_RULE = + 'extends: existence\nmessage: "x %s"\nlevel: error\nscope: raw\nnonword: true\ntokens:\n - bogus\n'; + +interface ScopeMeasurement { + candidate: ScopeCandidate; + verdict: ScopeVerdict; +} + +const scopeMeasurements: ScopeMeasurement[] = []; + +for (const candidate of SCOPE_CANDIDATES) { + const rule = + 'extends: existence\nmessage: "x %s"\nlevel: error\n' + + `scope: ${JSON.stringify(candidate.operand)}\ntokens:\n - bogus\n`; + + const reach = probe(REACH_RULE, candidate.fixture, candidate.ext); + if (reach.kind !== "clean") { + fatal(`the reach probe on the ${candidate.operand} fixture`, reach); + } + if (reach.findings === 0) { + throw new Error( + `scope '${candidate.operand}': the reach probe did not fire on its ` + + `.${candidate.ext} fixture, so the fixture was never linted — wrong ` + + `extension, unparseable format, or an unmatched glob. Any verdict ` + + `taken here would say nothing about the operand.` + ); + } + + const outcome = probe(rule, candidate.fixture, candidate.ext); + if (outcome.kind !== "clean") { + fatal(`probing scope '${candidate.operand}'`, outcome); + } + const verdict: ScopeVerdict = outcome.findings > 0 ? "fires" : "silent"; + scopeMeasurements.push({ candidate, verdict }); + console.log(` scope ${candidate.operand} (.${candidate.ext}): ${verdict}`); +} + +/** + * The operands the schema will honor: the ones measured firing, nothing else. + * + * An operand that never fired for any fixture is not in the artifact, whatever + * the documentation says about it. That is the whole asymmetry `scope` is + * subject to — a scope Vale does not have costs an author a rule that is inert + * forever, with no error anywhere. + */ +const firing = scopeMeasurements.filter(({ verdict }) => verdict === "fires"); +const scopeOperands = [ + ...new Set( + firing + .filter(({ candidate }) => candidate.prefix === undefined) + .map(({ candidate }) => candidate.operand) + ), +].toSorted(); +const scopePrefixes = [ + ...new Set( + firing + .filter(({ candidate }) => candidate.prefix !== undefined) + .map(({ candidate }) => candidate.prefix as string) + ), +].toSorted(); + +// --- Stage 4: the divergence report ------------------------------------------ + +/** + * Where the binary and the documentation disagree. + * + * These are the findings a generator must never drop silently. A documented + * operand that never fires is a trap an author walks into with the docs open; + * an undocumented one that works is the standing proof that a candidate list + * verifies rather than discovers, and that something real could be missing from + * it. Both go into a checked-in report, not into a console line nobody reads. + */ +interface Divergence { + subject: string; + finding: string; + consequence: string; +} + +const divergences: Divergence[] = []; + +const byOperand = new Map(); +for (const measurement of scopeMeasurements) { + const existing = byOperand.get(measurement.candidate.operand) ?? []; + existing.push(measurement); + byOperand.set(measurement.candidate.operand, existing); +} + +for (const [operand, measurements] of byOperand) { + const documented = measurements.some(({ candidate }) => candidate.documented); + const fires = measurements.some(({ verdict }) => verdict === "fires"); + const negativeControl = measurements.every(({ candidate }) => + (candidate.note ?? "").includes("negative control") + ); + + if (documented && !fires) { + divergences.push({ + subject: `scope: ${operand}`, + finding: + `Vale ${VALE_VERSION} documents this operand and it never fired, on ` + + `any fixture probed (${measurements + .map(({ candidate }) => `.${candidate.ext}`) + .join(", ")}).`, + consequence: + "It is omitted from the vocabulary, so `verify` rejects it. A rule " + + "written from the documentation would otherwise load, run, and match " + + "nothing, with no error reported anywhere.", + }); + } + + if (!documented && fires && !negativeControl) { + divergences.push({ + subject: `scope: ${operand}`, + finding: `This operand fired and Vale ${VALE_VERSION} documents it nowhere.`, + consequence: + "It is included in the vocabulary. It is also the standing " + + "counterexample to trusting the candidate list: a real operand nobody " + + "proposes is simply absent, and the schema then rejects a rule the " + + "binary honors.", + }); + } + + // A partial firing — the same operand alive in one tier and dead in another — + // is a divergence in its own right, and the one most likely to be read as a + // broken fixture rather than as a property of the binary. + const alive = measurements.filter(({ verdict }) => verdict === "fires"); + const dead = measurements.filter(({ verdict }) => verdict !== "fires"); + if (alive.length > 0 && dead.length > 0) { + divergences.push({ + subject: `scope: ${operand}`, + finding: + `Fires in ${alive.map(({ candidate }) => `.${candidate.ext}`).join(", ")} ` + + `and is silent in ${dead.map(({ candidate }) => `.${candidate.ext}`).join(", ")}, ` + + `on fixtures the reach probe confirmed were linted in both cases.`, + consequence: + "The operand stays in the vocabulary — it is real — but it is not " + + "portable across formats, and `verify` cannot tell an author which " + + "tier they are in.", + }); + } +} + +for (const check of permissiveChecks) { + divergences.push({ + subject: `extends: ${check}`, + finding: + `This check accepted '${SENTINEL_FIELD}', a key no check has. It does ` + + "not validate its keys at all.", + consequence: + "The schema is permissive here. Being strict would reject rules the " + + "binary runs, to catch a typo that costs nothing but a field quietly " + + "ignored — the too-strict direction, which is the worse failure.", + }); +} + +if (typeObjections.length > 0) { + divergences.push({ + subject: "field probes: membership inferred from a type complaint", + finding: + `${String(typeObjections.length)} probes drew an E201 that was not an ` + + `invalid-key list: ${typeObjections.join("; ")}.`, + consequence: + "Each is recorded as a member: Vale recognized the key and objected to " + + "the probe's arbitrary value instead, which is membership evidence. " + + "They are listed so the inference is auditable rather than assumed.", + }); +} + +if (panicObjections.length > 0) { + divergences.push({ + subject: "field probes: membership inferred from a crash", + finding: + `${String(panicObjections.length)} probes panicked the binary: ` + + `${panicObjections.join("; ")}.`, + consequence: + "Each is recorded as a member. A key Vale does not know is collected " + + "and reported as unused, never dereferenced — so reaching a type " + + "conversion at all proves the key was routed somewhere. This is the " + + "weakest inference the generator makes, which is why it is named here.", + }); +} + +// --- Emit -------------------------------------------------------------------- + +function literalList(values: readonly string[]): string { + return values.map((value) => ` ${JSON.stringify(value)},`).join("\n"); +} + +const artifact = `/** + * The Vale rule vocabulary, derived from the vendored binary. + * + * GENERATED FILE — DO NOT EDIT. Run \`pnpm generate:vale-schema\` in + * \`packages/cli\` to reproduce it. The generator is + * \`scripts/generate-vale-schema.ts\`, and its header explains what each value + * below was measured with and what it is worth. + * + * Derived against Vale ${VALE_VERSION}. Every value here is the recorded answer + * of that binary to a rule the generator wrote and ran; nothing is transcribed + * from documentation. Where the binary and the documentation disagree, the + * disagreement is in \`vale-vocabulary-report.md\` rather than dropped. + * + * \`src/schemas/vale-rule.ts\` turns these into the zod schema \`verify\` runs. + */ + +/** The binary this vocabulary was derived from. */ +export const VALE_VOCABULARY_VERSION = ${JSON.stringify(VALE_VERSION)}; + +/** + * Vale's check types, self-enumerated: an unknown \`extends\` makes the binary + * name the whole set, so this is discovered rather than proposed. + */ +export const VALE_CHECK_TYPES = [ +${literalList(CHECK_TYPES)} +] as const; + +/** + * Vale's severities, self-enumerated the same way, and left in the binary's + * own order because that order is severity. The value is case-sensitive. + */ +export const VALE_LEVELS = [ +${literalList(LEVELS)} +] as const; + +/** + * The checks that accept any key at all and ignore what they do not know. + * + * Measured by offering each check a sentinel key no check has. + */ +export const VALE_PERMISSIVE_CHECKS = [ +${literalList(permissiveChecks.toSorted())} +] as const; + +/** + * The fields every strict check accepts, as the intersection of their measured + * tables rather than as a declared list. + * + * Note what the intersection excludes: \`vocab\` is per-check, because several + * checks reject it. + */ +export const VALE_COMMON_FIELDS = [ +${literalList(commonFields)} +] as const; + +/** + * Each strict check's own fields — its measured table minus the common ones. + * + * Membership only: \`E201\` names the key you got wrong and never the ones you + * could have used, so every name here was proposed by the generator's candidate + * list and confirmed. A real field nobody proposed is absent. + */ +export const VALE_CHECK_FIELDS = { +${strictChecks + .toSorted() + .map( + (check) => + ` ${check}: [\n${must( + checkFields[check], + `the emitted table for ${check}` + ) + .map((field) => ` ${JSON.stringify(field)},`) + .join("\n")}\n ],` + ) + .join("\n")} +} as const; + +/** + * The \`scope\` operands measured firing on a fixture carrying their construct. + * + * \`scope\` has no oracle — an unknown scope is silent, and so is a valid scope + * with no construct to match — so an operand is here only if a rule using it + * flagged a fixture that a reach probe independently confirmed was linted. + */ +export const VALE_SCOPE_OPERANDS = [ +${literalList(scopeOperands)} +] as const; + +/** + * Scope families whose tail is author-supplied and cannot be enumerated. + * + * \`frontmatter.title\` names a key in the document's own front matter; + * \`text.class.callout\` names an HTML class. Rejecting an unfamiliar tail would + * be the too-strict failure against a value the binary honors. + */ +export const VALE_SCOPE_PREFIXES = [ +${literalList(scopePrefixes)} +] as const; + +/** + * Where Vale ${VALE_VERSION} and its documentation disagree. + * + * Carried in the artifact rather than only in the report, so that a consumer + * can render them and a reviewer cannot miss them in a diff. + */ +export const VALE_DIVERGENCES = [ +${divergences + .map( + (divergence) => + ` {\n subject: ${JSON.stringify(divergence.subject)},\n finding:\n ${JSON.stringify(divergence.finding)},\n consequence:\n ${JSON.stringify(divergence.consequence)},\n },` + ) + .join("\n")} +] as const; +`; + +writeFileSync(OUTPUT_PATH, artifact, "utf8"); + +const report = `# Vale ${VALE_VERSION} vocabulary: divergence report + +GENERATED FILE — DO NOT EDIT. Produced by \`pnpm generate:vale-schema\` +alongside \`vale-vocabulary.ts\`. + +Every value in the vocabulary is the recorded answer of the vendored Vale +${VALE_VERSION} binary. This file is what the binary said that its own +documentation does not, in both directions. A generator that dropped these +would be quietly deciding which of the two to believe. + +## What was derived, and what was seeded + +| Vocabulary | Oracle | Discovers? | +| --- | --- | --- | +| Check types (${String(CHECK_TYPES.length)}) | \`'extends' key must be one of [...]\` | yes | +| Levels (${String(LEVELS.length)}) | \`'level' must be one of [...]\` | yes | +| Per-check fields | \`has invalid keys: ''\` | **no — membership only** | +| Scope operands | none; an unknown scope is silent | **no — fixture probe** | + +The bottom two rows are verified, not discovered. \`E201\` names the key you got +wrong and never the ones you could have used, and an unknown scope produces no +error at all. A real field or operand that the generator's candidate list does +not propose is simply absent from the vocabulary, and the schema then rejects a +rule the binary accepts — the too-strict direction, which the design ranks as +the worse failure. + +## Divergences + +${ + divergences.length === 0 + ? "None. The binary and the documentation agreed everywhere probed." + : divergences + .map( + (divergence) => + `### \`${divergence.subject}\`\n\n${divergence.finding}\n\n**Consequence.** ${divergence.consequence}` + ) + .join("\n\n") +} + +## Every scope probed + +| Operand | Fixture | Documented | Verdict | +| --- | --- | --- | --- | +${scopeMeasurements + .map( + ({ candidate, verdict }) => + `| \`${candidate.operand}\` | \`.${candidate.ext}\` | ${candidate.documented ? "yes" : "no"} | ${verdict} |` + ) + .join("\n")} +`; + +writeFileSync(REPORT_PATH, report, "utf8"); + +// Formatting is delegated to the repository's own prettier rather than hand +// matched here, so the artifact survives `lint-staged` untouched and a +// re-generation is a no-op diff. +const formatted = spawnSync( + process.execPath, + [ + resolve( + PACKAGE_ROOT, + "..", + "..", + "node_modules", + "prettier", + "bin", + "prettier.cjs" + ), + "--write", + OUTPUT_PATH, + REPORT_PATH, + ], + { encoding: "utf8" } +); +if (formatted.status !== 0) { + throw new Error( + `prettier failed on the generated artifact: ${formatted.stderr}` + ); +} + +console.log(`\nWritten: ${OUTPUT_PATH}`); +console.log(`Written: ${REPORT_PATH}`); +console.log(`Divergences recorded: ${String(divergences.length)}`); +for (const divergence of divergences) { + console.log(` - ${divergence.subject}: ${divergence.finding}`); +} diff --git a/packages/cli/src/generated/vale-vocabulary-report.md b/packages/cli/src/generated/vale-vocabulary-report.md new file mode 100644 index 00000000..b42675ce --- /dev/null +++ b/packages/cli/src/generated/vale-vocabulary-report.md @@ -0,0 +1,112 @@ +# Vale 3.18.0 vocabulary: divergence report + +GENERATED FILE — DO NOT EDIT. Produced by `pnpm generate:vale-schema` +alongside `vale-vocabulary.ts`. + +Every value in the vocabulary is the recorded answer of the vendored Vale +3.18.0 binary. This file is what the binary said that its own +documentation does not, in both directions. A generator that dropped these +would be quietly deciding which of the two to believe. + +## What was derived, and what was seeded + +| Vocabulary | Oracle | Discovers? | +| ---------------- | ------------------------------------ | ------------------------ | +| Check types (12) | `'extends' key must be one of [...]` | yes | +| Levels (3) | `'level' must be one of [...]` | yes | +| Per-check fields | `has invalid keys: ''` | **no — membership only** | +| Scope operands | none; an unknown scope is silent | **no — fixture probe** | + +The bottom two rows are verified, not discovered. `E201` names the key you got +wrong and never the ones you could have used, and an unknown scope produces no +error at all. A real field or operand that the generator's candidate list does +not propose is simply absent from the vocabulary, and the schema then rejects a +rule the binary accepts — the too-strict direction, which the design ranks as +the worse failure. + +## Divergences + +### `scope: meta` + +Vale 3.18.0 documents this operand and it never fired, on any fixture probed (.md). + +**Consequence.** It is omitted from the vocabulary, so `verify` rejects it. A rule written from the documentation would otherwise load, run, and match nothing, with no error reported anywhere. + +### `scope: meta.class.title` + +Vale 3.18.0 documents this operand and it never fired, on any fixture probed (.md). + +**Consequence.** It is omitted from the vocabulary, so `verify` rejects it. A rule written from the documentation would otherwise load, run, and match nothing, with no error reported anywhere. + +### `scope: frontmatter` + +This operand fired and Vale 3.18.0 documents it nowhere. + +**Consequence.** It is included in the vocabulary. It is also the standing counterexample to trusting the candidate list: a real operand nobody proposes is simply absent, and the schema then rejects a rule the binary honors. + +### `scope: frontmatter.title` + +This operand fired and Vale 3.18.0 documents it nowhere. + +**Consequence.** It is included in the vocabulary. It is also the standing counterexample to trusting the candidate list: a real operand nobody proposes is simply absent, and the schema then rejects a rule the binary honors. + +### `extends: consistency` + +This check accepted 'taskless_generator_sentinel', a key no check has. It does not validate its keys at all. + +**Consequence.** The schema is permissive here. Being strict would reject rules the binary runs, to catch a typo that costs nothing but a field quietly ignored — the too-strict direction, which is the worse failure. + +### `extends: spelling` + +This check accepted 'taskless_generator_sentinel', a key no check has. It does not validate its keys at all. + +**Consequence.** The schema is permissive here. Being strict would reject rules the binary runs, to catch a typo that costs nothing but a field quietly ignored — the too-strict direction, which is the worse failure. + +### `field probes: membership inferred from a type complaint` + +10 probes drew an E201 that was not an invalid-key list: capitalization.action: expected a map, got 'bool'; conditional.action: expected a map, got 'bool'; existence.action: expected a map, got 'bool'; metric.action: expected a map, got 'bool'; occurrence.action: expected a map, got 'bool'; readability.action: expected a map, got 'bool'; repetition.action: expected a map, got 'bool'; script.action: expected a map, got 'bool'; sequence.action: expected a map, got 'bool'; substitution.action: expected a map, got 'bool'. + +**Consequence.** Each is recorded as a member: Vale recognized the key and objected to the probe's arbitrary value instead, which is membership evidence. They are listed so the inference is auditable rather than assumed. + +## Every scope probed + +| Operand | Fixture | Documented | Verdict | +| ------------------- | ------- | ---------- | ------- | +| `text` | `.md` | yes | fires | +| `code` | `.md` | yes | fires | +| `raw` | `.md` | yes | fires | +| `heading` | `.md` | yes | fires | +| `heading.h1` | `.md` | yes | fires | +| `heading.h2` | `.md` | yes | fires | +| `heading.h3` | `.md` | yes | fires | +| `heading.h4` | `.md` | yes | fires | +| `heading.h5` | `.md` | yes | fires | +| `heading.h6` | `.md` | yes | fires | +| `heading.h7` | `.md` | no | silent | +| `paragraph` | `.md` | yes | fires | +| `sentence` | `.md` | yes | fires | +| `list` | `.md` | yes | fires | +| `blockquote` | `.md` | yes | fires | +| `link` | `.md` | yes | fires | +| `alt` | `.md` | yes | fires | +| `summary` | `.md` | yes | fires | +| `strong` | `.md` | yes | fires | +| `emphasis` | `.md` | yes | fires | +| `table` | `.md` | yes | fires | +| `table.header` | `.md` | yes | fires | +| `table.cell` | `.md` | yes | fires | +| `table.caption` | `.html` | yes | fires | +| `table.row` | `.md` | no | silent | +| `figure.caption` | `.html` | yes | fires | +| `meta` | `.md` | yes | silent | +| `meta.class.title` | `.md` | yes | silent | +| `frontmatter` | `.md` | no | fires | +| `frontmatter.title` | `.md` | no | fires | +| `text.class.foo` | `.html` | yes | fires | +| `comment` | `.js` | yes | fires | +| `comment.line` | `.js` | yes | fires | +| `comment.block` | `.js` | yes | fires | +| `comment.line` | `.ts` | yes | fires | +| `comment.block` | `.ts` | yes | fires | +| `fenced` | `.md` | no | silent | +| `banana` | `.md` | no | silent | diff --git a/packages/cli/src/generated/vale-vocabulary.ts b/packages/cli/src/generated/vale-vocabulary.ts new file mode 100644 index 00000000..6454912a --- /dev/null +++ b/packages/cli/src/generated/vale-vocabulary.ts @@ -0,0 +1,216 @@ +/** + * The Vale rule vocabulary, derived from the vendored binary. + * + * GENERATED FILE — DO NOT EDIT. Run `pnpm generate:vale-schema` in + * `packages/cli` to reproduce it. The generator is + * `scripts/generate-vale-schema.ts`, and its header explains what each value + * below was measured with and what it is worth. + * + * Derived against Vale 3.18.0. Every value here is the recorded answer + * of that binary to a rule the generator wrote and ran; nothing is transcribed + * from documentation. Where the binary and the documentation disagree, the + * disagreement is in `vale-vocabulary-report.md` rather than dropped. + * + * `src/schemas/vale-rule.ts` turns these into the zod schema `verify` runs. + */ + +/** The binary this vocabulary was derived from. */ +export const VALE_VOCABULARY_VERSION = "3.18.0"; + +/** + * Vale's check types, self-enumerated: an unknown `extends` makes the binary + * name the whole set, so this is discovered rather than proposed. + */ +export const VALE_CHECK_TYPES = [ + "capitalization", + "conditional", + "consistency", + "existence", + "metric", + "occurrence", + "readability", + "repetition", + "script", + "sequence", + "spelling", + "substitution", +] as const; + +/** + * Vale's severities, self-enumerated the same way, and left in the binary's + * own order because that order is severity. The value is case-sensitive. + */ +export const VALE_LEVELS = ["suggestion", "warning", "error"] as const; + +/** + * The checks that accept any key at all and ignore what they do not know. + * + * Measured by offering each check a sentinel key no check has. + */ +export const VALE_PERMISSIVE_CHECKS = ["consistency", "spelling"] as const; + +/** + * The fields every strict check accepts, as the intersection of their measured + * tables rather than as a declared list. + * + * Note what the intersection excludes: `vocab` is per-check, because several + * checks reject it. + */ +export const VALE_COMMON_FIELDS = [ + "action", + "description", + "extends", + "level", + "limit", + "link", + "message", + "name", + "scope", +] as const; + +/** + * Each strict check's own fields — its measured table minus the common ones. + * + * Membership only: `E201` names the key you got wrong and never the ones you + * could have used, so every name here was proposed by the generator's candidate + * list and confirmed. A real field nobody proposed is absent. + */ +export const VALE_CHECK_FIELDS = { + capitalization: [ + "exceptions", + "indicators", + "match", + "prefix", + "style", + "threshold", + "vocab", + ], + conditional: ["exceptions", "first", "ignorecase", "second", "vocab"], + existence: [ + "append", + "exceptions", + "ignorecase", + "nonword", + "raw", + "tokens", + "vocab", + ], + metric: ["condition", "formula"], + occurrence: ["ignorecase", "max", "min", "token"], + readability: ["grade", "metrics"], + repetition: ["alpha", "exceptions", "ignorecase", "max", "tokens", "vocab"], + script: ["script"], + sequence: ["ignorecase", "tokens"], + substitution: [ + "capitalize", + "exceptions", + "ignorecase", + "nonword", + "pos", + "swap", + "vocab", + ], +} as const; + +/** + * The `scope` operands measured firing on a fixture carrying their construct. + * + * `scope` has no oracle — an unknown scope is silent, and so is a valid scope + * with no construct to match — so an operand is here only if a rule using it + * flagged a fixture that a reach probe independently confirmed was linted. + */ +export const VALE_SCOPE_OPERANDS = [ + "alt", + "blockquote", + "code", + "comment", + "comment.block", + "comment.line", + "emphasis", + "figure.caption", + "frontmatter", + "heading", + "heading.h1", + "heading.h2", + "heading.h3", + "heading.h4", + "heading.h5", + "heading.h6", + "link", + "list", + "paragraph", + "raw", + "sentence", + "strong", + "summary", + "table", + "table.caption", + "table.cell", + "table.header", + "text", +] as const; + +/** + * Scope families whose tail is author-supplied and cannot be enumerated. + * + * `frontmatter.title` names a key in the document's own front matter; + * `text.class.callout` names an HTML class. Rejecting an unfamiliar tail would + * be the too-strict failure against a value the binary honors. + */ +export const VALE_SCOPE_PREFIXES = ["frontmatter.", "text.class."] as const; + +/** + * Where Vale 3.18.0 and its documentation disagree. + * + * Carried in the artifact rather than only in the report, so that a consumer + * can render them and a reviewer cannot miss them in a diff. + */ +export const VALE_DIVERGENCES = [ + { + subject: "scope: meta", + finding: + "Vale 3.18.0 documents this operand and it never fired, on any fixture probed (.md).", + consequence: + "It is omitted from the vocabulary, so `verify` rejects it. A rule written from the documentation would otherwise load, run, and match nothing, with no error reported anywhere.", + }, + { + subject: "scope: meta.class.title", + finding: + "Vale 3.18.0 documents this operand and it never fired, on any fixture probed (.md).", + consequence: + "It is omitted from the vocabulary, so `verify` rejects it. A rule written from the documentation would otherwise load, run, and match nothing, with no error reported anywhere.", + }, + { + subject: "scope: frontmatter", + finding: "This operand fired and Vale 3.18.0 documents it nowhere.", + consequence: + "It is included in the vocabulary. It is also the standing counterexample to trusting the candidate list: a real operand nobody proposes is simply absent, and the schema then rejects a rule the binary honors.", + }, + { + subject: "scope: frontmatter.title", + finding: "This operand fired and Vale 3.18.0 documents it nowhere.", + consequence: + "It is included in the vocabulary. It is also the standing counterexample to trusting the candidate list: a real operand nobody proposes is simply absent, and the schema then rejects a rule the binary honors.", + }, + { + subject: "extends: consistency", + finding: + "This check accepted 'taskless_generator_sentinel', a key no check has. It does not validate its keys at all.", + consequence: + "The schema is permissive here. Being strict would reject rules the binary runs, to catch a typo that costs nothing but a field quietly ignored — the too-strict direction, which is the worse failure.", + }, + { + subject: "extends: spelling", + finding: + "This check accepted 'taskless_generator_sentinel', a key no check has. It does not validate its keys at all.", + consequence: + "The schema is permissive here. Being strict would reject rules the binary runs, to catch a typo that costs nothing but a field quietly ignored — the too-strict direction, which is the worse failure.", + }, + { + subject: "field probes: membership inferred from a type complaint", + finding: + "10 probes drew an E201 that was not an invalid-key list: capitalization.action: expected a map, got 'bool'; conditional.action: expected a map, got 'bool'; existence.action: expected a map, got 'bool'; metric.action: expected a map, got 'bool'; occurrence.action: expected a map, got 'bool'; readability.action: expected a map, got 'bool'; repetition.action: expected a map, got 'bool'; script.action: expected a map, got 'bool'; sequence.action: expected a map, got 'bool'; substitution.action: expected a map, got 'bool'.", + consequence: + "Each is recorded as a member: Vale recognized the key and objected to the probe's arbitrary value instead, which is membership evidence. They are listed so the inference is auditable rather than assumed.", + }, +] as const; From 33e98d44140f9eed3de7ab554e0360c8bc3930ed Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 23:47:39 -0700 Subject: [PATCH 12/17] refactor(cli): build the Vale schema from the derived vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enums and field tables move out of `vale-rule.ts` and into the generated artifact; what stays is the part worth reading — the zod construction, the two-stage `.pipe()` that reproduces the binary's own order, the case-folding transform, the scope grammar, and the error messages that explain blast radius to an author. Generating those would mean maintaining a code emitter to reproduce prose. Three things stay hand-written on purpose, and each says why in place: - The fatal shapes. Every key in a tokenless `sequence` is a legal field of its check; it is the shape that panics, which a field table cannot express. - The scope grammar. `~` and `&` are operators around the operands, and rejecting `~fenced` is a business rule rather than a transcription. - The union members, spelled out so each `z.literal` survives into `ValeRule`. An import-time guard requires every derived check type to appear exactly once and to be classified strict or permissive. The artifact's version is asserted against `VALE_VERSION` as a conditional type rather than an `if`. Both are string literal types, so the compiler settles it and a mismatch is a build failure naming the line, instead of a throw in front of a user on whichever command loads the schema first. All 86 existing corpus rows pass unmodified — that is what proves generation reproduced the transcription. Two rows are added for ground the generation newly measured: `comment.line` and `comment.block` in the TypeScript tier, which is the only operand family whose membership depends on which parser the extension routes to. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- packages/cli/src/schemas/vale-rule.ts | 329 ++++++++++++++------------ packages/cli/test/vale-corpus.ts | 38 ++- 2 files changed, 212 insertions(+), 155 deletions(-) diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts index 281fb040..3f06b0ea 100644 --- a/packages/cli/src/schemas/vale-rule.ts +++ b/packages/cli/src/schemas/vale-rule.ts @@ -5,15 +5,21 @@ * and the machine-readable field knowledge exists only behind the hosted MCP * server at `api.vale.sh/mcp`, which is a paid product and unavailable to * `verify`. So unlike `ast-grep-rule.ts` — which runs `z.fromJSONSchema()` over - * an upstream artifact fetched per tag — everything here is a - * **transcription**, and a transcription drifts. - * - * What makes it trustworthy is not care in the writing. It is - * `test/vale-corpus.ts`: a table of minimal rules, each with a document it must - * flag, run through the vendored binary and through this module, asserting the - * two agree. Every value enumerated below was measured accepted by the binary - * that way. A Vale bump that changes the vocabulary fails that test naming the - * field, rather than leaving this file quietly wrong. + * an upstream artifact fetched per tag — there is nothing here to fetch. + * + * There is, however, a binary that answers questions. Every vocabulary this + * module enumerates is **derived** from it by `scripts/generate-vale-schema.ts` + * and lives in `../generated/vale-vocabulary.ts`: check types and levels read + * out of the error Vale raises when it enumerates its own accepted set, field + * tables probed key by key, scope operands measured firing on a fixture. This + * file is the validation layer over that vocabulary, not a second copy of it. + * + * The distinction matters because it changes what a Vale bump costs. A + * transcription drifts silently and is re-authored by hand; a derivation is + * re-run, and where the answer moved, the artifact's diff says so. What holds + * both to the binary is unchanged: `test/vale-corpus.ts` is a table of minimal + * rules, each with a document it must flag, run through the vendored binary and + * through this module, asserting the two agree. * * Two directions of error, and they are not symmetric: * @@ -23,6 +29,14 @@ * blocks work that would have functioned. Where a measurement is ambiguous, * accept. * + * That asymmetry is also the honest limit of the generator. Field tables and + * scope operands are established by *membership*: Vale's `E201` names the key + * you got wrong and never the ones you could have used, and an unknown scope + * raises nothing at all. So both are verified rather than discovered, and a + * real field nobody proposed is absent — the too-strict direction. The + * generator's candidate list is seeded accordingly, and its provenance is + * documented there. + * * ## Shape * * Two stages, in the order the binary itself works: @@ -43,48 +57,60 @@ import { z } from "zod"; +import { + VALE_CHECK_FIELDS, + VALE_CHECK_TYPES, + VALE_COMMON_FIELDS, + VALE_LEVELS, + VALE_PERMISSIVE_CHECKS, + VALE_SCOPE_OPERANDS as DERIVED_SCOPE_OPERANDS, + VALE_SCOPE_PREFIXES, + VALE_VOCABULARY_VERSION, +} from "../generated/vale-vocabulary"; import { VALE_VERSION } from "../rules/capabilities"; import { schemaLayer, type SchemaLayerResult } from "./layer"; +export { + VALE_CHECK_TYPES, + VALE_LEVELS, + VALE_PERMISSIVE_CHECKS, +} from "../generated/vale-vocabulary"; + /** - * Vale's check types, quoted from the binary rather than the documentation. - * - * Obtained by giving Vale an `extends` it does not know, which is not silently - * ignored — it fails the file, exits 2, and enumerates the set: + * The Vale release both the vocabulary and the vendored binary refer to. * - * ``` - * 'extends' key must be one of [capitalization conditional consistency - * existence occurrence repetition substitution readability spelling sequence - * metric script]. - * ``` + * Regenerating the artifact and bumping {@link VALE_VERSION} are two separate + * edits and nothing sequences them. If they ever disagree, every enum in this + * module is a measurement of some *other* Vale — which is precisely the drift + * deriving the vocabulary was meant to end. * - * That settles the eleven-versus-twelve question the docs and Vale's own MCP - * guide disagree on: **twelve**. The docs' eleven folds `readability` into - * `metric`; the binary treats them as separate checks whose fields are - * disjoint, and each rejects the other's. + * The conditional type is what settles it. Both sides are string literal types, + * so the compiler can decide the question: on a mismatch the alias resolves to + * `never`, the assignment below is rejected, and the build fails naming this + * line. A runtime `if` would have thrown instead — later, on whichever command + * happened to load the schema first, in front of a user rather than an author. * - * Sorted here rather than left in the binary's order, so the value `verify` - * prints to an author reads predictably. + * Every message in this module interpolates this rather than + * {@link VALE_VERSION}, so the assertion cannot be quietly orphaned. */ -export const VALE_CHECK_TYPES = [ - "capitalization", - "conditional", - "consistency", - "existence", - "metric", - "occurrence", - "readability", - "repetition", - "script", - "sequence", - "spelling", - "substitution", -] as const; +type PinnedValeVersion = + typeof VALE_VOCABULARY_VERSION extends typeof VALE_VERSION + ? typeof VALE_VERSION + : never; -export type ValeCheckType = (typeof VALE_CHECK_TYPES)[number]; +const PINNED_VALE_VERSION: PinnedValeVersion = VALE_VERSION; -/** Vale's three severities. The value is case-sensitive: `WARNING` is rejected. */ -export const VALE_LEVELS = ["suggestion", "warning", "error"] as const; +/** + * The name of one of Vale's twelve check types. + * + * The set itself is derived: an unknown `extends` is not silently ignored, so + * the binary fails the file and enumerates its own accepted values, and the + * generator reads them back out. That is what settles the eleven-versus-twelve + * question the docs and Vale's own MCP guide disagree on — the docs' eleven + * folds `readability` into `metric`, and the binary treats them as separate + * checks whose fields are disjoint. + */ +export type ValeCheckType = (typeof VALE_CHECK_TYPES)[number]; /** * The keys Vale reads literally, before the case-insensitive field decode. @@ -105,49 +131,22 @@ const HEADER_KEYS = new Set(["extends", "message", "level"]); /** * The `scope` operands the binary was measured honoring. * - * Each was established by authoring a rule with that scope over a document the - * rule had to flag, and confirming the finding appeared. A scope that never - * fired for any control is not here. + * Each was derived by authoring a rule with that scope over a fixture carrying + * its construct and confirming the finding appeared, with an independent reach + * probe proving the fixture was linted at all. An operand that never fired is + * not here, whatever the documentation says about it. * * This is the highest-value list in the module, because `scope` is the one * field nothing downstream ever validates. An unrecognized `extends` fails the * run loudly; an unrecognized field raises `E201`. An unrecognized **scope** - * loads, runs, and matches nothing — no error, anywhere, ever. - * - * Note what is *not* here: `meta` and `meta.class.`. The v3.18.0 addition - * is `frontmatter` / `frontmatter.`, which is the vocabulary the binary - * carries and the one that fires. + * loads, runs, and matches nothing — no error, anywhere, ever. It is also the + * list with no oracle: `extends` and `level` enumerate themselves, and this + * one has to be proposed and then measured, which is why the generator emits + * `vale-vocabulary-report.md` alongside it. Two entries there are worth + * knowing: `meta` and `meta.class.` are documented and never fire, and + * `frontmatter` / `frontmatter.` fire while being documented nowhere. */ -const SCOPE_OPERANDS = new Set([ - "text", - "code", - "raw", - "heading", - "heading.h1", - "heading.h2", - "heading.h3", - "heading.h4", - "heading.h5", - "heading.h6", - "paragraph", - "sentence", - "list", - "blockquote", - "link", - "alt", - "summary", - "strong", - "emphasis", - "table", - "table.header", - "table.cell", - "table.caption", - "figure.caption", - "frontmatter", - "comment", - "comment.line", - "comment.block", -]); +const SCOPE_OPERANDS = new Set(DERIVED_SCOPE_OPERANDS); /** * Scope families whose tail is author-supplied and cannot be enumerated. @@ -157,13 +156,18 @@ const SCOPE_OPERANDS = new Set([ * neither has a closed set — rejecting an unfamiliar tail would be the * "too strict" failure against a value the binary honors. */ -const SCOPE_PREFIXES = ["frontmatter.", "text.class."]; +const SCOPE_PREFIXES: readonly string[] = VALE_SCOPE_PREFIXES; -/** Operands, for the message `verify` shows an author. */ +/** + * Operands, for the message `verify` shows an author. + * + * The open families are spelled with a placeholder tail rather than omitted: + * an author told only that `frontmatter.title` is wrong, with no mention of + * `frontmatter.` in the accepted set, learns the wrong lesson. + */ export const VALE_SCOPE_OPERANDS: readonly string[] = [ ...SCOPE_OPERANDS, - "frontmatter.", - "text.class.", + ...SCOPE_PREFIXES.map((prefix) => `${prefix}`), ].toSorted(); function isScopeOperand(operand: string): boolean { @@ -207,11 +211,11 @@ function scopeMessages(scope: string): string[] { if (isScopeOperand(operand)) continue; messages.push( negated - ? `scope: "~${operand}" negates a scope Vale ${VALE_VERSION} does ` + + ? `scope: "~${operand}" negates a scope Vale ${PINNED_VALE_VERSION} does ` + `not have, so it subtracts nothing and the rule fires everywhere — ` + `the exclusion you wrote it for is silently gone. Vale does not ` + `report this, so verify does. ${scopeVocabulary}` - : `scope: "${operand}" is not a Vale ${VALE_VERSION} scope. Vale ` + + : `scope: "${operand}" is not a Vale ${PINNED_VALE_VERSION} scope. Vale ` + `does not reject an unknown scope — the rule loads, runs, and ` + `matches nothing. ${scopeVocabulary}` ); @@ -246,7 +250,7 @@ const valeHeaderSchema = z ) { fail( ["extends"], - `extends "${extendsValue}" is not a Vale ${VALE_VERSION} check type. ` + + `extends "${extendsValue}" is not a Vale ${PINNED_VALE_VERSION} check type. ` + `Vale fails the whole run over this, taking every other Vale rule's ` + `findings with it. Accepted: ${VALE_CHECK_TYPES.join(", ")}.` ); @@ -314,23 +318,24 @@ function canonicalKeys(rule: Record): Record { // --- Stage 2: the per-check field tables ------------------------------------- /** - * Fields every check accepts, whatever it extends. + * Fields every strict check accepts, whatever it extends. + * + * Derived as the *intersection* of the twelve measured field tables rather + * than declared, which is a stronger statement than a hand-written list: no + * one had to remember that `vocab` does not belong here because five checks + * reject it — the intersection simply does not contain it. * - * `vocab` is deliberately absent: five of the twelve reject it, so it is listed - * per check instead. Values are `unknown` throughout — the binary decides what - * a field's type means, and guessing here would be the "too strict" failure - * against types nothing measured. + * Values are `unknown` throughout. The binary decides what a field's type + * means, and guessing here would be the "too strict" failure against types + * nothing measured. */ -const commonFields = { +const commonFields: Record = { + ...Object.fromEntries( + VALE_COMMON_FIELDS.map((field) => [field, z.unknown()]) + ), extends: z.string(), message: z.string(), level: z.string().optional(), - scope: z.unknown(), - link: z.unknown(), - limit: z.unknown(), - action: z.unknown(), - description: z.unknown(), - name: z.unknown(), }; /** The field names above, for the message a rejected field gets. */ @@ -368,9 +373,10 @@ function check(name: ValeCheckType, fields: readonly string[]) { /** * The two checks that validate nothing. * - * Measured: `bananafield: true` on a `consistency` or a `spelling` rule loads - * without complaint and is ignored. They do not use the strict decode the other - * ten do, so the binary raises no `E201` for a foreign field there. + * Derived by offering every check a sentinel key no check has: `consistency` + * and `spelling` load without complaint and ignore it. They do not use the + * strict decode the other ten do, so the binary raises no `E201` for a foreign + * field there. * * `z.looseObject` follows the binary rather than the docs. Being strict here * would reject rules that work — the "too strict" direction — to catch a typo @@ -381,60 +387,21 @@ function permissiveCheck(name: ValeCheckType) { return z.looseObject({ ...commonFields, extends: z.literal(name) }); } -export const VALE_PERMISSIVE_CHECKS: readonly ValeCheckType[] = [ - "consistency", - "spelling", -]; - /** - * The per-check field tables, measured by adding each candidate to a minimal - * rule of that check and watching for `E201: has invalid keys`. + * The per-check field tables, derived rather than transcribed. * - * Kept as a table rather than inlined into the union below, because this *is* - * the measurement — the union is only how it gets enforced. Three entries - * contradict Vale's published docs, and the binary wins: `capitalization` takes - * `prefix` **singular** and rejects `prefixes` and `suffixes`; `capitalization` - * rejects `ignorecase`; `occurrence` rejects `exceptions` and `vocab`. + * Each name here was proposed by the generator's candidate list, added to a + * minimal working rule of that check, and confirmed by the binary. Three + * entries contradict Vale's published docs and the binary wins: + * `capitalization` takes `prefix` **singular** and rejects `prefixes` and + * `suffixes`; `capitalization` rejects `ignorecase`; `occurrence` rejects + * `exceptions` and `vocab`. All three names stay in the generator's candidate + * list precisely so that those stay recorded findings rather than omissions. * * `consistency` and `spelling` are absent on purpose — see * {@link permissiveCheck}. */ -const CHECK_FIELDS = { - existence: [ - "tokens", - "raw", - "ignorecase", - "nonword", - "exceptions", - "append", - "vocab", - ], - substitution: [ - "swap", - "ignorecase", - "nonword", - "exceptions", - "capitalize", - "pos", - "vocab", - ], - capitalization: [ - "match", - "style", - "exceptions", - "threshold", - "indicators", - "prefix", - "vocab", - ], - occurrence: ["token", "max", "min", "ignorecase"], - repetition: ["tokens", "alpha", "ignorecase", "exceptions", "max", "vocab"], - conditional: ["first", "second", "exceptions", "ignorecase", "vocab"], - metric: ["formula", "condition"], - readability: ["metrics", "grade"], - sequence: ["tokens", "ignorecase"], - script: ["script"], -} as const; +const CHECK_FIELDS = VALE_CHECK_FIELDS; /** * The `E201` class, as schema shape. @@ -529,6 +496,70 @@ function fatalShapeMessages( return fatal; } +/** + * The union's members, spelled out, and the guard that keeps them honest. + * + * They are spelled out rather than mapped over {@link CHECK_FIELDS} because + * that is what carries each `z.literal` into {@link ValeRule}; a mapped union + * infers `extends: string` and the type stops saying anything. The cost is + * that this list is hand-maintained while the vocabulary it draws from is + * derived — so a Vale release that adds or renames a check type would leave a + * member missing here, and the schema would reject a rule the binary runs. + * + * This block closes that gap at import: every derived check type must appear + * below exactly once, and must be classified as either strict or permissive. + * The failure is then a sentence naming the check, rather than a rule that + * mysteriously stops verifying six months later. + */ +const UNION_MEMBERS = [ + "existence", + "substitution", + "capitalization", + "occurrence", + "repetition", + "conditional", + "metric", + "readability", + "sequence", + "script", + "consistency", + "spelling", +] as const; + +{ + const spelled = new Set(UNION_MEMBERS); + const derived = new Set(VALE_CHECK_TYPES); + const missing = [...derived].filter((name) => !spelled.has(name)); + const extra = [...spelled].filter((name) => !derived.has(name)); + const permissive = new Set(VALE_PERMISSIVE_CHECKS); + const unclassified = [...derived].filter( + (name) => !permissive.has(name) && !(name in CHECK_FIELDS) + ); + + if (missing.length > 0) { + throw new Error( + `Vale ${PINNED_VALE_VERSION} has a ${missing.join(", ")} check and the schema's ` + + `union does not, so every rule extending it would be rejected. Add ` + + `the member in src/schemas/vale-rule.ts.` + ); + } + if (extra.length > 0) { + throw new Error( + `the schema's union has a ${extra.join(", ")} member and Vale ` + + `${PINNED_VALE_VERSION} has no such check, so those rules would verify clean ` + + `and then fail the whole run. Remove the member in ` + + `src/schemas/vale-rule.ts.` + ); + } + if (unclassified.length > 0) { + throw new Error( + `${unclassified.join(", ")} is in neither the derived field tables nor ` + + `the permissive set, so there is nothing to build a union member from. ` + + `Re-run the generator: pnpm generate:vale-schema.` + ); + } +} + const valeBodySchema = z .discriminatedUnion( "extends", diff --git a/packages/cli/test/vale-corpus.ts b/packages/cli/test/vale-corpus.ts index 5167e811..ba89fda7 100644 --- a/packages/cli/test/vale-corpus.ts +++ b/packages/cli/test/vale-corpus.ts @@ -1,12 +1,21 @@ /** * The corpus that makes `src/schemas/vale-rule.ts` true. * - * The schema is a hand transcription — Vale publishes no JSON Schema, and the - * machine-readable field knowledge sits behind its paid hosted MCP. What makes - * a transcription trustworthy is not care in the writing; it is this table. - * Every entry is a minimal rule plus a document the rule must flag, run through - * **both** the vendored binary and the schema in - * `vale-schema-contract.test.ts`, asserting the two agree. + * The schema's vocabulary is derived from the vendored binary by + * `scripts/generate-vale-schema.ts` — Vale publishes no JSON Schema, and the + * machine-readable field knowledge sits behind its paid hosted MCP, so the only + * thing left to ask is the binary. Every entry below is a minimal rule plus a + * document the rule must flag, run through **both** the vendored binary and the + * schema in `vale-schema-contract.test.ts`, asserting the two agree. + * + * A derivation needs this table for a different reason than a transcription + * did, and needs it just as much. A transcription could be wrong because + * someone mistyped; a derivation can be wrong because the generator asked the + * wrong question — a membership probe that misreads a crash as an acceptance, + * a scope fixture that never carried the construct it claimed to test. The + * generator cannot catch either, because both are errors in its own method. + * This table can, because it asks the binary a different question: not "is this + * a legal key" but "does this whole rule do what a rule is for". * * ## Why a verdict is not an exit code * @@ -323,6 +332,23 @@ const SCOPES: ValeCorpusEntry[] = [ control: "/*\n simply block\n*/\nconst x = 1;\n", ext: "js", }, + // The same two operands one tier over. `comment.*` is the only family whose + // membership depends on which parser the extension routes to, so a row that + // only ever asks JavaScript is asserting less than it looks like it is — + // "fires in a source file" and "fires in *this* source file" are different + // claims, and the schema makes the wider one. + { + name: "scope/comment.line-ts", + scope: "comment.line", + control: "// simply line\nconst x: number = 1;\n", + ext: "ts", + }, + { + name: "scope/comment.block-ts", + scope: "comment.block", + control: "/*\n simply block\n*/\nconst x: number = 1;\n", + ext: "ts", + }, // The operators. `scope` is a grammar, not an enum: a flat enum would reject // every one of these, which is worse than the gap the schema closes. { name: "scope/negation", scope: "~code", control: MIXED }, From abea321230ea7f63d9fdee934ae338a09c16182e Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 24 Aug 2026 23:47:48 -0700 Subject: [PATCH 13/17] chore(openspec): record and archive the vale-schema-generation change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `cli-rule-validation` requirement "The Vale rule schema is pinned to the vendored binary" was written for a hand-authored transcription. It becomes a derivation, and gains the three constraints that make one trustworthy: take every verdict from the exit status and the structured output rather than from an error phrase; fail rather than emit a partial enumeration, because a short enum is stricter than the binary; and report a divergence from Vale's documentation rather than resolving it silently. Delivery shape is a single PR — the generator, its artifact, and the schema's switch to importing it are only correct together, and there is no intermediate state that reaches production safely. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .changeset/vale-schema-generation.md | 48 ++++++++ .../.openspec.yaml | 2 + .../design.md | 110 ++++++++++++++++++ .../proposal.md | 48 ++++++++ .../specs/cli-rule-validation/spec.md | 45 +++++++ .../tasks.md | 58 +++++++++ openspec/specs/cli-rule-validation/spec.md | 43 +++++-- 7 files changed, 346 insertions(+), 8 deletions(-) create mode 100644 .changeset/vale-schema-generation.md create mode 100644 openspec/changes/archive/2026-08-25-vale-schema-generation/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-25-vale-schema-generation/design.md create mode 100644 openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md create mode 100644 openspec/changes/archive/2026-08-25-vale-schema-generation/specs/cli-rule-validation/spec.md create mode 100644 openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md diff --git a/.changeset/vale-schema-generation.md b/.changeset/vale-schema-generation.md new file mode 100644 index 00000000..b9e6e52a --- /dev/null +++ b/.changeset/vale-schema-generation.md @@ -0,0 +1,48 @@ +--- +"@taskless/cli": patch +--- + +Derive the Vale rule schema's vocabulary from the vendored binary instead of +transcribing it by hand. + +`pnpm generate:vale-schema` runs the pinned Vale against rules it writes itself +and emits `src/generated/vale-vocabulary.ts` — twelve check types, three levels, +ten per-check field tables, twenty-eight scope operands, two open scope +families — plus a divergence report. `src/schemas/vale-rule.ts` imports it and +stays what it was: the zod layer, the scope grammar, and the error messages that +explain blast radius to an author. Nothing about `verify`'s behavior changes; all +86 existing corpus rows pass against the generated schema unmodified, and two +were added for ground the generation newly measured. + +What a transcription lost was not the answer but the question. Every value in the +previous schema _was_ measured — by a script that was then discarded, leaving the +next person to raise `VALE_VERSION` with a failing test and no way to reproduce +the measurement it was failing against. + +The measuring is also where the errors live, so the generator is built around one +rule: **every verdict comes from the process exit status and the structured JSON +output, never from matching stdout against an error phrase.** A Go panic contains +no `has invalid keys` string, so a phrase-grep scores a crash as a clean run — +which is exactly how a tokenless `sequence` rule once came to look like a check +that validates nothing. A run's outcome is a closed set of `clean`, `diagnostic`, +`panic`, and `unrecognized`, and the last one is fatal at every call site. + +Two of the four vocabularies are self-enumerating: an unknown `extends` or +`level` makes the binary name its own accepted set. If either of those lines +stops matching, generation **fails** rather than emitting a short enum — a +truncated enum is _stricter_ than the binary, which is the direction that blocks +rules that would have worked. + +The other two are honest about their limit, and the artifact says so. `E201` +names the key you got wrong and never the ones you could have used, and an +unrecognized `scope` raises nothing at all — so field tables and scope operands +are **verified, not discovered**, from a candidate list seeded from four sources +with its provenance recorded. A scope verdict is three-valued, and a `scope: raw` +reach probe must fire on every fixture, so an operand cannot be dropped because +its fixture was never linted. + +Where the binary and Vale's documentation disagree, `vale-vocabulary-report.md` +records it rather than either side being quietly dropped: `meta` and +`meta.class.` are documented and never fire; `frontmatter` and +`frontmatter.` fire and are documented nowhere; `consistency` and `spelling` +validate no keys at all. diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/.openspec.yaml b/openspec/changes/archive/2026-08-25-vale-schema-generation/.openspec.yaml new file mode 100644 index 00000000..e685d45e --- /dev/null +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-25 diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/design.md b/openspec/changes/archive/2026-08-25-vale-schema-generation/design.md new file mode 100644 index 00000000..8f16709b --- /dev/null +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/design.md @@ -0,0 +1,110 @@ +# Design + +## The method rule, stated first because it is the whole risk + +**Every verdict comes from the process exit status and the structured JSON output. Never from grepping stdout for an error phrase.** + +This is not style guidance. It has already produced one wrong answer in this project: + +| Probe | Verdict | +| -------------------------------------------------- | --------------------------------------- | +| `sequence`, bare rule, grep for `has invalid keys` | "accepts any key" — permissive | +| `sequence`, bare rule, actual behavior | `panic: interface conversion: … is nil` | + +A Go panic contains no `has invalid keys` string, so a phrase-grep scores a crash as a clean run. Run Vale with `--output=JSON`; on a config error it writes `{Line, Path, Text, Code, Span}` to **stderr**. The generator therefore models the outcome of a run as a closed set with no escape hatch: + +``` +clean status 0, stdout parses as a findings map +diagnostic status ≠ 0, stderr parses as Vale's JSON object, keyed on Code +panic stderr contains a Go panic header +unrecognized everything else +``` + +`unrecognized` is fatal at every call site. There is deliberately no branch that folds an unfamiliar shape into "fine", because that is the failure the corpus caught the first time and cannot be relied on to catch the second. + +## What is derived and what is seeded + +Four vocabularies, and they do not have equal standing. + +| Vocabulary | Oracle | Discovers? | +| ---------------- | -------------------------------------------------- | ---------- | +| Check types (12) | `'extends' key must be one of [...]` | **yes** | +| Levels (3) | `'level' must be one of [...]` | **yes** | +| Per-check fields | `has invalid keys: ''` — names the _bad_ key | no | +| Scope operands | none; an unknown scope is silent | no | + +The top two are self-enumerating: give the binary a sentinel and it names its own accepted set. A Vale release that adds a check type is picked up rather than merely failing a test. + +The bottom two are **verified, not discovered**, and the difference is load-bearing. `E201` names the key you got wrong and never the ones you could have used, so a field table can only be built by proposing a candidate and asking. A real field nobody proposes is absent from the artifact, and the schema then rejects a rule Vale accepts — the too-strict direction, which the design of the schema itself ranks as the worse failure. The candidate list is therefore seeded from four independent sources and its provenance is documented in the generator: + +1. Vale's published key documentation, **including names the binary rejects** — `prefixes`, `suffixes`, and `ignorecase` on `capitalization` stay in the list precisely so their rejection is a recorded finding rather than an omission. +2. The hand transcription this replaces, so the generator is a superset of what a human already established. +3. Vale's shared keys: `link`, `limit`, `action`, `scope`, `description`, `name`, `comment`, `vocab`. +4. Every check's fields offered to every other check. That cross-probe is what makes the per-check tables a measured _partition_ rather than twelve unrelated lists. + +## Membership, and the two inferences + +The probe sets a candidate key to an arbitrary value (`true`) and reads the outcome as evidence about the **key**, not the value: + +| Outcome | Verdict | Why | +| ----------------------------------------- | ---------------- | --------------------------------------------------------- | +| clean | member | the key decoded | +| `has invalid keys` naming this field | **not** a member | the only negative oracle Vale offers | +| any other `E201` | member | Vale knew the key and objected to the value | +| panic | member | an unknown key is collected as unused, never dereferenced | +| invalid-key list naming a _different_ key | fatal | the base rule is contaminated | + +The last two rows of the positive column are inferences rather than readings, so both are collected and written into the divergence report. On Vale 3.18.0 the type-complaint case fires ten times, all `action: expected a map, got 'bool'`, and the panic case fires zero times. An inference nobody can audit is a grep with better manners; naming them is what keeps the distinction real. + +Two probes are excluded by construction rather than measured: + +- **The header keys.** `extends`, `message` and `level` are in every base rule and were verified running clean. Probing them is also unsound: `extends: true` reaches an `interface{}` → `string` conversion and panics. +- **The sentinel.** The one probe where a panic is _not_ read as evidence. It decides whether a check validates keys at all, and a permissive verdict taken from a crash would make the schema loose for a check that is strict — which is precisely the `sequence` error, re-run. + +Each check needs a **base rule**: a minimal working rule of that check to add the candidate to. That is hand-seeded and has to be, since several checks are not valid empty. Every base is verified clean before a single verdict is taken against it. + +**Common fields are derived, not declared.** They are the _intersection_ of the ten strict checks' measured tables. That is a stronger statement than a hand-written list: nobody had to remember that `vocab` is per-check because five checks reject it — the intersection simply does not contain it. The permissive checks are excluded because they accept everything and so constrain nothing. + +## Scope: three-valued, and only a fixture separates two of the values + +`scope` has no oracle at all. An invalid scope does not fail — the rule loads, runs, and matches nothing, which from outside is identical to a valid scope whose construct is absent from the document. So the verdict is three-valued: + +| Verdict | What happened | +| ------------- | ------------------------------------------------- | +| `fires` | the rule flagged the fixture: the operand is real | +| `silent` | the fixture was linted and the rule found nothing | +| `unreachable` | the fixture was never linted at all | + +Only a hand-written fixture separates the middle from the last, and the generator refuses to guess: every fixture contains the word `bogus`, a `scope: raw` **reach probe** must fire on it, and if it does not the run is fatal. That guard is not theoretical — `figure.caption` measures `silent` when its fixture nests the caption inside a `
`, and so does `scope: text` over the same document, because Vale drops everything inside that element. Without the reach guard the operand would be dropped as one Vale ignores, and every rule using it would then fail `verify`. + +The file extension is part of the fixture, not a detail: it decides which parser Vale routes the document to, and `comment.*` only exists in a source tier. + +## Divergences are reported, not resolved + +Where the binary and the documentation disagree, a generator that dropped the finding would be quietly deciding which to believe. The artifact carries `VALE_DIVERGENCES` and the run writes `vale-vocabulary-report.md`. Measured on 3.18.0: + +- **`meta` and `meta.class.` are documented and never fire.** Omitted from the vocabulary, so `verify` rejects them. A rule written from the docs would otherwise be inert forever with no error anywhere. +- **`frontmatter` and `frontmatter.` fire and are documented nowhere.** Included. This is the standing proof that a candidate list verifies rather than discovers: nobody proposing from the documentation alone would have found them. +- **`consistency` and `spelling` validate no keys at all**, measured with a sentinel. The schema stays permissive there; being strict would reject rules the binary runs. +- **Ten field probes drew a type complaint rather than a key rejection**, all on `action`. Recorded as members, and named so the inference is auditable. + +Three claims carried into this change from earlier measurement did **not** reproduce, and the report is where that surfaced: + +- `figure.caption` does fire, on a _bare_ `
` in an `.html` document. What never fires is the nested-in-`
` form, and the earlier note conflated the two. +- `comment.block` fires — in `.js` and in `.ts` alike. It does not "never fire while `comment.line` catches both". +- `comment.*` does not need a `.ts` fixture specifically; `.js` reaches the same tier. Rows for both now exist in the corpus, because "fires in a source file" and "fires in _this_ source file" are different claims and the schema makes the wider one. + +## What stays hand-written, and why each is not a generation failure + +- **The three fatal shapes.** A tokenless `sequence`, a `sequence` whose `tokens` is not a list, a `metric` with a `formula` and no `condition`. Every key in these rules is a legal field of its check — it is the _shape_ that is fatal, so a field table cannot express it. A panic is also a wider blast radius than `E201`: no rule name, no findings for anything, nothing to act on. +- **The scope grammar.** `~` negation and `&` chaining are operators around the operands, not values in the enum. Rejecting `~fenced` is a deliberate business rule, not a transcription, and the corpus carries it as a recorded divergence. +- **The union members.** They are spelled out rather than mapped over the generated table, because that is what carries each `z.literal` into `ValeRule`; a mapped union infers `extends: string`. The cost is a hand-maintained list beside a derived one, so an import-time guard requires every derived check type to appear exactly once and to be classified strict or permissive. The failure is a sentence naming the check. +- **The version pin.** The artifact records the version it was derived from, and `vale-rule.ts` asserts it against `VALE_VERSION` — as a **conditional type**, not an `if`. Both are string literal types, so the compiler settles it and a mismatch is a build failure naming the line, rather than a throw in front of a user on whichever command loads the schema first. + +## Alternatives considered + +**Generate the zod schema itself.** Rejected. The interesting content of `vale-rule.ts` is not the enums; it is the error messages that explain blast radius to an author, the two-stage `.pipe()` that reproduces the binary's own order, and the case-folding transform. Generating that would mean maintaining a code emitter to reproduce prose. Generating only the vocabulary keeps the derived part small and the reviewed part readable. + +**Emit JSON, like `ast-grep-rule-schema.json`.** Rejected. That file is JSON because upstream publishes JSON. Here the generator authors the artifact, and a `.ts` file gets `as const` — which is what makes `VALE_CHECK_TYPES` a literal union rather than `string[]`, and the import-time guard checkable at all. + +**Keep the transcription and add a checked-in probe script.** Rejected: two sources of truth that agree only by discipline. The failure mode is the one already observed — the answer survives, the question does not. diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md b/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md new file mode 100644 index 00000000..cbb314a6 --- /dev/null +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md @@ -0,0 +1,48 @@ +## Why + +`vale-authoring-polish` landed a Vale rule schema and called it, in its own module comment, **a transcription**: twelve check types, three levels, ten field tables and twenty-eight scope operands read off a binary by hand. + +Every one of those was measured — by a script that was then thrown away. What survives is the answer, not the question. The next person to raise `VALE_VERSION` gets a test failure naming a field and no way to reproduce the measurement it is failing against. + +The measuring is also where the errors live, and one is on the record. A probe grepping stdout for `has invalid keys` reads a Go panic as a clean run — a panic contains no such string — which is how a tokenless `sequence` rule came to look like a check that validates nothing: + +``` +sequence, bare rule, probe grepping for "has invalid keys" -> read as "accepts any key" +sequence, bare rule, actual behavior -> panic: interface conversion +``` + +A method that can make that mistake is worth writing down once, correctly. + +## What Changes + +- **A generator derives the vocabulary from the vendored binary.** `pnpm generate:vale-schema` writes `packages/cli/src/generated/vale-vocabulary.ts`, a checked-in artifact pinned to `VALE_VERSION`, mirroring how `fetch-ast-grep-schema.ts` produces `ast-grep-rule-schema.json`. `vale-rule.ts` imports it and keeps zod as the validation layer. +- **Verdicts come from the exit status and the structured JSON, never from a phrase match.** Vale's `--output=JSON` emits `{Line, Path, Text, Code, Span}` on stderr for a config error. The generator keys on `Code`, parses `Text` structurally, detects panics explicitly, and treats any outcome it does not recognize as fatal. +- **The generator fails loudly rather than emitting a short enum.** If the `'extends' key must be one of [...]` line stops matching, it errors. A truncated enum is _stricter_ than the binary, which is the direction that blocks working rules. +- **A divergence report ships with the artifact.** Where the binary and Vale's documentation disagree, the disagreement is written to `vale-vocabulary-report.md` rather than silently dropped on one side or the other. +- **The three fatal-shape checks stay hand-written.** A `sequence` with no `tokens`, a `sequence` whose `tokens` is not a list, and a `metric` with a `formula` and no `condition` each panic the binary. That is behavior a field table cannot express, so it stays a `.check()` beside the union. +- **No behavior change.** All 86 existing corpus rows pass unmodified against the generated schema; two rows are added for ground the generation newly measured. + +## Capabilities + +### New Capabilities + +None. This changes how an existing capability's data is produced. + +### Modified Capabilities + +- `cli-rule-validation`: the requirement "The Vale rule schema is pinned to the vendored binary" says the schema "SHALL be authored in this repository" and calls it a transcription. It becomes a derivation: the vocabulary SHALL be generated from the vendored binary by a script in the repository, and the requirement gains the two constraints that make a derivation trustworthy — fail loudly on an unrecognized error shape, and report rather than drop a divergence. + +## Impact + +- `packages/cli/scripts/generate-vale-schema.ts` — new. +- `packages/cli/src/generated/vale-vocabulary.ts` and `vale-vocabulary-report.md` — new, checked in. +- `packages/cli/src/schemas/vale-rule.ts` — the enums and field tables are replaced by imports; the zod construction, the scope grammar, and the fatal-shape checks are unchanged. +- `packages/cli/package.json` — one script. +- `packages/cli/test/vale-corpus.ts` — two rows added, none modified. +- Refs #171. + +## Delivery shape + +**Single PR.** The generator, the artifact it produces, and the schema's switch to importing it are one reviewable diff and are only correct together: the artifact is meaningless without the generator that reproduces it, and `vale-rule.ts` does not compile without the artifact. There is no intermediate state that reaches production safely, and no unit that a reviewer would be better off seeing alone. + +It stacks on `openspec/vale-authoring-polish-schema` (#175), whose schema it rewrites, so it is the tip of that stack and archives the change. diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/specs/cli-rule-validation/spec.md b/openspec/changes/archive/2026-08-25-vale-schema-generation/specs/cli-rule-validation/spec.md new file mode 100644 index 00000000..bf0de1ba --- /dev/null +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/specs/cli-rule-validation/spec.md @@ -0,0 +1,45 @@ +## MODIFIED Requirements + +### Requirement: The Vale rule schema is pinned to the vendored binary + +The Vale rule schema's vocabulary SHALL be derived from the vendored binary by a generator in this repository, written to a checked-in artifact pinned to `VALE_VERSION`, and a differential test SHALL hold the schema's claims against that binary. + +Vale publishes no JSON Schema for its check types. The machine-readable field knowledge exists only behind its hosted MCP server, which is a paid product and unavailable to `verify`. The binary, not the documentation, SHALL therefore be the authority for what the schema asserts — and it SHALL be asked by a script that can be re-run, rather than by hand once. What a transcription loses is not the answer but the question: a later reader inherits a value with no way to reproduce the measurement that produced it. + +The generator SHALL take every verdict from the process exit status and the structured JSON output, and SHALL NOT take one by matching output against an error phrase. A configuration error is reported on stderr as an object carrying a `Code`; a crash produces no such object at all. A probe that matches on a phrase scores a crash as a clean run, which has already produced a wrong entry in this schema. + +Where the derivation rests on a proposed candidate rather than on a set the binary enumerates, that SHALL be recorded as a limit of the artifact. Vale names the key an author got wrong and never the ones they could have used, and an unrecognized scope raises nothing at all — so field tables and scope operands are verified rather than discovered, and a value nobody proposes is absent. + +#### Scenario: A Vale upgrade that invalidates the schema fails loudly + +- **WHEN** `VALE_VERSION` is raised to a version whose accepted check types, fields, or scopes differ from the artifact +- **THEN** the differential test SHALL fail +- **AND** the failure SHALL name the construct whose treatment changed + +#### Scenario: The generator refuses to emit a vocabulary it cannot read + +- **WHEN** the binary's enumeration of its own accepted values no longer matches the shape the generator parses +- **THEN** the generator SHALL fail with an error naming what it could not read +- **AND** it SHALL NOT emit a partial enumeration + +A short enum is _stricter_ than the binary, so a silent partial parse would begin rejecting rules Vale accepts — blocking work that would have functioned, which is the worse of the two failure directions. + +#### Scenario: A divergence from Vale's documentation is reported, not dropped + +- **WHEN** the binary's measured behavior disagrees with Vale's published documentation, in either direction +- **THEN** generation SHALL emit that disagreement as part of its output +- **AND** the artifact SHALL record which side the schema followed + +A documented value that never fires is a trap an author walks into with the docs open. An undocumented value that works is evidence that something real may be missing from the candidate list. Neither may be resolved silently. + +#### Scenario: The vocabulary describes the binary the build ships + +- **WHEN** the checked-in artifact records a different Vale version than `VALE_VERSION` +- **THEN** the build SHALL fail +- **AND** the failure SHALL name the two versions + +#### Scenario: Behavior a schema cannot express stays explicit + +- **WHEN** a rule shape crashes the binary although every key in it is a legal field of its check +- **THEN** the schema SHALL reject that shape through a check stated alongside the generated field tables +- **AND** the rejection SHALL NOT be encoded as a field-table fact diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md b/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md new file mode 100644 index 00000000..3a4b5213 --- /dev/null +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md @@ -0,0 +1,58 @@ +## 1. The probe, built to the method rule + +- [x] 1.1 Run Vale with `--output=JSON` in an isolated temp config (`BasedOnStyles =`, `--no-exit`) and model one run's outcome as a closed set: `clean`, `diagnostic`, `panic`, `unrecognized`. No branch folds an unfamiliar shape into "fine". +- [x] 1.2 Parse the config-error object off **stderr** and key on `Code`. Confirmed the shape on 3.18.0: `{Line, Path, Text, Code: "E201", Span}`. +- [x] 1.3 Make `unrecognized` fatal at every call site, and detect panics explicitly rather than by the absence of a phrase. +- [x] 1.4 Resolve the binary through the shipped `findValeBinary`, and refuse to run if its self-reported version differs from `VALE_VERSION`. + +## 2. Derive the self-enumerating vocabularies + +- [x] 2.1 Read the check types out of `'extends' key must be one of [...]`. Twelve on 3.18.0. +- [x] 2.2 Read the levels out of `'level' must be one of [...]`. Three, left in the binary's own order because that order is severity. +- [x] 2.3 Throw rather than emit a short enum if either line stops matching, with a message saying why a truncated enum is the worse failure. Verified by hand against a deliberately broken pattern. + +## 3. Derive the per-check field tables + +- [x] 3.1 Hand-seed a minimal working base rule per check, and verify each runs clean before any verdict is taken against it. Several checks are not valid empty. +- [x] 3.2 Seed the candidate universe from four sources and document the provenance in the generator, including names the binary rejects. +- [x] 3.3 Cross-probe every check's fields against every other check, so the tables are a measured partition. +- [x] 3.4 Read membership from the outcome set: clean → member, `has invalid keys` naming the field → not a member, any other `E201` → member, panic → member. Record the last two as auditable inferences. Ten type complaints on 3.18.0 (all `action`), zero panics. +- [x] 3.5 Exclude the header keys from probing — they are members by construction, and `extends: true` panics. +- [x] 3.6 Detect the permissive checks with a sentinel key, and do **not** read a panic as evidence there. `consistency` and `spelling`. +- [x] 3.7 Derive the common fields as the intersection of the strict tables rather than declaring them. + +## 4. Derive the scope operands + +- [x] 4.1 Write a fixture per candidate carrying its construct, with the extension that routes it to the right parser. +- [x] 4.2 Run a `scope: raw` reach probe on every fixture and make an unreachable fixture fatal. +- [x] 4.3 Record the three-valued verdict per candidate, including negative controls (`fenced`, `banana`, `heading.h7`, `table.row`). +- [x] 4.4 Include an operand only if it fired; record `documented` per candidate so a disagreement can be stated. + +## 5. Emit + +- [x] 5.1 Write `src/generated/vale-vocabulary.ts` — check types, levels, permissive checks, common fields, per-check tables, scope operands, scope prefixes, divergences — with no timestamp, so the artifact is a pure function of the binary. +- [x] 5.2 Write `src/generated/vale-vocabulary-report.md`: what was derived versus seeded, the divergences, and every scope probed. +- [x] 5.3 Format both through the repository's own prettier, so a re-generation is a no-op diff. +- [x] 5.4 Add `pnpm generate:vale-schema`. +- [x] 5.5 Verify idempotence: deleted both artifacts, re-ran from clean, byte-identical. + +## 6. Wire the schema to the artifact + +- [x] 6.1 Replace the hand-maintained enums and field tables in `vale-rule.ts` with imports. Keep zod as the validation layer. +- [x] 6.2 Keep the scope grammar, the case-folding transform, the two-stage `.pipe()`, and the three fatal-shape checks exactly as they were. +- [x] 6.3 Assert the artifact's version against `VALE_VERSION` as a conditional type, so a mismatch is a build failure rather than a runtime throw. Verified by flipping the artifact's version and watching `tsc` fail naming the line. +- [x] 6.4 Guard the spelled-out union against the derived check types at import: every derived type present exactly once, and classified strict or permissive. + +## 7. Prove it against the corpus + +- [x] 7.1 All 86 existing corpus rows pass unmodified. No row edited to fit the output. +- [x] 7.2 Add rows for ground the generation newly measured: `comment.line` and `comment.block` in the TypeScript tier. 88 rows. +- [x] 7.3 Record the three earlier claims that did not reproduce — `figure.caption`, `comment.block`, and the `.ts` requirement — in design.md. +- [x] 7.4 `pnpm typecheck`, `pnpm lint`, `pnpm test` green. +- [x] 7.5 `pnpm build`, then verify the real CLI by hand on a broken rule. + +## 8. Land + +- [x] 8.1 One changeset on this branch. +- [x] 8.2 Archive the change — this is the tip of the stack. +- [x] 8.3 Open the PR against `openspec/vale-authoring-polish-schema`, referencing #171. diff --git a/openspec/specs/cli-rule-validation/spec.md b/openspec/specs/cli-rule-validation/spec.md index 91849ef0..fea2ff1d 100644 --- a/openspec/specs/cli-rule-validation/spec.md +++ b/openspec/specs/cli-rule-validation/spec.md @@ -121,17 +121,44 @@ The rule generation loop SHALL run `verify` and then `test` against a newly auth ### Requirement: The Vale rule schema is pinned to the vendored binary -The Vale rule schema SHALL be authored in this repository and pinned to `VALE_VERSION`, and a vendor-contract test SHALL hold its claims against the vendored binary. +The Vale rule schema's vocabulary SHALL be derived from the vendored binary by a generator in this repository, written to a checked-in artifact pinned to `VALE_VERSION`, and a differential test SHALL hold the schema's claims against that binary. -Vale publishes no JSON Schema for its check types. The machine-readable field knowledge exists only behind its hosted MCP server, which is a paid product and unavailable to `verify`. The schema is therefore a transcription, and a transcription drifts — so the binary, not the documentation, SHALL be the authority for what the schema asserts. +Vale publishes no JSON Schema for its check types. The machine-readable field knowledge exists only behind its hosted MCP server, which is a paid product and unavailable to `verify`. The binary, not the documentation, SHALL therefore be the authority for what the schema asserts — and it SHALL be asked by a script that can be re-run, rather than by hand once. What a transcription loses is not the answer but the question: a later reader inherits a value with no way to reproduce the measurement that produced it. + +The generator SHALL take every verdict from the process exit status and the structured JSON output, and SHALL NOT take one by matching output against an error phrase. A configuration error is reported on stderr as an object carrying a `Code`; a crash produces no such object at all. A probe that matches on a phrase scores a crash as a clean run, which has already produced a wrong entry in this schema. + +Where the derivation rests on a proposed candidate rather than on a set the binary enumerates, that SHALL be recorded as a limit of the artifact. Vale names the key an author got wrong and never the ones they could have used, and an unrecognized scope raises nothing at all — so field tables and scope operands are verified rather than discovered, and a value nobody proposes is absent. #### Scenario: A Vale upgrade that invalidates the schema fails loudly -- **WHEN** `VALE_VERSION` is raised to a version whose accepted check types or scopes differ from the schema -- **THEN** the vendor-contract test SHALL fail -- **AND** the failure SHALL name the field whose accepted values changed +- **WHEN** `VALE_VERSION` is raised to a version whose accepted check types, fields, or scopes differ from the artifact +- **THEN** the differential test SHALL fail +- **AND** the failure SHALL name the construct whose treatment changed + +#### Scenario: The generator refuses to emit a vocabulary it cannot read + +- **WHEN** the binary's enumeration of its own accepted values no longer matches the shape the generator parses +- **THEN** the generator SHALL fail with an error naming what it could not read +- **AND** it SHALL NOT emit a partial enumeration + +A short enum is _stricter_ than the binary, so a silent partial parse would begin rejecting rules Vale accepts — blocking work that would have functioned, which is the worse of the two failure directions. + +#### Scenario: A divergence from Vale's documentation is reported, not dropped + +- **WHEN** the binary's measured behavior disagrees with Vale's published documentation, in either direction +- **THEN** generation SHALL emit that disagreement as part of its output +- **AND** the artifact SHALL record which side the schema followed + +A documented value that never fires is a trap an author walks into with the docs open. An undocumented value that works is evidence that something real may be missing from the candidate list. Neither may be resolved silently. + +#### Scenario: The vocabulary describes the binary the build ships + +- **WHEN** the checked-in artifact records a different Vale version than `VALE_VERSION` +- **THEN** the build SHALL fail +- **AND** the failure SHALL name the two versions -#### Scenario: The schema is not derived from documentation alone +#### Scenario: Behavior a schema cannot express stays explicit -- **WHEN** a value is added to the schema's `extends` or `scope` enumerations -- **THEN** it SHALL be one the vendored binary was measured accepting +- **WHEN** a rule shape crashes the binary although every key in it is a legal field of its check +- **THEN** the schema SHALL reject that shape through a check stated alongside the generated field tables +- **AND** the rejection SHALL NOT be encoded as a field-table fact From 6fe85409c9b95e0cbcbf3e5908e7a16da9cf6242 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Tue, 25 Aug 2026 14:19:25 -0700 Subject: [PATCH 14/17] fix(vale): guard the union itself, and stop duplicating the diagnostic reader Three review findings on the derived Vale schema. The import-time guard in src/schemas/vale-rule.ts validated a freestanding list of check names that nothing else used: the discriminatedUnion array was a separate hand-written list, so the guard proved the name list agreed with the derived vocabulary and said nothing about the union. A maintainer could answer the guard's own error by adding the name, leave the union a member short, and get a schema that is silently too strict for the new check. The union array is now UNION_MEMBERS itself, the guard reads its discriminants off the built members, and a duplicate member is rejected too, so "exactly once" is checked rather than claimed. Verified by removing a member: the import now throws where it previously passed. The generator's ValeDiagnostic/readDiagnostic duplicated ValeConfigError and asValeConfigError from src/rules/vale/map.ts. It now imports them, the same way it already imports findValeBinary and VALE_VERSION. probe()'s isolating config and buildIsolatingConfig share three load-bearing details but differ on StylesPath, so they now cross-reference each other instead of being merged: reuse would mean staging a fake .taskless layout in a temp directory to satisfy a path convention no probe has. pnpm generate:vale-schema reproduces both generated artifacts byte for byte. --- packages/cli/scripts/generate-vale-schema.ts | 47 +++++++------ packages/cli/src/rules/vale/verify.ts | 5 ++ packages/cli/src/schemas/vale-rule.ts | 73 +++++++++++--------- 3 files changed, 72 insertions(+), 53 deletions(-) diff --git a/packages/cli/scripts/generate-vale-schema.ts b/packages/cli/scripts/generate-vale-schema.ts index e1e89b5c..5276643e 100644 --- a/packages/cli/scripts/generate-vale-schema.ts +++ b/packages/cli/scripts/generate-vale-schema.ts @@ -57,6 +57,8 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import type { ValeConfigError } from "../src/rules/vale/map.js"; + const scriptDirectory = dirname(fileURLToPath(import.meta.url)); const PACKAGE_ROOT = resolve(scriptDirectory, ".."); const OUTPUT_PATH = resolve( @@ -85,6 +87,7 @@ const REPORT_PATH = resolve( */ const { findValeBinary } = await import("../src/rules/vale/binary.js"); const { VALE_VERSION } = await import("../src/rules/capabilities.js"); +const { asValeConfigError } = await import("../src/rules/vale/map.js"); const resolution = findValeBinary(); if (resolution.path === undefined) { @@ -110,15 +113,6 @@ const VALE = resolution.path; // --- The probe --------------------------------------------------------------- -/** Vale's structured diagnostic, as `--output=JSON` writes it to stderr. */ -interface ValeDiagnostic { - Line: number; - Path: string; - Text: string; - Code: string; - Span: number; -} - /** * What one run of the binary did, as a closed set of outcomes. * @@ -129,7 +123,7 @@ interface ValeDiagnostic { */ type ProbeOutcome = | { kind: "clean"; findings: number } - | { kind: "diagnostic"; diagnostic: ValeDiagnostic } + | { kind: "diagnostic"; diagnostic: ValeConfigError } | { kind: "panic"; trace: string } | { kind: "unrecognized"; @@ -147,6 +141,20 @@ type ProbeOutcome = * * `--no-exit` suppresses the non-zero status Vale returns merely for *finding* * something, so a non-zero status here means the config itself failed. + * + * **This config is the sibling of `buildIsolatingConfig` in + * `src/rules/vale/verify.ts`, and the two are kept deliberately separate.** + * They share three load-bearing details, documented there: `MinAlertLevel = + * suggestion`, an empty `BasedOnStyles =`, and exactly one assignment of the + * enabled key in exactly one matcher. They differ on the fourth, which is why + * this is not a call to that function: `buildIsolatingConfig` needs an + * absolute `StylesPath` because it writes its config to a temp directory while + * the styles stay in the user's `.taskless/rules/vale/` tree, whereas this + * probe owns the whole temp directory and points at a `styles/probe/probe.yml` + * beside the config. Reuse would mean staging a fake `.taskless` layout in tmp + * just to satisfy a path convention no probe has. If a future requirement is + * discovered against either recipe (another key Vale needs to isolate a rule), + * apply it to both. */ function probe( rule: string, @@ -213,8 +221,14 @@ function probe( * returns `undefined` and becomes {@link ProbeOutcome} `unrecognized`, which * every caller treats as fatal. That is the point: an unreadable diagnostic * must never be mistaken for a clean run. + * + * The shape check itself is {@link asValeConfigError}, the one the CLI already + * runs over the same payload at runtime. Deriving against a second local copy + * of `{Code, Text, ...}` would mean a future change to Vale's diagnostic + * envelope has to be found twice, and nothing would report the half that was + * missed. */ -function readDiagnostic(stderr: string): ValeDiagnostic | undefined { +function readDiagnostic(stderr: string): ValeConfigError | undefined { const trimmed = stderr.trim(); if (trimmed === "") return undefined; let parsed: unknown; @@ -223,16 +237,7 @@ function readDiagnostic(stderr: string): ValeDiagnostic | undefined { } catch { return undefined; } - const first: unknown = Array.isArray(parsed) ? parsed[0] : parsed; - if (typeof first !== "object" || first === null) return undefined; - const candidate = first as Partial; - if ( - typeof candidate.Text !== "string" || - typeof candidate.Code !== "string" - ) { - return undefined; - } - return candidate as ValeDiagnostic; + return asValeConfigError(Array.isArray(parsed) ? parsed[0] : parsed); } /** diff --git a/packages/cli/src/rules/vale/verify.ts b/packages/cli/src/rules/vale/verify.ts index d3c1197c..b3d03500 100644 --- a/packages/cli/src/rules/vale/verify.ts +++ b/packages/cli/src/rules/vale/verify.ts @@ -36,6 +36,11 @@ function stylesPath(cwd: string): string { * positional (a later matcher wins; a repeat inside one matcher is * discarded), so a config that assigned it twice would be relying on the * rule that bit the scoping spec. + * + * `scripts/generate-vale-schema.ts` builds a second isolating config for its + * probes. It cannot call this one, because it owns its whole temp directory + * and so has no absolute StylesPath to hand over, but it shares the other two + * details above. A new requirement found here belongs there too. */ export function buildIsolatingConfig(cwd: string, ruleId: string): string { return [ diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts index 3f06b0ea..684794fe 100644 --- a/packages/cli/src/schemas/vale-rule.ts +++ b/packages/cli/src/schemas/vale-rule.ts @@ -503,31 +503,40 @@ function fatalShapeMessages( * that is what carries each `z.literal` into {@link ValeRule}; a mapped union * infers `extends: string` and the type stops saying anything. The cost is * that this list is hand-maintained while the vocabulary it draws from is - * derived — so a Vale release that adds or renames a check type would leave a + * derived, so a Vale release that adds or renames a check type would leave a * member missing here, and the schema would reject a rule the binary runs. * - * This block closes that gap at import: every derived check type must appear - * below exactly once, and must be classified as either strict or permissive. - * The failure is then a sentence naming the check, rather than a rule that - * mysteriously stops verifying six months later. + * This block closes that gap at import, and it reads the discriminants off + * *these members* rather than off a second list of names. A name list would + * only prove that the list agrees with the vocabulary: a maintainer could + * satisfy the guard by adding the name and never adding the `check(...)` call, + * leaving the union a member short while the error that prompted the edit went + * away. Asking the members themselves means the only way to satisfy the guard + * is to build the member. Every derived check type must appear here exactly + * once, and must be classified as either strict or permissive. The failure is + * then a sentence naming the check, rather than a rule that mysteriously stops + * verifying six months later. */ const UNION_MEMBERS = [ - "existence", - "substitution", - "capitalization", - "occurrence", - "repetition", - "conditional", - "metric", - "readability", - "sequence", - "script", - "consistency", - "spelling", + check("existence", CHECK_FIELDS.existence), + check("substitution", CHECK_FIELDS.substitution), + check("capitalization", CHECK_FIELDS.capitalization), + check("occurrence", CHECK_FIELDS.occurrence), + check("repetition", CHECK_FIELDS.repetition), + check("conditional", CHECK_FIELDS.conditional), + check("metric", CHECK_FIELDS.metric), + check("readability", CHECK_FIELDS.readability), + check("sequence", CHECK_FIELDS.sequence), + check("script", CHECK_FIELDS.script), + permissiveCheck("consistency"), + permissiveCheck("spelling"), ] as const; { - const spelled = new Set(UNION_MEMBERS); + const discriminants = UNION_MEMBERS.map( + (member) => member.shape.extends.value + ); + const spelled = new Set(discriminants); const derived = new Set(VALE_CHECK_TYPES); const missing = [...derived].filter((name) => !spelled.has(name)); const extra = [...spelled].filter((name) => !derived.has(name)); @@ -558,25 +567,25 @@ const UNION_MEMBERS = [ `Re-run the generator: pnpm generate:vale-schema.` ); } + if (spelled.size !== discriminants.length) { + // "Exactly once" is a real requirement rather than tidiness: zod matches a + // discriminated union by discriminant, so a second member for a check that + // already has one is dead code that no rule ever reaches. + const repeated = [...spelled].filter( + (name) => discriminants.filter((other) => other === name).length > 1 + ); + throw new Error( + `the schema's union has more than one ${repeated.join(", ")} ` + + `member, and only the first is reachable. Remove the duplicate in ` + + `src/schemas/vale-rule.ts.` + ); + } } const valeBodySchema = z .discriminatedUnion( "extends", - [ - check("existence", CHECK_FIELDS.existence), - check("substitution", CHECK_FIELDS.substitution), - check("capitalization", CHECK_FIELDS.capitalization), - check("occurrence", CHECK_FIELDS.occurrence), - check("repetition", CHECK_FIELDS.repetition), - check("conditional", CHECK_FIELDS.conditional), - check("metric", CHECK_FIELDS.metric), - check("readability", CHECK_FIELDS.readability), - check("sequence", CHECK_FIELDS.sequence), - check("script", CHECK_FIELDS.script), - permissiveCheck("consistency"), - permissiveCheck("spelling"), - ], + UNION_MEMBERS, { // Unreachable in practice — the header rejects an unknown `extends` first, // with a message naming all twelve. Supplied so that if the two ever drift From 2f5e7ae518a2641738939287c9bc66988127969a Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Tue, 25 Aug 2026 14:20:37 -0700 Subject: [PATCH 15/17] docs: repair the corpus-rows clause in the changeset and proposal "two rows are added for ground the generation newly measured" is missing its verb. Reads "to cover ground" in all three places it appears, including the release note. --- .changeset/vale-schema-generation.md | 2 +- .../archive/2026-08-25-vale-schema-generation/proposal.md | 2 +- .../changes/archive/2026-08-25-vale-schema-generation/tasks.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/vale-schema-generation.md b/.changeset/vale-schema-generation.md index b9e6e52a..a344fb6e 100644 --- a/.changeset/vale-schema-generation.md +++ b/.changeset/vale-schema-generation.md @@ -12,7 +12,7 @@ families — plus a divergence report. `src/schemas/vale-rule.ts` imports it and stays what it was: the zod layer, the scope grammar, and the error messages that explain blast radius to an author. Nothing about `verify`'s behavior changes; all 86 existing corpus rows pass against the generated schema unmodified, and two -were added for ground the generation newly measured. +were added to cover ground the generation newly measured. What a transcription lost was not the answer but the question. Every value in the previous schema _was_ measured — by a script that was then discarded, leaving the diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md b/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md index cbb314a6..54ca7420 100644 --- a/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/proposal.md @@ -20,7 +20,7 @@ A method that can make that mistake is worth writing down once, correctly. - **The generator fails loudly rather than emitting a short enum.** If the `'extends' key must be one of [...]` line stops matching, it errors. A truncated enum is _stricter_ than the binary, which is the direction that blocks working rules. - **A divergence report ships with the artifact.** Where the binary and Vale's documentation disagree, the disagreement is written to `vale-vocabulary-report.md` rather than silently dropped on one side or the other. - **The three fatal-shape checks stay hand-written.** A `sequence` with no `tokens`, a `sequence` whose `tokens` is not a list, and a `metric` with a `formula` and no `condition` each panic the binary. That is behavior a field table cannot express, so it stays a `.check()` beside the union. -- **No behavior change.** All 86 existing corpus rows pass unmodified against the generated schema; two rows are added for ground the generation newly measured. +- **No behavior change.** All 86 existing corpus rows pass unmodified against the generated schema; two rows are added to cover ground the generation newly measured. ## Capabilities diff --git a/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md b/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md index 3a4b5213..b99d5f69 100644 --- a/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md +++ b/openspec/changes/archive/2026-08-25-vale-schema-generation/tasks.md @@ -46,7 +46,7 @@ ## 7. Prove it against the corpus - [x] 7.1 All 86 existing corpus rows pass unmodified. No row edited to fit the output. -- [x] 7.2 Add rows for ground the generation newly measured: `comment.line` and `comment.block` in the TypeScript tier. 88 rows. +- [x] 7.2 Add rows to cover ground the generation newly measured: `comment.line` and `comment.block` in the TypeScript tier. 88 rows. - [x] 7.3 Record the three earlier claims that did not reproduce — `figure.caption`, `comment.block`, and the `.ts` requirement — in design.md. - [x] 7.4 `pnpm typecheck`, `pnpm lint`, `pnpm test` green. - [x] 7.5 `pnpm build`, then verify the real CLI by hand on a broken rule. From dc2f50ce6e6661b626121c4c70f73eed0dc81f48 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Tue, 25 Aug 2026 14:21:39 -0700 Subject: [PATCH 16/17] docs(vale): say precisely which isolating-config details the two recipes share --- packages/cli/scripts/generate-vale-schema.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/cli/scripts/generate-vale-schema.ts b/packages/cli/scripts/generate-vale-schema.ts index 5276643e..4cd6d3bf 100644 --- a/packages/cli/scripts/generate-vale-schema.ts +++ b/packages/cli/scripts/generate-vale-schema.ts @@ -144,12 +144,13 @@ type ProbeOutcome = * * **This config is the sibling of `buildIsolatingConfig` in * `src/rules/vale/verify.ts`, and the two are kept deliberately separate.** - * They share three load-bearing details, documented there: `MinAlertLevel = - * suggestion`, an empty `BasedOnStyles =`, and exactly one assignment of the - * enabled key in exactly one matcher. They differ on the fourth, which is why - * this is not a call to that function: `buildIsolatingConfig` needs an - * absolute `StylesPath` because it writes its config to a temp directory while - * the styles stay in the user's `.taskless/rules/vale/` tree, whereas this + * They agree on three details: `MinAlertLevel = suggestion`, an empty + * `BasedOnStyles =`, and exactly one assignment of the enabled key in exactly + * one matcher, the last two of which that docstring explains are load-bearing. + * They differ on the fourth, which is why this is not a call to that function: + * `buildIsolatingConfig` needs an absolute `StylesPath` because it writes its + * config to a temp directory while the styles stay in the user's + * `.taskless/rules/vale/` tree, whereas this * probe owns the whole temp directory and points at a `styles/probe/probe.yml` * beside the config. Reuse would mean staging a fake `.taskless` layout in tmp * just to satisfy a path convention no probe has. If a future requirement is From 7d61ad390d465be48dbe4732c6f178bdcd47fd7a Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Tue, 25 Aug 2026 15:47:14 -0700 Subject: [PATCH 17/17] fix(vale): derive which keys Vale reads literally, and stop lowercasing them The schema assumed the keys Vale reads off the raw mapping were the three header ones. Measured, they are five: `scope` and `name` are read literally too, so `Scope: raw` fails the run with `E201 has invalid keys: 'scope'` while the schema, having canonicalised the key to lowercase, accepted it. For `scope` that was the worse half, because canonicalising also routed the value past the scope grammar, the only thing that ever inspects it. The set is now measured per key as a two-run differential and emitted as `VALE_LITERAL_KEYS`: the lowercase spelling must run clean, and the capitalised one either runs clean too (decoded) or draws a diagnostic (read literally). Anything else is fatal rather than guessed at. Corpus rows cover `Scope:`, `Name:`, and `Tokens:` as the contrast that keeps them from proving too much; the three rejecting rows fail against the old three-key set. Also from review: - `enumeratedCheckTypes` now reads Vale's structured diagnostic (exit status, `asValeConfigError`, `Code === "E201"`) before touching the free-text `Text`, matching `enumerateFromBinary` and every other diagnostic-reading test in the file, instead of regexing concatenated stdout and stderr. - `HEADER_KEYS` no longer duplicates the generator's `HEADER_FIELDS`; it is emitted as `VALE_HEADER_FIELDS` and imported. - `commonFields` filters the typed header keys out of the `VALE_COMMON_FIELDS` spread rather than relying on later-key-wins to overwrite them. - `runOne` and `REACH_PROBE` carry cross-references to their siblings in the generator and `verify.ts`, and those siblings point back. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017cEN93Acyp4zBwP3oDnyy1 --- .changeset/vale-schema-generation.md | 20 ++- packages/cli/scripts/generate-vale-schema.ts | 141 +++++++++++++++++- packages/cli/src/generated/vale-vocabulary.ts | 33 ++++ packages/cli/src/rules/vale/verify.ts | 7 +- packages/cli/src/schemas/vale-rule.ts | 64 ++++++-- packages/cli/test/vale-corpus.ts | 54 ++++++- .../cli/test/vale-schema-contract.test.ts | 21 +++ .../cli/test/vale-vendor-contract.test.ts | 65 +++++++- .../cli/test/verify-test-commands.test.ts | 1 + 9 files changed, 379 insertions(+), 27 deletions(-) diff --git a/.changeset/vale-schema-generation.md b/.changeset/vale-schema-generation.md index a344fb6e..8b600f66 100644 --- a/.changeset/vale-schema-generation.md +++ b/.changeset/vale-schema-generation.md @@ -10,9 +10,23 @@ and emits `src/generated/vale-vocabulary.ts` — twelve check types, three level ten per-check field tables, twenty-eight scope operands, two open scope families — plus a divergence report. `src/schemas/vale-rule.ts` imports it and stays what it was: the zod layer, the scope grammar, and the error messages that -explain blast radius to an author. Nothing about `verify`'s behavior changes; all -86 existing corpus rows pass against the generated schema unmodified, and two -were added to cover ground the generation newly measured. +explain blast radius to an author. Every corpus row that predates the generation +passes against the generated schema unmodified, and rows were added to cover +ground the generation newly measured. + +One of those measurements found a real gap. Vale decodes a check's own fields +case-insensitively, so `Tokens:` means `tokens:`, but a few keys are read off +the raw mapping before that decode and are not synonyms of their capitalised +spellings. Assuming that set was the three header keys was wrong: `scope` and +`name` are read literally too, so `Scope: raw` fails the run with +`E201 has invalid keys: 'scope'` while the schema, having lowercased it, was +accepting it. For `scope` that was the worse half of the bug, because +canonicalising the key also routed the value past the scope grammar, which is +the only thing that ever inspects it. The set is now derived as a per-key +two-run differential (the lowercase spelling must run clean; the capitalised one +either runs clean too or draws a diagnostic) and emitted as +`VALE_LITERAL_KEYS`, with corpus rows for `Scope:`, `Name:` and, as the contrast +that keeps them from proving too much, `Tokens:`. What a transcription lost was not the answer but the question. Every value in the previous schema _was_ measured — by a script that was then discarded, leaving the diff --git a/packages/cli/scripts/generate-vale-schema.ts b/packages/cli/scripts/generate-vale-schema.ts index 4cd6d3bf..5cc5efe2 100644 --- a/packages/cli/scripts/generate-vale-schema.ts +++ b/packages/cli/scripts/generate-vale-schema.ts @@ -153,9 +153,12 @@ type ProbeOutcome = * `.taskless/rules/vale/` tree, whereas this * probe owns the whole temp directory and points at a `styles/probe/probe.yml` * beside the config. Reuse would mean staging a fake `.taskless` layout in tmp - * just to satisfy a path convention no probe has. If a future requirement is - * discovered against either recipe (another key Vale needs to isolate a rule), - * apply it to both. + * just to satisfy a path convention no probe has. + * + * `runOne` in `test/vale-schema-contract.test.ts` is a third copy, for the same + * reason as this one and differing only in the style name. If a future + * requirement is discovered against any of the three recipes (another key Vale + * needs to isolate a rule), apply it to all three. */ function probe( rule: string, @@ -515,6 +518,11 @@ function toYaml(rule: Record): string { * Probing them would also be actively unsound: the probe value is arbitrary, * and `extends: true` reaches a bare `interface{}` to `string` conversion that * panics the process rather than reporting anything. + * + * Emitted as `VALE_HEADER_FIELDS` rather than kept local, because the schema + * needs the same three names to decide which keys survive canonicalisation. + * Hand-typing them in both files would put the fact that decides the casing + * behaviour in two places with nothing holding them together. */ const HEADER_FIELDS = ["extends", "message", "level"]; @@ -666,6 +674,93 @@ for (const check of strictChecks) { ); } +// --- Stage 2b: which keys survive a differently-cased spelling --------------- + +/** + * A value that makes each common field valid on an `existence` rule. + * + * The casing probe below is a differential, and a differential needs both runs + * to differ in exactly one thing. If the value were arbitrary the lowercase + * control would fail for reasons of its own, and "the capitalised spelling was + * rejected" would stop being a statement about casing. Each value here was + * chosen so the lowercase control runs clean, and the script checks that it + * does rather than assuming it. + */ +const CASING_VALUES: Record = { + action: { name: "remove" }, + description: "a description", + extends: "existence", + level: "warning", + limit: 1, + link: "https://example.test", + message: "x %s", + name: "a name", + scope: "raw", +}; + +/** + * Build an `existence` rule carrying `field` under the spelling given. + * + * The three header keys are candidates too, so the default for whichever field + * is under test is removed before the spelling is added. That is what lets + * `Extends:` be measured at all: the rule it produces has no `extends` key, + * which is precisely the state Vale complains about. + */ +function casingProbe(field: string, spelling: string): ProbeOutcome { + const rule: Record = { + extends: "existence", + message: "x %s", + level: "warning", + tokens: ["simply"], + }; + delete rule[field]; + rule[spelling] = must(CASING_VALUES[field], `a casing value for '${field}'`); + return probe(toYaml(rule), "Just simply do it.\n", "md"); +} + +/** + * The keys Vale reads literally, measured rather than assumed. + * + * Vale decodes a check's own fields through a case-insensitive map, so + * `Tokens:` means `tokens:`. A handful of keys are read off the raw mapping + * before that decode ever runs, and for those a differently-cased spelling is + * not a synonym: the literal read finds nothing, and the leftover key then + * reaches the field decode as an unknown one and draws an `E201`. + * + * This is measured because assuming it was wrong. The obvious three are + * `extends`, `message` and `level`, and they are not the whole set: `scope` + * and `name` are read literally too, so `Scope: fenced` fails the run where + * `Tokens:` does not. A schema that lowercased `Scope` to `scope` would accept + * a rule Vale refuses to load, and worse, would route it past the scope + * grammar that is the only thing checking that value at all. + * + * The probe is a two-run differential per field: the lowercase spelling must + * run clean, and then the capitalised one either runs clean too (decoded) or + * draws a diagnostic (read literally). Anything else is fatal rather than + * guessed at, because both wrong answers are real: a key wrongly listed here + * makes the schema reject a spelling Vale accepts, and a key wrongly omitted + * makes it accept one Vale refuses. + * + * Scope of the claim: the common fields, which is what the schema's + * canonicalisation can act on, since it runs before the check type is known. + * A per-check field read literally would not be caught here. + */ +const literalKeys: string[] = []; +for (const field of commonFields) { + const control = casingProbe(field, field); + if (control.kind !== "clean") { + fatal(`the lowercase casing control for '${field}'`, control); + } + const capitalised = field.charAt(0).toUpperCase() + field.slice(1); + const variant = casingProbe(field, capitalised); + if (variant.kind === "clean") continue; + if (variant.kind !== "diagnostic") { + fatal(`the '${capitalised}' casing probe`, variant); + } + literalKeys.push(field); +} +console.log(`Keys read literally: ${literalKeys.toSorted().join(", ")}`); + // --- Stage 3: the scope operands --------------------------------------------- /** @@ -945,7 +1040,14 @@ const SCOPE_CANDIDATES: readonly ScopeCandidate[] = [ type ScopeVerdict = "fires" | "silent" | "unreachable"; -/** The reach probe: proves a fixture was linted at all, independent of scope. */ +/** + * The reach probe: proves a fixture was linted at all, independent of scope. + * + * `REACH_PROBE` in `test/vale-corpus.ts` is the same technique over the corpus + * controls, with its own token and level. See its docstring for why the two are + * separate literals; a change to what makes a reach probe reliable belongs in + * both. + */ const REACH_RULE = 'extends: existence\nmessage: "x %s"\nlevel: error\nscope: raw\nnonword: true\ntokens:\n - bogus\n'; @@ -1188,6 +1290,37 @@ export const VALE_COMMON_FIELDS = [ ${literalList(commonFields)} ] as const; +/** + * The keys Vale reads literally, before the case-insensitive field decode. + * + * Not probed and not discovered. Every base rule the generator runs carries + * all three, so they are members of every check by construction. They are + * emitted so that the schema's header stage and this generator read one copy + * of the fact rather than two hand-typed ones. It is the fact that decides + * which spellings survive canonicalisation, so a drift between two copies + * would change validation without changing either list visibly. + */ +export const VALE_HEADER_FIELDS = [ +${literalList(HEADER_FIELDS.toSorted())} +] as const; + +/** + * The keys Vale reads off the raw mapping, before the case-insensitive decode. + * + * Measured per key as a two-run differential: the lowercase spelling runs + * clean, and the capitalised one either runs clean too or draws an \`E201\`. + * The set is wider than the three header keys, which is the reason it is + * derived instead of written down: \`scope\` and \`name\` are read literally as + * well, so \`Scope: fenced\` fails the run where \`Tokens:\` is simply decoded. + * + * A consumer must not lowercase these. Doing so turns a rule Vale refuses to + * load into one that validates clean, and for \`scope\` it also routes the value + * past the only grammar check it ever gets. + */ +export const VALE_LITERAL_KEYS = [ +${literalList(literalKeys.toSorted())} +] as const; + /** * Each strict check's own fields — its measured table minus the common ones. * diff --git a/packages/cli/src/generated/vale-vocabulary.ts b/packages/cli/src/generated/vale-vocabulary.ts index 6454912a..d4f14795 100644 --- a/packages/cli/src/generated/vale-vocabulary.ts +++ b/packages/cli/src/generated/vale-vocabulary.ts @@ -68,6 +68,39 @@ export const VALE_COMMON_FIELDS = [ "scope", ] as const; +/** + * The keys Vale reads literally, before the case-insensitive field decode. + * + * Not probed and not discovered. Every base rule the generator runs carries + * all three, so they are members of every check by construction. They are + * emitted so that the schema's header stage and this generator read one copy + * of the fact rather than two hand-typed ones. It is the fact that decides + * which spellings survive canonicalisation, so a drift between two copies + * would change validation without changing either list visibly. + */ +export const VALE_HEADER_FIELDS = ["extends", "level", "message"] as const; + +/** + * The keys Vale reads off the raw mapping, before the case-insensitive decode. + * + * Measured per key as a two-run differential: the lowercase spelling runs + * clean, and the capitalised one either runs clean too or draws an `E201`. + * The set is wider than the three header keys, which is the reason it is + * derived instead of written down: `scope` and `name` are read literally as + * well, so `Scope: fenced` fails the run where `Tokens:` is simply decoded. + * + * A consumer must not lowercase these. Doing so turns a rule Vale refuses to + * load into one that validates clean, and for `scope` it also routes the value + * past the only grammar check it ever gets. + */ +export const VALE_LITERAL_KEYS = [ + "extends", + "level", + "message", + "name", + "scope", +] as const; + /** * Each strict check's own fields — its measured table minus the common ones. * diff --git a/packages/cli/src/rules/vale/verify.ts b/packages/cli/src/rules/vale/verify.ts index b3d03500..f1ae27c4 100644 --- a/packages/cli/src/rules/vale/verify.ts +++ b/packages/cli/src/rules/vale/verify.ts @@ -38,9 +38,10 @@ function stylesPath(cwd: string): string { * rule that bit the scoping spec. * * `scripts/generate-vale-schema.ts` builds a second isolating config for its - * probes. It cannot call this one, because it owns its whole temp directory - * and so has no absolute StylesPath to hand over, but it shares the other two - * details above. A new requirement found here belongs there too. + * probes, and `runOne` in `test/vale-schema-contract.test.ts` a third. Neither + * can call this one, because each owns its whole temp directory and so has no + * absolute StylesPath to hand over, but both share the other two details + * above. A new requirement found here belongs in both of them too. */ export function buildIsolatingConfig(cwd: string, ruleId: string): string { return [ diff --git a/packages/cli/src/schemas/vale-rule.ts b/packages/cli/src/schemas/vale-rule.ts index 684794fe..d9c26527 100644 --- a/packages/cli/src/schemas/vale-rule.ts +++ b/packages/cli/src/schemas/vale-rule.ts @@ -44,7 +44,9 @@ * 1. **The header.** Vale reads `extends`, `message` and `level` literally, * before it decodes anything else, and gives up if they are wrong. `scope` * rides along because it is common to all twelve checks and is a grammar - * rather than a value. + * rather than a value. It is also, measured, read literally itself, which is + * why the set of literally-read keys is derived rather than assumed to be + * the header three. * 2. **The check's own fields**, as a `z.discriminatedUnion` on `extends`. * That is Vale's `E201` class expressed as schema shape: a strict object per * check type, so a field belonging to another one is rejected by the union @@ -61,7 +63,9 @@ import { VALE_CHECK_FIELDS, VALE_CHECK_TYPES, VALE_COMMON_FIELDS, + VALE_HEADER_FIELDS, VALE_LEVELS, + VALE_LITERAL_KEYS, VALE_PERMISSIVE_CHECKS, VALE_SCOPE_OPERANDS as DERIVED_SCOPE_OPERANDS, VALE_SCOPE_PREFIXES, @@ -115,16 +119,35 @@ export type ValeCheckType = (typeof VALE_CHECK_TYPES)[number]; /** * The keys Vale reads literally, before the case-insensitive field decode. * - * Measured: `Tokens:` and `ignoreCase:` are read exactly as their lowercase - * spellings, but `EXTENDS:` fails with "Missing the required 'extends' key" and - * `Message:` with the same for `message`. `LEVEL: warning` is stranger still — - * it reaches the field decode as `level`, which the header reader has already - * removed, so it comes back as `E201 has invalid keys: 'level'`. + * Measured, and wider than it looks. `Tokens:` and `ignoreCase:` are read + * exactly as their lowercase spellings, but `EXTENDS:` fails with "Missing the + * required 'extends' key" and `Message:` with the same for `message`. + * `LEVEL: warning` is stranger still: it reaches the field decode as `level`, + * which the header reader has already removed, so it comes back as + * `E201 has invalid keys: 'level'`. `Scope:` and `Name:` behave the same way. + * + * That last pair is why this is a derived set rather than the three header + * keys typed out here. `scope` and `name` are ordinary members of every + * check's field table, so they look like body fields, and lowercasing them + * would have made `Scope: fenced` validate clean against a rule Vale refuses + * to load. For `scope` it is worse than an equality of outcomes: the grammar + * in {@link scopeMessages} is the only thing that ever inspects that value, and + * a canonicalised `Scope` reaches stage 2 as a plain `z.unknown()` field. + * + * {@link canonicalKeys} reproduces the split, which is why these are named as + * a set. + */ +const LITERAL_KEYS = new Set(VALE_LITERAL_KEYS); + +/** + * The three keys stage 1 gives real types to. * - * {@link canonicalKeys} reproduces that split, which is why these three are - * named as a set. + * A subset of {@link LITERAL_KEYS}, and a different statement: these are the + * keys Vale needs before it can decode a check at all, so the schema types them + * rather than leaving them `unknown`. Generated alongside the rest of the + * vocabulary so this file and the generator read one copy of the fact. */ -const HEADER_KEYS = new Set(["extends", "message", "level"]); +const TYPED_HEADER_KEYS = new Set(VALE_HEADER_FIELDS); // --- The scope grammar ------------------------------------------------------- @@ -274,6 +297,16 @@ const valeHeaderSchema = z // `scope` is common to all twelve checks, so it is stated once here rather // than repeated across every member of the union below. + // + // Read case-sensitively, which is the measured behaviour rather than an + // oversight. `scope` is one of the keys Vale reads literally, so `Scope:` + // is not a synonym for it: the binary fails the run with + // `E201 has invalid keys: 'scope'`. {@link LITERAL_KEYS} keeps that + // spelling out of canonicalisation, so stage 2's strict object rejects it + // exactly as Vale does, and the grammar here never has to guess which + // spelling it is looking at. Lowercasing it instead would be the worst of + // both: a rule Vale refuses to load would validate clean, having reached + // stage 2 as a plain `z.unknown()` field with the grammar below skipped. const { scope } = rule; if (scope === undefined) return; if (typeof scope !== "string" && !Array.isArray(scope)) { @@ -310,7 +343,7 @@ function canonicalKeys(rule: Record): Record { const canonical: Record = {}; for (const [key, value] of Object.entries(rule)) { const lower = key.toLowerCase(); - canonical[HEADER_KEYS.has(lower) && key !== lower ? key : lower] = value; + canonical[LITERAL_KEYS.has(lower) && key !== lower ? key : lower] = value; } return canonical; } @@ -330,8 +363,17 @@ function canonicalKeys(rule: Record): Record { * nothing measured. */ const commonFields: Record = { + // The header keys are filtered out rather than left to be overwritten. They + // are part of the measured intersection, so an unfiltered spread creates all + // three as `z.unknown()` and the typed entries below survive only because a + // later key in an object literal wins. That is a real dependency on line + // order for the typing of `extends`, `message` and `level`, and nothing in + // the shape of the code says so; filtering makes the override the structure + // instead of a side effect of it. ...Object.fromEntries( - VALE_COMMON_FIELDS.map((field) => [field, z.unknown()]) + VALE_COMMON_FIELDS.filter((field) => !TYPED_HEADER_KEYS.has(field)).map( + (field) => [field, z.unknown()] + ) ), extends: z.string(), message: z.string(), diff --git a/packages/cli/test/vale-corpus.ts b/packages/cli/test/vale-corpus.ts index ba89fda7..ac67a113 100644 --- a/packages/cli/test/vale-corpus.ts +++ b/packages/cli/test/vale-corpus.ts @@ -724,6 +724,44 @@ const HEADER: ValeCorpusEntry[] = [ control: PROSE, expected: "rejected", }, + // `scope` and `name` are the rows that make the split measurable rather than + // assumed. Both are ordinary members of every check's field table, so they + // read like body fields, and both are in fact read literally: Vale answers a + // capitalised spelling with `E201 has invalid keys`, not with the silent + // synonym `Tokens:` gets. Canonicalising either one to lowercase makes a rule + // Vale refuses to load validate clean, and for `scope` it also routes the + // value past the grammar that is the only thing checking it. + { + name: "header/scope-wrong-case-key", + construct: "Scope, spelled with a capital", + rule: existence("Scope: raw\n"), + control: PROSE, + expected: "rejected", + }, + { + name: "header/scope-wrong-case-key-unknown-operand", + construct: "Scope, spelled with a capital, naming a scope Vale lacks", + rule: existence("Scope: fenced\n"), + control: THREE_PLACES, + expected: "rejected", + }, + { + name: "header/name-wrong-case-key", + construct: "Name, spelled with a capital", + rule: existence('Name: "a name"\n'), + control: PROSE, + expected: "rejected", + }, + // The contrast that keeps the rows above from proving too much: a field that + // is NOT read literally is a plain synonym, and rejecting it would be the + // too-strict failure against a spelling the binary honors. + { + name: "header/tokens-wrong-case-key", + construct: "Tokens, spelled with a capital", + rule: 'extends: existence\nmessage: "x %s"\nlevel: warning\nTokens:\n - simply\n', + control: PROSE, + expected: "accepted", + }, ]; export const VALE_CORPUS: readonly ValeCorpusEntry[] = [ @@ -737,5 +775,19 @@ export const VALE_CORPUS: readonly ValeCorpusEntry[] = [ ...HEADER, ]; -/** The reach guard's rule: it must fire on every control in the corpus. */ +/** + * The reach guard's rule: it must fire on every control in the corpus. + * + * Same technique as `REACH_RULE` in `scripts/generate-vale-schema.ts`: a + * `scope: raw` existence rule with `nonword: true`, used to prove a fixture was + * linted at all before any "did not fire" verdict is trusted. The two are + * deliberately separate values rather than one shared constant, because the + * only things they share are the two fields that make the technique work: the + * token differs (`simply`, the word every control here carries, versus the + * generator's own `bogus`), and so does the level. A shared constant would have + * to be a builder taking both, living in a module reachable from `scripts/` and + * `test/` alike, which is more machinery than two one-line literals earn. What + * they do share is the technique, so a change to what makes a reach probe + * reliable (a field Vale starts requiring, say) belongs in both. + */ export const REACH_PROBE = existence("scope: raw\nnonword: true\n"); diff --git a/packages/cli/test/vale-schema-contract.test.ts b/packages/cli/test/vale-schema-contract.test.ts index 8a0a2ac6..3347cb25 100644 --- a/packages/cli/test/vale-schema-contract.test.ts +++ b/packages/cli/test/vale-schema-contract.test.ts @@ -50,6 +50,27 @@ const withVale = binary === undefined ? describe.skip : describe; /** Where the corpus rule id lands, and therefore what `verify` would call it. */ const RULE_ID = "corpus"; +/** + * Run one corpus rule over its control, in a config isolated from everything + * else, and read back a three-way verdict. + * + * **This is the third copy of the isolating-config recipe**, alongside + * `probe()` in `scripts/generate-vale-schema.ts` and `buildIsolatingConfig` in + * `src/rules/vale/verify.ts`, whose docstrings cross-reference each other. All + * three agree on `MinAlertLevel = suggestion`, an empty `BasedOnStyles =`, and + * exactly one assignment of the enabled key in exactly one matcher. This copy + * differs from `buildIsolatingConfig` the same way `probe()` does: it owns its + * whole temp directory, so it has no absolute `StylesPath` to hand over, and + * differs from `probe()` only in naming the style after {@link RULE_ID}, so + * that a failure prints the id `verify` would use. + * + * It stays separate because it answers a different question: `probe()` returns + * a probe outcome the generator interprets as evidence about a *key*, + * while this returns the accepted/ignored/rejected verdict the differential + * compares against the schema. A future requirement discovered against any of + * the three recipes (another key Vale needs to isolate a rule) belongs in all + * three. + */ function runOne(rule: string, control: string, extension: string): ValeVerdict { const cwd = mkdtempSync(join(tmpdir(), "vale-corpus-")); try { diff --git a/packages/cli/test/vale-vendor-contract.test.ts b/packages/cli/test/vale-vendor-contract.test.ts index 661d4741..bd6a900c 100644 --- a/packages/cli/test/vale-vendor-contract.test.ts +++ b/packages/cli/test/vale-vendor-contract.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { findValeBinary } from "../src/rules/vale/binary"; +import { asValeConfigError } from "../src/rules/vale/map"; import { VALE_COMMENT_EXTENSIONS, VALE_CONVERTER_CHECKERS, @@ -705,7 +706,19 @@ withVale("Vale engine capabilities", () => { }); }); -/** The set Vale prints when rejecting an unknown `extends`. */ +/** + * The set Vale prints when rejecting an unknown `extends`. + * + * Structure first, free text second, the same order `enumerateFromBinary` in + * `scripts/generate-vale-schema.ts` uses over the same diagnostic, and for the + * same reason. This function feeds `VALE_CHECK_TYPES`, which the schema treats + * as authoritative, and **a truncated enum is stricter than the binary**: a + * loose scan of concatenated `stdout`+`stderr` that half-matched a changed + * message would quietly shrink the vocabulary and start rejecting rules Vale + * runs. So the exit status, the JSON envelope and the `E201` code are all + * checked before the `Text` is read at all, and anything unfamiliar throws + * rather than guesses. + */ function enumeratedCheckTypes(): string[] { const cwd = project( `${header}\n[*.md]\nrules.bogus = YES\n`, @@ -713,13 +726,55 @@ function enumeratedCheckTypes(): string[] { { "doc.md": "Just simply do it.\n" } ); const result = runRaw(cwd, ["doc.md"], ["--no-exit"]); - const message = `${result.stdout}${result.stderr}`; - const listed = /'extends' key must be one of \[([^\]]+)]/.exec(message); + if (result.status === 0) { + throw new Error( + `Vale ${VALE_VERSION} no longer fails the run over an unknown ` + + `'extends'. stdout: ${result.stdout}` + ); + } + + // `asValeConfigError` is the CLI's own guard over this payload, reused + // rather than re-stated, so a change to Vale's diagnostic envelope has one + // place to be found instead of three. + let parsed: unknown; + try { + parsed = JSON.parse(result.stderr.trim()); + } catch { + throw new Error( + `Vale ${VALE_VERSION} did not answer an unknown 'extends' with a JSON ` + + `diagnostic on stderr: ${result.stderr}` + ); + } + const diagnostic = asValeConfigError( + Array.isArray(parsed) ? parsed[0] : parsed + ); + if (diagnostic === undefined) { + throw new Error( + `Vale ${VALE_VERSION} answered with a payload that is not a config ` + + `error: ${result.stderr}` + ); + } + if (diagnostic.Code !== "E201") { + throw new Error( + `expected diagnostic code E201 for an unknown 'extends', got ` + + `${diagnostic.Code} ("${diagnostic.Text}"). Refusing to read a ` + + `vocabulary out of an error shape this test does not know.` + ); + } + + const listed = /^'extends'[^[\]]*\[([^\]]+)]\.?$/.exec( + diagnostic.Text.trim() + ); const names = listed?.[1]; if (names === undefined) { - throw new Error(`Vale no longer enumerates its check types: ${message}`); + throw new Error( + `Vale ${VALE_VERSION} answered "${diagnostic.Text}", which no longer ` + + `matches the enumeration shape this test reads. A short enum is ` + + `STRICTER than the binary, so a partial parse would start rejecting ` + + `rules Vale accepts. Fix the pattern against the new message.` + ); } - return names.split(/\s+/).filter(Boolean).toSorted(); + return names.trim().split(/\s+/).filter(Boolean).toSorted(); } /** diff --git a/packages/cli/test/verify-test-commands.test.ts b/packages/cli/test/verify-test-commands.test.ts index e66279e5..2988531b 100644 --- a/packages/cli/test/verify-test-commands.test.ts +++ b/packages/cli/test/verify-test-commands.test.ts @@ -3,6 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; + import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { findValeBinary } from "../src/rules/vale/binary";