From 535ba8758ce4698c7b5f7b11c9ff86f4e82d0639 Mon Sep 17 00:00:00 2001 From: Shantanu Agarwal Date: Sat, 14 Feb 2026 22:06:04 +0530 Subject: [PATCH 1/6] feat(dirty_mvp): initial spec and implementation --- .github/workflows/ci.yml | 113 +- .gitignore | 3 + CHANGELOG.md | 2 + changelog/v0.2.3.md | 27 + changelog/v0.2.3.yaml | 56 + docs/agents/agents.md | 10 +- docs/agents/manifest.json | 12 +- .../trinity_spec_deep_review_2026-02-13.md | 415 ++ docs/designs/trinity_conformance_checklist.md | 51 + docs/designs/trinity_spec.md | 1161 +++ docs/designs/trinity_state_machine.json | 301 + .../design/migration_system_spec.md | 17 +- docs/developers/getting_started.md | 12 +- docs/developers/index.md | 1 + docs/developers/reference.md | 28 +- .../developers/tools/trinity_observability.md | 168 + docs/developers/workflows/discovery.md | 2 +- docs/developers/workflows/spec_to_impl.md | 7 +- prompts/prompt_00_project_charter.md | 4 +- prompts/prompt_01_capabilities.md | 4 +- prompts/prompt_02_system_sketch.md | 4 +- prompts/prompt_02a_delivery_baseline.md | 4 +- prompts/prompt_03_glossary.md | 4 +- prompts/prompt_04_functional_requirements.md | 4 +- prompts/prompt_05_interface_contracts.md | 4 +- prompts/prompt_06_invariants.md | 4 +- prompts/prompt_07_nfrs.md | 4 +- prompts/prompt_08_fixtures.md | 4 +- prompts/prompt_09_impl_plan.md | 4 +- prompts/prompt_10_governance.md | 4 +- prompts/prompt_11_redteam.md | 5 +- prompts/prompt_12_ci_gates.md | 4 +- prompts/prompt_13_extension_generator.md | 2 +- prompts/prompt_13a_completeness_assessment.md | 4 +- prompts/prompt_14_roadmap.md | 4 +- prompts/prompt_15_scaffold.md | 4 +- prompts/prompt_16_impl_context.md | 1803 +---- prompts/prompt_16a_impl_planner.md | 222 +- prompts/prompt_16b_impl_coder.md | 89 +- prompts/prompt_16c_impl_reviewer.md | 113 +- prompts/trinity/70_researcher.md | 133 + prompts/trinity/80_tool_usage.md | 157 + prompts/trinity/90_summarizer.md | 114 + prompts/trinity/99_auditor.md | 156 + schema/16_impl_context.schema.json | 3253 +++++---- schema/trinity/context_pack.schema.json | 314 + schema/trinity/eval_export_row.schema.json | 306 + schema/trinity/log_capture_policy.schema.json | 236 + schema/trinity/scratchpad_state.schema.json | 138 + schema/trinity/session_event.schema.json | 1084 +++ schema/trinity/session_state.schema.json | 132 + schema/trinity/spawn_log.schema.json | 112 + schema/trinity/task_input.schema.json | 154 + schema/trinity/task_result.schema.json | 197 + schema/trinity/tool_call_request.schema.json | 727 ++ schema/trinity/tool_call_result.schema.json | 944 +++ schema/trinity/utility_call.schema.json | 44 + schema/trinity/utility_result.schema.json | 109 + spec/common/seed_manifest.json | 131 - tests/fixtures/step_16/invalid_bad_enum.json | 6 + ...delivery_planned_missing_verification.json | 89 + ...lid_delivery_planned_unverified_items.json | 97 + .../step_16/invalid_invalid_layer.json | 6 + .../step_16/invalid_invalid_type.json | 6 + .../step_16/invalid_missing_evidence.json | 6 + .../step_16/invalid_missing_fixture_ref.json | 6 + .../step_16/invalid_missing_nfr_refs.json | 6 + .../invalid_missing_plan_docs_impact.json | 59 + ...alid_missing_plan_review_requirements.json | 60 + .../invalid_missing_plan_spec_alignment.json | 39 + .../step_16/invalid_missing_plan_summary.json | 54 + ...id_delivery_planned_with_verification.json | 106 + .../step_16/valid_empty_execution_review.json | 11 +- tests/fixtures/step_16/valid_full.json | 21 +- tests/fixtures/step_16/valid_minimal.json | 11 +- tests/integration/test_step_16.py | 204 + .../test_step_16_spec_ref_grounding.py | 207 + tests/integration/test_trinity_eval_replay.py | 680 ++ .../test_trinity_runtime_orchestration.py | 1664 +++++ .../test_trinity_runtime_validation.py | 1810 +++++ tools/pyproject.toml | 2 +- tools/schema_registry.json | 15 +- tools/specdev_tools/cli.py | 308 +- tools/specdev_tools/seed_lint.py | 10 +- tools/specdev_tools/trinity_dashboard.py | 141 + tools/specdev_tools/trinity_eval_export.py | 260 + tools/specdev_tools/trinity_eval_publish.py | 259 + tools/specdev_tools/trinity_remediation.py | 284 + tools/specdev_tools/trinity_replay.py | 237 + tools/specdev_tools/trinity_runtime.py | 6459 +++++++++++++++++ .../specdev_tools/trinity_runtime_validate.py | 1485 ++++ tools/specdev_tools/validate.py | 89 +- tools/specdev_tools/validators/step_16.py | 675 +- 93 files changed, 24640 insertions(+), 3857 deletions(-) create mode 100644 changelog/v0.2.3.md create mode 100644 changelog/v0.2.3.yaml create mode 100644 docs/audit/trinity_spec_deep_review_2026-02-13.md create mode 100644 docs/designs/trinity_conformance_checklist.md create mode 100644 docs/designs/trinity_spec.md create mode 100644 docs/designs/trinity_state_machine.json create mode 100644 docs/developers/tools/trinity_observability.md create mode 100644 prompts/trinity/70_researcher.md create mode 100644 prompts/trinity/80_tool_usage.md create mode 100644 prompts/trinity/90_summarizer.md create mode 100644 prompts/trinity/99_auditor.md create mode 100644 schema/trinity/context_pack.schema.json create mode 100644 schema/trinity/eval_export_row.schema.json create mode 100644 schema/trinity/log_capture_policy.schema.json create mode 100644 schema/trinity/scratchpad_state.schema.json create mode 100644 schema/trinity/session_event.schema.json create mode 100644 schema/trinity/session_state.schema.json create mode 100644 schema/trinity/spawn_log.schema.json create mode 100644 schema/trinity/task_input.schema.json create mode 100644 schema/trinity/task_result.schema.json create mode 100644 schema/trinity/tool_call_request.schema.json create mode 100644 schema/trinity/tool_call_result.schema.json create mode 100644 schema/trinity/utility_call.schema.json create mode 100644 schema/trinity/utility_result.schema.json delete mode 100644 spec/common/seed_manifest.json create mode 100644 tests/fixtures/step_16/invalid_delivery_planned_missing_verification.json create mode 100644 tests/fixtures/step_16/invalid_delivery_planned_unverified_items.json create mode 100644 tests/fixtures/step_16/invalid_missing_plan_docs_impact.json create mode 100644 tests/fixtures/step_16/invalid_missing_plan_review_requirements.json create mode 100644 tests/fixtures/step_16/invalid_missing_plan_spec_alignment.json create mode 100644 tests/fixtures/step_16/invalid_missing_plan_summary.json create mode 100644 tests/fixtures/step_16/valid_delivery_planned_with_verification.json create mode 100644 tests/integration/test_step_16_spec_ref_grounding.py create mode 100644 tests/integration/test_trinity_eval_replay.py create mode 100644 tests/integration/test_trinity_runtime_orchestration.py create mode 100644 tests/integration/test_trinity_runtime_validation.py create mode 100644 tools/specdev_tools/trinity_dashboard.py create mode 100644 tools/specdev_tools/trinity_eval_export.py create mode 100644 tools/specdev_tools/trinity_eval_publish.py create mode 100644 tools/specdev_tools/trinity_remediation.py create mode 100644 tools/specdev_tools/trinity_replay.py create mode 100644 tools/specdev_tools/trinity_runtime.py create mode 100644 tools/specdev_tools/trinity_runtime_validate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41a1e8b6..16082e92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,22 @@ on: branches: ["**"] pull_request: workflow_dispatch: + inputs: + require_trinity_logs: + description: "Fail observability job when no Trinity session logs are discovered." + required: false + type: boolean + default: false + trinity_logs_glob: + description: "Optional additional recursive glob for Trinity session logs." + required: false + type: string + default: "" + require_eval_publish: + description: "Require external eval publish to succeed." + required: false + type: boolean + default: false schedule: - cron: "0 2 * * *" # daily drift audit (UTC) @@ -65,10 +81,105 @@ jobs: - name: Lint adversarial fixtures (same mechanism as fixtures) run: python -m specdev_tools.cli fixtures-lint spec + trinity-observability: + name: Trinity Replay & Eval Dashboard + needs: validate + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + cache: "pip" + - name: Install tooling + run: pip install -e tools/ + - name: Discover Trinity session logs + id: discover + env: + TRINITY_LOGS_GLOB: ${{ inputs.trinity_logs_glob }} + run: | + python - <<'PY' + import glob + import os + paths = set(glob.glob(".trinity/sessions/*.jsonl")) + paths.update(glob.glob("**/.trinity/sessions/*.jsonl", recursive=True)) + extra = os.getenv("TRINITY_LOGS_GLOB", "").strip() + if extra: + paths.update(glob.glob(extra, recursive=True)) + paths = sorted(paths) + os.makedirs(".trinity/eval", exist_ok=True) + with open(".trinity/eval/session_logs.txt", "w", encoding="utf-8") as f: + for p in paths: + f.write(p + "\n") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"count={len(paths)}\n") + print(f"Discovered {len(paths)} session logs") + PY + - name: Enforce logs for verification run + if: github.event_name == 'workflow_dispatch' && inputs.require_trinity_logs == true && steps.discover.outputs.count == '0' + run: | + echo "require_trinity_logs=true but no session logs were found." >&2 + exit 1 + - name: Build replay/export artifacts + if: steps.discover.outputs.count != '0' + run: | + while IFS= read -r log; do + [ -z "$log" ] && continue + base="$(basename "$log" .jsonl)" + rows_out=".trinity/eval/${base}_rows.jsonl" + replay_out=".trinity/eval/${base}_replay.json" + remediation_out=".trinity/eval/${base}_remediation.json" + python -m specdev_tools.cli trinity-export-eval "$log" --repo-root . --out "$rows_out" + python -m specdev_tools.cli trinity-replay "$log" --repo-root . --out "$replay_out" + python -m specdev_tools.cli trinity-remediate "$replay_out" --repo-root . --session-log "$log" --out "$remediation_out" --json > /dev/null + done < .trinity/eval/session_logs.txt + - name: Build dashboard summary + if: steps.discover.outputs.count != '0' + run: | + python -m specdev_tools.cli trinity-dashboard \ + --rows-glob ".trinity/eval/*_rows.jsonl" \ + --replay-glob ".trinity/eval/*_replay.json" \ + --out-json ".trinity/eval/dashboard.json" \ + --out-md ".trinity/eval/dashboard.md" + cat .trinity/eval/dashboard.md >> "$GITHUB_STEP_SUMMARY" + - name: Publish eval bundle (optional external sink) + if: steps.discover.outputs.count != '0' + env: + TRINITY_EVAL_EXPORT_ENDPOINT: ${{ secrets.TRINITY_EVAL_EXPORT_ENDPOINT }} + TRINITY_EVAL_EXPORT_TOKEN: ${{ secrets.TRINITY_EVAL_EXPORT_TOKEN }} + run: | + REQUIRE_PUBLISH="" + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.require_eval_publish }}" = "true" ]; then + REQUIRE_PUBLISH="--require-publish" + fi + python -m specdev_tools.cli trinity-publish-eval \ + --rows-glob ".trinity/eval/*_rows.jsonl" \ + --replay-glob ".trinity/eval/*_replay.json" \ + --dashboard-json ".trinity/eval/dashboard.json" \ + --out ".trinity/eval/export_bundle.json" \ + --endpoint-env "TRINITY_EVAL_EXPORT_ENDPOINT" \ + --auth-token-env "TRINITY_EVAL_EXPORT_TOKEN" \ + --source "github-actions:${GITHUB_REPOSITORY}:${GITHUB_RUN_ID}" \ + ${REQUIRE_PUBLISH} + - name: No logs summary + if: steps.discover.outputs.count == '0' + run: | + echo "## Trinity Eval Dashboard" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "- No session logs found in this run." >> "$GITHUB_STEP_SUMMARY" + - name: Upload Trinity observability artifacts + uses: actions/upload-artifact@v4 + with: + name: trinity-observability + path: .trinity/eval/ + if-no-files-found: warn + deploy-staging: name: Deploy Staging (placeholder) if: ${{ github.ref == 'refs/heads/main' }} - needs: redteam + needs: [redteam, trinity-observability] runs-on: ubuntu-latest steps: - run: echo "Deploy to staging goes here (wire your real deploy step)." diff --git a/.gitignore b/.gitignore index 1ee0b9b8..d2d52412 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ devspec_env/ .vscode/ .idea/ .DS_Store + +# Trinity runtime +.trinity/ diff --git a/CHANGELOG.md b/CHANGELOG.md index cf43b1dc..c232a26a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ All detailed version records are stored in the [`changelog/`](./changelog/) dire | Version | Release Date | Documentation | Migration Spec | Status | | :--- | :--- | :--- | :--- | :--- | +| **[0.2.3]** | 2026-02-12 | [v0.2.3.md](changelog/v0.2.3.md) | [v0.2.3.yaml](changelog/v0.2.3.yaml) | ⚠️ **Breaking** | +| **[0.2.2]** | 2026-02-12 | [v0.2.2.md](changelog/v0.2.2.md) | [v0.2.2.yaml](changelog/v0.2.2.yaml) | ✅ Patch | | **[0.2.1]** | 2026-02-07 | [v0.2.1.md](changelog/v0.2.1.md) | [v0.2.1.yaml](changelog/v0.2.1.yaml) | ⚠️ **Breaking** | | **[0.2.0]** | 2026-01-17 | [v0.2.0.md](changelog/v0.2.0.md) | [v0.2.0.yaml](changelog/v0.2.0.yaml) | ⚠️ **Breaking** (Schema Hardening) | | **[0.1.1]** | 2026-01-13 | [v0.1.1.md](changelog/v0.1.1.md) | [v0.1.1.yaml](changelog/v0.1.1.yaml) | ✅ Patch (Fixes) | diff --git a/changelog/v0.2.3.md b/changelog/v0.2.3.md new file mode 100644 index 00000000..6429aa0d --- /dev/null +++ b/changelog/v0.2.3.md @@ -0,0 +1,27 @@ +# v0.2.3 - 2026-02-12 + +### Added +- Step 16 schema now requires all core plan sections: `summary`, `docs_impact`, `spec_alignment`, and `review_requirements`. +- Structured reviewer delivery verification fields: + - `review.delivery_status.dashboards_verified[]` + - `review.delivery_status.alerts_verified[]` + +### Changed +- If `plan.delivery.status == planned`, reviewer output must include delivery verification entries. +- Step 16 deep validator now enforces planned dashboard/alert mapping to matching verified entries. +- Step 16 prompts were aligned: + - Planner prompt schema guidance includes required `docs_impact`. + - Reviewer prompt guidance uses `dashboards_verified` / `alerts_verified` and planned-delivery gate rule. + - Anchor prompt quick reference now marks `plan.review_requirements` as required. +- Developer workflow docs now reflect required Step 16 plan sections and structured delivery verification evidence. + +### Testing +- Added failing fixtures for missing required plan sections. +- Added valid/invalid fixtures for planned delivery verification behavior. +- Added integration tests for new Step 16 schema and validator expectations. + +### Breaking +- Existing Step 16 artifacts that omit any required plan section will fail validation. +- Existing Step 16 artifacts with `plan.delivery.status == planned` but missing reviewer verification evidence will fail validation. + +[0.2.3]: https://github.com/vichitracollective/devspec_toolkit/releases/tag/v0.2.3 diff --git a/changelog/v0.2.3.yaml b/changelog/v0.2.3.yaml new file mode 100644 index 00000000..94603164 --- /dev/null +++ b/changelog/v0.2.3.yaml @@ -0,0 +1,56 @@ +version: "0.2.3" +release_date: "2026-02-12" +breaking: true +description: | + Hardens Step 16 plan requirements, adds structured delivery verification fields + for reviewer artifacts, and aligns validator/test/prompt behavior with the new contract. + +changes: + - type: add_constraint + step_id: "16_impl_context" + path: "plan.required" + description: "Step 16 plan now requires summary, docs_impact, spec_alignment, and review_requirements." + migration: + action: ai_assisted + prompt: template_add_required_fields.md + + - type: add_field + step_id: "16_impl_context" + path: "review.delivery_status.dashboards_verified[]" + required: false + migration: + action: auto + operations: + - add_field + + - type: add_field + step_id: "16_impl_context" + path: "review.delivery_status.alerts_verified[]" + required: false + migration: + action: auto + operations: + - add_field + + - type: add_constraint + step_id: "16_impl_context" + path: "plan.delivery.status=planned -> review.delivery_status" + description: "If plan.delivery.status is planned, review.delivery_status must include at least one verification entry." + migration: + action: ai_assisted + prompt: template_add_field.md + + - type: add_constraint + step_id: "16_impl_context" + path: "plan.delivery.(dashboards|alerts) -> review.delivery_status.(dashboards_verified|alerts_verified)" + description: "Planned dashboard/alert items must have matching verified entries in reviewer output." + migration: + action: ai_assisted + prompt: template_add_field.md + + - type: add_constraint + step_id: "16_impl_context" + path: "prompts/16a,16c + validators/step_16 + tests/fixtures/step_16" + description: "Prompts, deep validator, and integration fixtures/tests are aligned with the new schema constraints." + migration: + action: auto diff --git a/docs/agents/agents.md b/docs/agents/agents.md index 5386d2c8..84aca35c 100644 --- a/docs/agents/agents.md +++ b/docs/agents/agents.md @@ -33,10 +33,11 @@ This toolkit uses a two‑phase interaction to maximize completeness without har - Stop and wait for human answers. 4. **Phase B — Emit** - Once answers resolve gating items, run the same prompt to generate the artifact. - - Output exactly one fenced `json` code block that validates against the embedded schema; no extra prose. + - Write or update the artifact JSON file at the expected `spec/...` path. + - Return only a concise confirmation with artifact path and validation status (no fenced JSON). - Populate `seed_refs` with the seeds actually used. 5. **Persist Artifact** - - Replace the contents of `spec/NN_name.json` with the generated block. + - Ensure `spec/NN_name.json` was written with the generated content. - Preserve the `$schema` field already present in the file. 6. **Validate** - Run the [core validation commands](../developers/reference.md#core-validation-commands) and inspect results. @@ -58,7 +59,7 @@ This toolkit uses a two‑phase interaction to maximize completeness without har | 13a | `spec/13a_completeness_assessment.json` | Check for gaps before Roadmap | | 14 | `spec/14_roadmap.json` | Initiate JIT implementation loop | | 15 | `spec/15_scaffold.json` | Implement scaffold manually | -| 16a | `spec/impl_context/{step_id}.json` | Trinity Plan: Tasks, Security, Delivery, Drift | +| 16a | `spec/impl_context/{step_id}.json` | Trinity Plan: Checklist, Security, Delivery, Drift | | 16b | `spec/impl_context/{step_id}.json` | Trinity Build: Code, Configs, Docs | | 16c | `spec/impl_context/{step_id}.json` | Trinity Review: Verification & closure | @@ -79,7 +80,7 @@ When reporting back to humans, include: 2. Commands executed and whether they succeeded. 3. Outstanding issues or reasons for escalation. -During Phase A (Clarify), output only a bulleted list of questions grouped by topic. During Phase B (Emit), output only the single fenced `json` block. +During Phase A (Clarify), output only a bulleted list of questions grouped by topic. During Phase B (Emit), write the artifact to disk and output only a concise confirmation (no fenced JSON). ## 8. Runner Tips - Treat prompts as the contract; do not modify them at runtime. @@ -87,6 +88,7 @@ During Phase A (Clarify), output only a bulleted list of questions grouped by to - Read only the paths listed in the prompt’s “Context To Ingest” for that step; avoid external sources. - Build private ledgers in memory only; never persist them or include them in outputs. - In Phase A, emit only grouped, concise questions; never include JSON, code fences, or speculative answers. +- In Phase B, treat the filesystem artifact as the source of truth and return concise status text only; do not paste artifact JSON in chat. - Stop generation when the self‑audit gate is not met; wait for human input rather than guessing. - Prefer deterministic decoding to keep outputs stable across retries. - De‑duplicate questions and prioritize gating items that block emission. diff --git a/docs/agents/manifest.json b/docs/agents/manifest.json index 0404f617..1d156058 100644 --- a/docs/agents/manifest.json +++ b/docs/agents/manifest.json @@ -18,11 +18,13 @@ "emit": { "purpose": "Generate the canonical JSON artifact.", "output_contract": { - "type": "json_artifact", - "fenced_code_block": true, - "language": "json", + "type": "disk_artifact_status", + "fenced_code_block": false, + "language": "none", "restrictions": [ - "Exactly one fenced json block" + "Write artifact to disk first", + "Return concise status only", + "Do not emit fenced JSON" ] } } @@ -77,4 +79,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/docs/audit/trinity_spec_deep_review_2026-02-13.md b/docs/audit/trinity_spec_deep_review_2026-02-13.md new file mode 100644 index 00000000..b78be032 --- /dev/null +++ b/docs/audit/trinity_spec_deep_review_2026-02-13.md @@ -0,0 +1,415 @@ +# Trinity Agent Specification — Deep Review Report + +**Date**: 2026-02-13 +**Reviewer**: AI Agent (Antigravity) +**Scope**: Full review of `trinity_spec.md`, Step 16 prompt system, state machine, utility prompts, and Trinity runtime schemas +**Artifacts Reviewed**: +- [trinity_spec.md](file:///Users/vichitracollective/vc-code/devspec_toolkit/devspec_toolkit/docs/designs/trinity_spec.md) (1162 lines, 18 sections) +- [trinity_state_machine.json](file:///Users/vichitracollective/vc-code/devspec_toolkit/devspec_toolkit/docs/designs/trinity_state_machine.json) +- [prompt_16_impl_context.md](file:///Users/vichitracollective/vc-code/devspec_toolkit/devspec_toolkit/prompts/prompt_16_impl_context.md) +- [prompt_16a_impl_planner.md](file:///Users/vichitracollective/vc-code/devspec_toolkit/devspec_toolkit/prompts/prompt_16a_impl_planner.md) +- [prompt_16b_impl_coder.md](file:///Users/vichitracollective/vc-code/devspec_toolkit/devspec_toolkit/prompts/prompt_16b_impl_coder.md) +- [prompt_16c_impl_reviewer.md](file:///Users/vichitracollective/vc-code/devspec_toolkit/devspec_toolkit/prompts/prompt_16c_impl_reviewer.md) +- Utility prompts: `70_researcher.md`, `80_tool_usage.md`, `90_summarizer.md`, `99_auditor.md` +- Trinity schemas: 13 files under `schema/trinity/` +- Unstaged changes: only `migration_system_spec.md` (disk-first IO fix) + +--- + +## Table of Contents + +1. [Executive Summary](#1-executive-summary) +2. [Alignment with DevSpec Toolkit and Step 16](#2-alignment-with-devspec-toolkit-and-step-16) +3. [Context Management and Passing](#3-context-management-and-passing) +4. [Context Ingestion Flow](#4-context-ingestion-flow) +5. [Tool Definitions and Protocols](#5-tool-definitions-and-protocols) +6. [Session and Context Management](#6-session-and-context-management) +7. [Agent Protocol and Architecture (Deep Review)](#7-agent-protocol-and-architecture-deep-review) +8. [Logging Infrastructure](#8-logging-infrastructure) +9. [Correctness, Completeness, Consistency Audit](#9-correctness-completeness-consistency-audit) +10. [Gaps, Bugs, and Scope of Improvement](#10-gaps-bugs-and-scope-of-improvement) +11. [Assumptions and Hallucination Risks](#11-assumptions-and-hallucination-risks) +12. [Usefulness Assessment](#12-usefulness-assessment) + +--- + +## 1. Executive Summary + +The Trinity Agent specification is an **impressively thorough and architecturally sound** document. At 1162 lines covering 18 sections, it defines a complete lifecycle engine for automating DevSpec Step 16 implementation loops using LLM-driven agents. + +**Overall Assessment: 8.2/10** + +### Strengths +- **Exhaustive formal rigor**: Every protocol, artifact contract, and state transition is explicitly defined. The spec reads more like a RFC or protocol standard than a design doc. +- **Anti-hallucination focus**: The Zero-Assumption Protocol, evidence binding, and spec_ref grounding requirements are well-conceived defenses against the primary LLM failure mode. +- **Fractal architecture is elegant**: The three-level loop model (L1 macro → L2 persona → L3 atomic) provides clean separation of concerns and composable retry semantics. +- **Disk-first artifact exchange**: Using filesystem artifacts as the single source of truth between agents eliminates the context-window coupling problem that plagues most multi-agent setups. +- **Schema governance**: 13 Trinity-specific schemas enforce machine-checkable contracts at every boundary. +- **Remediation always routes through Planner first**: This prevents Builder from ad-hoc self-replanning, which is a common source of scope drift. +- **Logging is designed for downstream ML/eval**: The session event schema, evidence binding, and export adapters show clear thinking about the fine-tuning pipeline. + +### Weaknesses +- **Complexity cliff**: The sheer volume of normative requirements makes implementation daunting. Many rules are specified but the priority/MVP-criticality is unclear. +- **Runtime code is nascent**: The extensive spec far outpaces the current implementation. `.trinity/` directory exists but most runtime behaviors are not yet implemented. +- **Some internal inconsistencies and numbering errors** (dual §4.5, dual §7.6) that reduce document reliability. +- **Token budget arithmetic is under-specified** for real-world context windows. +- **Missing concrete integration tests** or conformance tests that would prove the spec works end-to-end. + +--- + +## 2. Alignment with DevSpec Toolkit and Step 16 + +### Strong Alignment ✅ + +| Aspect | Assessment | +|--------|-----------| +| Prompt Map (§2.2) | Correctly maps Planner→16a, Builder→16b, Verifier→16c to existing DevSpec prompts | +| Schema Authority | All prompts reference `schema/16_impl_context.schema.json` as the single source of truth | +| Checklist-First Model | Trinity uses `plan.spec_alignment.checklist[]` — aligned with prompt 16a/16b/16c contracts | +| Disk-First IO | Each prompt has matching Phase A (questions) / Phase B (artifact on disk) — matches §5 exactly | +| Zero-Assumption Protocol | All four prompts enforce it; no room for guessed values | +| Seed Manifest Governance | §4.1 correctly requires `seed_manifest.json` first; all prompts follow `global_seed_order` | +| Evidence Binding | §8.1 matches `prompt_16b`'s EVIDENCE BINDING section (SHA-256 hashing, verbatim excerpts) | +| Forbidden Actions | Prompts and spec are aligned — no `plan.tasks`, no `metadata`, no placeholder hashes | + +### Gaps / Misalignments ⚠️ + +| ID | Finding | Severity | +|----|---------|----------| +| A-1 | **Anchor prompt (16) is underweighted in Trinity spec**. The Orchestrator's prompt source (§2.2 line 108) lists `prompt_16_impl_context.md` but the spec never clearly defines when the Orchestrator invokes the anchor prompt vs. just running transition logic. The anchor regeneration policy (§3.3) and the anchor prompt mechanics are described in different sections without a clear connection. | Major | +| A-2 | **Manual vs Trinity harness mode is documented in each prompt BUT the divergence semantics differ subtly**. §14.3 defines the Trinity/Manual divergence, but each prompt's "Output Mode" section has slightly different wording. These should converge on a single normative reference. | Minor | +| A-3 | **Prompt 16a allows seed expansion (up to 5 new seeds)** but the spec's §4.5 "Seed Mutation Ownership" says "owned by Orchestrator + Planner phases only." This is consistent, but the spec never says what happens when Planner adds seeds mid-plan — does the Orchestrator need to create a new spec baseline commit (§10.2)? The chain of custody is implied but not explicit. | Major | + +--- + +## 3. Context Management and Passing + +### Architecture Assessment: Well-Designed ✅ + +The context passing model (§4.4, §9.6.7) is the spec's strongest architectural feature: + +1. **Pointers first, bulk only when necessary** — This is the right approach for bounded context windows. +2. **ContextResolver and SpecRefResolver** (§4.2, §4.3) are deterministic resolvers that eliminate ambiguity about what context an agent sees. +3. **Context pack budget** (§4.5 on line 207) enforces hard/soft token limits and graceful degradation via truncation. + +### Issues ⚠️ + +| ID | Finding | Severity | +|----|---------|----------| +| C-1 | **Context pack `allowed_read_paths` is never narrowed for Builder/Verifier**. §4.2 says `ContextResolver` returns allowed read paths, but the spec never defines per-phase narrowing rules. The `context_pack.schema.json` has `allowed_read_paths` as an array, but there's no normative rule saying Builder gets a narrower set than Planner. In practice, Builder should NOT be able to read `spec/common/seed_manifest.json` for mutation, but it could still read it for context — the distinction between "read for context" vs "read for mutation intent" isn't enforced at the tool layer. | Major | +| C-2 | **Token budget truncation by reverse `global_seed_order` is sound** but the spec doesn't define what happens when a *critical* seed (e.g., `spec/04_fr_list.json`) gets truncated because lower-priority seeds filled the budget. There's no priority override mechanism for must-have seeds. | Minor | +| C-3 | **`context_pack.json` is generated alongside `task_input.json`** but the spec doesn't specify ordering guarantees. If context pack generation fails halfway (e.g., missing seed file), is `task_input.json` already written? The transaction boundary contract (state machine §76) addresses child handoffs but not context pack creation atomicity. | Minor | +| C-4 | **Staleness check for `spec_ref` (§4.3 line 204-205)** is well-conceived — comparing `commit_hash` vs HEAD and emitting `drift` warnings — but the spec doesn't define the threshold at which drift becomes blocking. A single character change in one line of a spec file triggers a drift warning, but that could be a comment edit or a critical contract change. No severity classification for drift signals. | Minor | + +--- + +## 4. Context Ingestion Flow + +### Assessment: Correct and Robust ✅ with caveats + +The ingestion flow follows a clean producer → pass → review → ingest pipeline documented in §9.6: + +``` +Parent creates task_input.json + context_pack.json +→ Child reads task_input.json +→ Child loads governed context from context_pack pointers +→ Child executes (using tools within scope) +→ Child writes task_result.json +→ Parent ingests task_result.json + validates +→ Parent merges validated results into milestone artifact +``` + +### Issues ⚠️ + +| ID | Finding | Severity | +|----|---------|----------| +| I-1 | **No explicit schema for `context_pack.json` field `seed_files_ordered`**. The spec says it contains "already resolved by ContextResolver" content (§9.6.1 line 721), but `context_pack.schema.json` would need to encode the resolution format. I verified the schema exists but didn't see if it enforces this. | Minor | +| I-2 | **Child loading strategy is undefined**. §4.4 says "target_files (paths only; child loads file content as needed)" but there's no specification for *when* or *how* the child reads those files. Does it read them all at the start of execution? Lazily when needed? This matters for token budgets in the child's context window. | Major | +| I-3 | **Parent ingestion is field-level merge** (§9.3 line 636), but the merge strategy is described as "latest valid child artifact wins" (line 639). This is under-specified: if two consecutive Builder runs touch the same checklist item, is it the full checklist item that's replaced or individual fields within it? The granularity of merge isn't defined. | Major | +| I-4 | **Crash recovery ingestion path is partially specified**. §9.4 defines scratchpad recovery, but if a crash occurs between child writing `task_result.json` and parent reading it, the spec says "Crash consistency requirement: spawn IO write, validation result, and session-log event append must be atomic as a transaction boundary" (§9.3 line 640). In practice, this requires a write-ahead log or similar mechanism that isn't detailed. | Major | + +--- + +## 5. Tool Definitions and Protocols + +### Assessment: Comprehensive Foundation ✅ with notable gaps + +The tool protocol (§7) defines 9 categories of tools with typed contracts, deterministic behavior, and scope enforcement. + +### Strengths +- **Write-path guardrails** (§7.2): path allowlists at the tool layer prevent scope creep. +- **Command capture contract** (§7.3): every command invocation records `command`, `exit_code`, `duration_ms`, `timestamp`, `working_dir`. +- **Multi-tier JSON extraction** (§7.5): three-tier parser for LLM output JSON is a practical defense against malformed outputs. +- **Tool schema budget policy** (§7.6b, line 450): catalog-first with on-demand expansion is a smart token-saving strategy. + +### Gaps ⚠️ + +| ID | Finding | Severity | +|----|---------|----------| +| T-1 | **No `create_dir` tool**. The tool list supports `write_file` (which creates parent dirs?) and `list_dir`, but there's no explicit directory creation tool. If the Builder needs to create `src/new_module/`, does `write_file("src/new_module/init.py", ...)` implicitly create the directory? Not specified. | Minor | +| T-2 | **No `delete_file` or `remove_file` tool in §7.1**, but `move_file` and `remove_file` appear in the `session_event.schema.json` tool list (line 213-214). The spec's required tools (§7.1) and the schema's tool enum are misaligned. | Major | +| T-3 | **`exec_cmd` mode selection** (§7.4) says "use summarized mode when output length is expected to exceed bounded output thresholds" but doesn't define those thresholds numerically. The deterministic pre-scan (§7.4 line 423) is well-specified but the threshold for triggering summarized mode is left to implementation judgment. | Minor | +| T-4 | **No interactive/streaming command support**. `exec_cmd` assumes synchronous command execution with bounded output. For long-running dev servers, database migrations, or compilation jobs, there's no streaming or progress-reporting mechanism. The timeout configuration (§12.1 `child_timeout_seconds`) addresses wall-clock limits but not incremental progress. | Minor | +| T-5 | **apply_patch vs edit_file**: Both are listed but their relationship isn't defined. When should one be used over the other? | Nit | +| T-6 | **Table 18 (§18) acknowledges missing tools** (tree-sitter navigation, semantic diff, dependency graph, test isolation, artifact comparison) — these are honest and well-documented gaps, not findings. | Informational | + +--- + +## 6. Session and Context Management + +### Assessment: Well-Architected ✅ + +The session model (§9) is one of the spec's most impressive sections: + +- **Disk call stack** (§9.1): parent terminates before child runs, child writes result, parent resumes. This is a clean CSP-like process model. +- **Loop detection** (§9.2): tracking repeated spawn intents prevents infinite remediation loops. +- **Scratchpad lifecycle** (§9.4): create → persist on boundary → archive on success → recover on crash — comprehensive lifecycle. +- **Session log rotation** (§9.3 line 661): 5000-event threshold with compaction summary is practical. +- **Workspace cleanup** (§9.4 line 665): deferred vs eager modes give operational flexibility. + +### Issues ⚠️ + +| ID | Finding | Severity | +|----|---------|----------| +| S-1 | **Atomicity of parent terminate → child start** is the critical gap. §9.1 says "Parent terminates" and "Child executes." In practice, this means a separate process invocation. The spec doesn't define: (a) how the harness knows to start the child after parent terminates, (b) what watchdog/supervisor process manages the hand-off, (c) what happens if the machine reboots between parent terminate and child start. A process supervisor or job queue is implied but never specified. | Blocking | +| S-2 | **Scratchpad schema (`scratchpad_state.schema.json`) doesn't specify recovery fields**. §9.4 says "Scratchpad must include phase, checklist scope, last successful validation gate, and pending next action pointer" but these are normative spec requirements — whether the schema enforces them needs verification. | Major | +| S-3 | **No specification for partial milestone state**. If the system crashes after Builder completes 5/10 checklist items, the scratchpad/milestone artifact preserves which items are done, but the spec doesn't define: (a) does resume restart all of Builder's L2 loop or just the remaining items? (b) does the context pack need regeneration on resume? (c) are checkpoint commits from the interrupted run preserved or rolled back? | Major | +| S-4 | **Spawn directory naming convention** (`.trinity/runtime/spawns//`) doesn't define `child_id` format. Is it a UUID? A deterministic hash? This matters for debugging and for deterministic replay. | Minor | + +--- + +## 7. Agent Protocol and Architecture (Deep Review) + +### 7.1 Three-Level Fractal Model + +The L1/L2/L3 model is the right architecture. The key insight — that each level has the same Draft → Audit → Refine loop shape — enables compositional reasoning. + +**State Machine Analysis** (from `trinity_state_machine.json`): + +The state machine defines 9 states and 13 transitions. Key observations: + +| Property | Assessment | +|----------|-----------| +| Determinism | ✅ All transitions have explicit guards and actions | +| Completeness | ✅ Every non-terminal state has at least one outgoing transition | +| Terminal states | ✅ Three: `COMPLETED`, `BLOCKED`, `ERROR` | +| Recovery paths | ✅ `QUESTIONS_PENDING → $originating_phase` enables resume | +| Error handling | ✅ `t-any-to-error` covers infrastructure failures from all non-terminal states | + +**State Machine Gaps**: + +| ID | Finding | Severity | +|----|---------|----------| +| SM-1 | **`REMEDIATE_16A` has no retry transition**. `PLAN_16A` has `t-16a-retry`, but `REMEDIATE_16A` doesn't. If remediation planning fails validation once, it goes to `BLOCKED` with no retry opportunity. This is asymmetric with the initial planning state. | Major | +| SM-2 | **No `QUESTIONS_PENDING` → `BLOCKED` transition**. If the user never responds to a question, there's no timeout-to-blocked path. The system would wait indefinitely. | Minor | +| SM-3 | **Milestone retry cap interaction with per-phase caps is underspecified in the state machine**. The spec (§6.5 line 355) explains the semantics in prose, but the state machine doesn't encode the milestone-level retry counter or its interaction with per-phase counters. | Minor | +| SM-4 | **`t-remediate-to-16b` skips `16a → validate` pattern**. After remediation (which IS a 16a run), the transition goes directly to `BUILD_16B`. But shouldn't it go through the same phase gate as `t-16a-pass-to-16b`? The guards are the same (`phase=16a, all phase_gate_requirements.16a pass`), so functionally it's correct, but having a separate transition name could mask gate bypass bugs. | Nit | + +### 7.2 Persona Boundaries + +The spec correctly separates concerns: +- **Planner** creates plan; never executes code +- **Builder** executes plan; never creates/reorders checklist items +- **Verifier** audits evidence; never modifies implementation + +The "proposed_additions" mechanism (§6.2.2 line 305) for Builder to surface gaps without modifying the plan is a smart boundary-preserving pattern. + +### 7.3 Utility Sub-Agent Integration + +The four utility prompts are well-structured: + +| Utility | Input/Output Contract | Loop Policy | Assessment | +|---------|----------------------|-------------|-----------| +| Researcher | Structured JSON I/O | Draft→Review→Refine | ✅ Well-bounded | +| ToolUser | Structured JSON I/O | Exempt (deterministic) | ✅ Correct exemption | +| Summarizer | Structured JSON I/O | Draft→Review→Refine | ✅ Verbatim-only rule is critical for evidence | +| Auditor | Structured JSON I/O | Draft→Review→Refine | ✅ Severity policy enforcement | + +**Utility Gap**: + +| ID | Finding | Severity | +|----|---------|----------| +| U-1 | **No utility prompt for `Worker`**. §2.2 and §9.6.4 define the Worker role as the L3 atomic executor, but there's no dedicated prompt file for Worker. In practice, the Worker would use `prompt_16b_impl_coder.md` scoped to a single checklist item, but this should be explicit. | Minor | +| U-2 | **Utility fast-path (§2.3 line 134)** allows skipping file-based IO for simple utility calls, but the conditions for when this is safe aren't checkable. "Single artifact and single structured response" is subjective. | Minor | + +--- + +## 8. Logging Infrastructure + +### Assessment: Excellent for Eval/Fine-Tuning ✅ + +The logging design (§8, §17) is among the best I've reviewed for LLM eval purposes. Key strengths: + +1. **Chained event hashes** (`prev_event_sha256` + `event_sha256`): enables tamper detection and deterministic replay ordering. +2. **Outcome labels** (§17.1): `success`, `partial`, `failure`, `skip`, `retry` directly map to fine-tuning dataset labels. +3. **Correctness tags** (§17.2): `correct`, `self_corrected`, `failed` on tool calls enable interaction-level scoring. +4. **Quality metrics** (§17.4): aggregate metrics on session close provide per-run quality scorecards. +5. **Multi-format export** (§17.5): JSONL → eval-rows → summary adapters cover the full pipeline from raw logs to training data. +6. **Capture policy tuning** (§12 line 987-991): budget caps for full-capture prevent token-window inflation in logs. +7. **Custom redaction patterns** (§17.3): domain-specific redaction beyond built-in patterns shows operational maturity. + +### Issues ⚠️ + +| ID | Finding | Severity | +|----|---------|----------| +| L-1 | **`outcome_label` is in §17.1 but not in `session_event.schema.json`**. The schema's `metadata` object (line 338ff) doesn't include an `outcome_label` field. The spec defines it normatively but the schema doesn't enforce it — classic spec/schema drift. | Major | +| L-2 | **`correctness_tag` (§17.2) is also not in the schema**. Same issue as L-1. | Major | +| L-3 | **Quality metrics (§17.4) have no schema**. The "session close" event should include these metrics, but the `session_event.schema.json` doesn't have a conditional schema for TERMINATE events that requires quality metrics. | Major | +| L-4 | **Secret scanning is MVP-scoped** (§8.4): "best-effort redaction metadata" + "command allow/deny policy" is pragmatic, but the spec doesn't define the actual deny list. Common secret-leaking commands (`cat .env`, `printenv`, `history`, `env`) should be enumerated in the spec or in a policy file. The `80_tool_usage.md` utility prompt *does* include `forbidden_commands` in its input contract, which is good, but this should be normative at the harness level too. | Minor | +| L-5 | **No log rotation for `.trinity/sessions/*.jsonl` across milestone runs**. §9.3 defines intra-session rotation at 5000 events, but nothing prevents session files from accumulating across many milestone runs. A cleanup/archival policy for old sessions is missing. | Minor | + +--- + +## 9. Correctness, Completeness, Consistency Audit + +### Numbering and Structural Issues + +| ID | Finding | Severity | +|----|---------|----------| +| N-1 | **Duplicate §4.5**: Lines 207 and 246 both use section number 4.5 — first for "Context Pack Budget", second for "Seed Mutation Ownership". One should be §4.6. | Minor | +| N-2 | **Duplicate §7.6**: Lines 431 and 450 both use 7.6 — first for "Runtime Protocol Schemas", second for "Prompt-Side Tool Schema Budget Policy". One should be §7.7. | Minor | +| N-3 | **§4.4 appears after §4.5** (line 213 vs 207). The section numbering is out of order. | Minor | + +### Cross-Reference Consistency + +| ID | Finding | Severity | +|----|---------|----------| +| CR-1 | **State machine references `QUESTIONS_PENDING` and `ERROR` states** — both are defined in both the spec (§14.2 line 1039, §9) and the state machine JSON. ✅ Consistent. | | +| CR-2 | **Verdict enum** (`verified`, `deferred`, `rejected`) is consistent across §6.6, §9.6.2, and `prompt_16c` line 117-121. ✅ Consistent. | | +| CR-3 | **Tool list mismatch**: Spec §7.1 lists 9 tool categories but doesn't include `move_file` or `remove_file`, which appear in the `session_event.schema.json` tool enum (line 211-214). ⚠️ Inconsistent. | Major | +| CR-4 | **Phase gate requirements** in state machine match §6.1-6.3 normative expectations (schema valid, deep valid, checklist non-empty for 16a, etc.). ✅ | | + +### Schema-Spec Alignment + +| ID | Finding | Severity | +|----|---------|----------| +| SC-1 | **`task_input.schema.json` enforces all fields from spec §4.4** (protocol_version, child_id, parent_id, role, phase, step_id, task_description, expected_output_schema, context_pack_ref, target_files, spec_refs, role_metadata). ✅ Complete alignment. | | +| SC-2 | **`task_input.schema.json` `spec_refs` requires `path` and `commit_hash` are optional in schema** (not in `required`), but the spec §4.3 says they're mandatory for grounding. The schema allows partial spec refs; the spec doesn't. | Major | +| SC-3 | **Session event schema lacks §17 fields** (outcome_label, correctness_tag, quality_metrics). | Major | + +--- + +## 10. Gaps, Bugs, and Scope of Improvement + +### Critical Gaps + +| # | Gap | Impact | Recommendation | +|---|-----|--------|---------------| +| 1 | **No process supervisor specification** (S-1) | Parent→Child handoff has no watchdog. Crash between parent terminate and child start is unrecoverable. | Add a lightweight supervisor/queue specification. Could be as simple as a shell loop that reads pending spawn entries. | +| 2 | **Schema doesn't enforce spec assertions** (SC-2, L-1, L-2, L-3) | Spec says fields are mandatory but schemas allow omission. Runtime validation will accept non-compliant artifacts. | Update schemas to match spec normative requirements. This is straightforward but essential. | +| 3 | **Tool list inconsistency** (T-2, CR-3) | `move_file` and `remove_file` exist in schema but not in spec §7.1. Agents may reference tools that the spec doesn't authorize. | Add `move_file` and `remove_file` to §7.1 or remove from schema. | +| 4 | **Merge strategy under-specification** (I-3) | Field-level vs object-level merge for checklist items isn't defined. Conflicting Builder runs could silently lose data. | Define explicit merge granularity: full checklist-item replacement keyed by `id`, or field-level merge with explicit precedence rules. | +| 5 | **REMEDIATE_16A has no retry** (SM-1) | Remediation planning failure goes to BLOCKED immediately, while initial planning gets retry opportunities. Asymmetric behavior. | Add `t-remediate-retry` self-loop with same guards as `t-16a-retry`. | + +### Improvement Recommendations + +| # | Recommendation | Priority | +|---|---------------|----------| +| 1 | **Add a "Conformance Test Matrix"** mapping each normative spec assertion to a testable schema constraint, deep validator check, or integration test. §12 line 975 references `trinity_conformance_checklist.md` — this should be populated. | High | +| 2 | **Define an MVP scope checklist** explicitly. The spec has 18 sections of normative requirements. Define a minimal vertical slice (e.g., "one milestone, deterministic mode, no utility agents, no knowledge base") so implementation has a clear first target. | High | +| 3 | **Add concrete examples to §9.6**. The ingestion flow is described in prose but lacks worked examples showing actual JSON payloads flowing through the pipeline. The utility prompts (70/80/90/99) have examples — extend this pattern to L1/L2 flows. | Medium | +| 4 | **Specify child_id format** (S-4). Recommend `{role}-{phase}-{step_id}-{timestamp_ms}` for debuggability. | Low | +| 5 | **Add drift severity classification** (C-4). Comments-only drift → `info`, signature change → `major`, file deletion → `blocking`. | Low | +| 6 | **Add explicit `create_dir` tool and verify `move_file`/`remove_file` coverage** in §7.1. | Medium | + +--- + +## 11. Assumptions and Hallucination Risks + +The spec is remarkably assumption-free for a design document of this scope. The Zero-Assumption Protocol is threaded through every prompt and the spec itself. + +### Remaining Assumption Hotspots + +| # | Assumption | Risk | +|---|-----------|------| +| 1 | **LLM can reliably produce schema-valid JSON in one shot**. The three-tier parser (§7.5) is a defense, but the spec doesn't quantify expected success rate or define the retry budget for malformed outputs (distinct from semantic retries). | Medium | +| 2 | **Tool results are deterministic**. `exec_cmd` captures output, but the spec assumes commands produce consistent output across runs. Build tools, test runners, and linters may produce non-deterministic ordering. | Low | +| 3 | **Context pack content fits in the child's context window**. The token budget (§4.5) enforces limits on the context pack, but the system prompt (utility prompt) + context pack + tool responses could exceed the actual model's window. The spec refers to `limits.hard_token_limit` but doesn't account for system prompt overhead. | Medium | +| 4 | **Git operations succeed atomically**. Checkpoint commits (§10.3) assume git add + commit is atomic. In practice, concurrent processes or hooks could cause failures. | Low | +| 5 | **Single active agent process** (§1.3) — the spec enforces this by design, but there's no lockfile or mutex specification to prevent accidental concurrent invocations. A second `specdev trinity` launched accidentally could corrupt shared state. | Medium | + +--- + +## 12. Usefulness Assessment + +### Rating: 7.5/10 — Justified with caveats + +### Does This Make Sense? + +**Yes, unambiguously.** The DevSpec toolkit + Trinity agent addresses a real and significant gap in today's AI-assisted development: + +| Feature | What Existing Agents (OpenCode, Cursor, Claude Code, etc.) Do | What Trinity Does Differently | +|---------|------------------------------------------------------|------------------------------| +| **Spec grounding** | None — agents work from natural language | Every action traced to governed spec artifacts with commit hashes | +| **Evidence binding** | None — agents claim "tests pass" without proof | Verbatim evidence with SHA-256 hashing | +| **Scope enforcement** | None — agents freely modify any file | File-level allowlist via `target_file_patterns` | +| **Context continuity** | Conversation history (lossy, window-limited) | Filesystem artifacts (lossless, resumable) | +| **Quality gates** | None — user manually checks | Schema + deep validation at every phase boundary | +| **Multi-phase orchestration** | Single-shot or conversation-based | Formal state machine with plan→build→review loop | +| **Anti-hallucination** | Varies; mostly conversation-level | Systematic spec_ref grounding + Zero-Assumption Protocol | +| **Fine-tuning data** | Not captured | Structured session logs with outcome labels and correctness tags | + +### When Trinity > Existing Agents + +1. **Compliance-heavy projects**: Financial, medical, or government software where every code change must trace to a requirement. +2. **Large codebases with established specs**: If you have DevSpec artifacts, Trinity leverages them as guardrails. +3. **Unattended batch runs**: CI-driven implementation where there's no human in the loop. +4. **Team projects**: Where multiple contributors need a shared, auditable trail of AI-generated changes. +5. **Fine-tuning pipeline**: If you're building domain-specific coding models, Trinity's structured logs are training data gold. + +### When Existing Agents > Trinity + +1. **Greenfield prototyping**: When you don't have specs and just want to iterate fast. Trinity's overhead is unjustified. +2. **One-off fixes**: For a 5-minute bug fix, running a full 16a→16b→16c cycle is massive overkill. +3. **Exploratory development**: When requirements are fluid and you need creative freedom, not guardrails. +4. **Small solo projects**: If you're the only developer and the codebase fits in one context window, the orchestration overhead isn't worth it. + +### Justification Verdict + +> **Building Trinity is justified IF and ONLY IF**: +> +> 1. You plan to use the DevSpec toolkit for multiple projects (amortize the spec investment). +> 2. You want to build a fine-tuning dataset from structured implementation traces. +> 3. You need audit trails for compliance or team governance. +> 4. You're running against local/self-hosted LLMs where you control the full pipeline. +> +> If you're just doing personal projects with OpenCode/Claude Code, **the marginal benefit over existing agents doesn't justify the implementation cost** — at least not until Trinity's runtime is mature enough for one-command operation. + +### Implementation Readiness + +The spec is 95% complete as a design document. The implementation is ~10% complete (schemas exist, runtime directory structure exists, CLI entry point exists). To reach usable MVP: + +| Milestone | Effort Estimate | Complexity | +|-----------|----------------|------------| +| Core orchestrator (L1 state machine) | 2-3 weeks | High | +| Context resolver + pack builder | 1-2 weeks | Medium | +| LLM integration (OpenAI-compatible chat) | 1 week | Medium | +| Tool implementations (read/write/edit/exec) | 1-2 weeks | Medium | +| Schema validation gates | 1 week | Low | +| Session logging | 1 week | Medium | +| **Total MVP** | **7-10 weeks** | — | + +### Bottom Line + +Trinity is a **well-architected spec for a real problem**. It's not theoretical — the Step 16 prompts already work in manual mode, and Trinity automates the human-in-the-loop parts with formal contracts. The risk is that the implementation effort is substantial and the spec's comprehensiveness could make it rigid. + +**My recommendation**: Continue building, but aggressively scope the MVP to: +1. L1 state machine with deterministic mode first +2. One milestone end-to-end in deterministic mode +3. Add LLM mode once deterministic mode passes conformance +4. Defer utility agents (Researcher, ToolUser) to post-MVP + +This gets you a working system faster while preserving the spec's long-term architecture. + +--- + +## Appendix: Finding Summary by Severity + +| Severity | Count | IDs | +|----------|-------|-----| +| Blocking | 1 | S-1 | +| Major | 12 | A-1, A-3, C-1, I-2, I-3, I-4, T-2, SM-1, SC-2, SC-3, L-1, L-2 | +| Minor | 14 | A-2, C-2, C-3, C-4, I-1, T-1, T-3, T-4, S-2, S-3, S-4, L-4, L-5, SM-2, SM-3 | +| Nit | 5 | T-5, SM-4, N-1, N-2, N-3 | +| Informational | 3 | T-6, U-1, U-2 | +| **Total** | **35** | — | diff --git a/docs/designs/trinity_conformance_checklist.md b/docs/designs/trinity_conformance_checklist.md new file mode 100644 index 00000000..874c3f3f --- /dev/null +++ b/docs/designs/trinity_conformance_checklist.md @@ -0,0 +1,51 @@ +# Trinity Conformance Checklist + +This matrix maps normative Trinity requirements to machine-enforced controls (schema, deep validator, CLI defaults, and integration tests). + +## Core Contract Mapping + +| Rule ID | Normative Requirement | Enforcement Surface | +|---|---|---| +| TRI-CTX-001 | `context_pack.required_spec_refs` must be non-empty for `16a/16b/16c` | `schema/trinity/context_pack.schema.json`; `tools/specdev_tools/trinity_runtime_validate.py::_validate_context_pack_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_context_pack_missing_required_spec_refs_fails` | +| TRI-CTX-002 | `context_pack.seed_files_ordered` must include `seed_manifest.step_requirements[phase]` | `tools/specdev_tools/trinity_runtime_validate.py::_validate_context_pack_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_context_pack_missing_phase_required_seed_fails` | +| TRI-CTX-003 | `task_input.step_id` and `context_pack.step_id` must match | `tools/specdev_tools/trinity_runtime_validate.py::_validate_task_input_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_task_input_step_id_mismatch_context_pack` | +| TRI-GRD-001 | `required_spec_refs` must be git-grounded (`path`, `line_range`, `commit_hash`) | `tools/specdev_tools/trinity_runtime_validate.py::_validate_required_spec_refs_grounding`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_context_pack_invalid_ungrounded_spec_ref_commit` | +| TRI-PHASE-001 | `task_result.status=success` for `16a/16b/16c` must reference schema-valid Step 16 artifact(s) | `tools/specdev_tools/trinity_runtime_validate.py::_validate_task_result_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_task_result_success_artifact_must_be_step16_valid` | +| TRI-PHASE-002 | Phase completeness gates required for successful `16a/16b/16c` artifacts | `tools/specdev_tools/trinity_runtime_validate.py::_validate_task_result_deep`; `tests/integration/test_trinity_runtime_validation.py` phase-gate tests | +| TRI-LOG-001 | Session events must be hash-chained and contiguous | `tools/specdev_tools/trinity_runtime_validate.py::_validate_session_event_log_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_log_invalid_hash_chain` | +| TRI-LOG-002 | SPAWN and TERMINATE refs must be canonical and paired | `schema/trinity/session_event.schema.json`; `tools/specdev_tools/trinity_runtime_validate.py::_validate_session_event_log_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_spawn_ref_must_be_canonical`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_log_fails_when_spawn_not_terminated` | +| TRI-LOG-003 | Child handoff transaction must include pass VALIDATION for input and result closures | `tools/specdev_tools/trinity_runtime_validate.py::_validate_session_event_log_deep`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_log`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_log_missing_validation_closure_fails` | +| TRI-LOG-004 | Secret leakage in persisted log artifacts is blocked | `tools/specdev_tools/trinity_runtime_validate.py` sensitive detectors + enforcement; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_detects_sensitive_content` | +| TRI-LOG-005 | Unattended command safety policy blocks common secret-dumping command patterns | `tools/specdev_tools/trinity_runtime.py::ToolExecutor._exec_cmd`; `tests/integration/test_trinity_runtime_orchestration.py::test_exec_cmd_blocks_secret_dump_patterns` | +| TRI-TOOL-001 | Tool request payloads are typed by `tool_name` (no ad-hoc args) | `schema/trinity/tool_call_request.schema.json`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_request_schema`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_request_invalid_exec_cmd_args` | +| TRI-TOOL-002 | Tool result payloads are typed by `tool_name` with deterministic result envelope | `schema/trinity/tool_call_result.schema.json`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_result_schema` | +| TRI-TOOL-003 | Artifact hash pairing is bidirectional when artifact refs are present | `schema/trinity/tool_call_result.schema.json`; `schema/trinity/session_event.schema.json`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_result_artifact_hash_pairing` | +| TRI-TOOL-004 | Prompt-side tool schema payload uses catalog-first + on-demand full-schema expansion to control token budget | `docs/designs/trinity_spec.md` Section 7.6; `schema/trinity/session_event.schema.json` (`metadata.tool_schema_context` on `TOOL_CALL/TOOL_RESULT`); `tools/specdev_tools/trinity_runtime_validate.py::_validate_session_event_log_deep` (context consistency + schema-hash checks); `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_tool_call_missing_tool_schema_context_fails`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_session_event_tool_call_on_demand_context_requires_expanded_tool` | +| TRI-TOOL-005 | Checkpoint/request arg names remain schema-consistent (`branch_name`, `base_rev/head_rev`, `use_regex`) | `tools/specdev_tools/trinity_runtime.py`; `schema/trinity/tool_call_request.schema.json`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_request_checkpoint_branch_branch_name`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_request_git_diff_base_head`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_tool_call_request_search_text_use_regex` | +| TRI-TOOL-006 | `apply_patch` and content-bearing `read_file` are executable runtime capabilities (not catalog-only placeholders) | `tools/specdev_tools/trinity_runtime.py::ToolExecutor._apply_patch`; `tools/specdev_tools/trinity_runtime.py::ToolExecutor._read_file` | +| TRI-RUN-001 | Runtime entrypoint exposes one-milestone orchestration via `specdev trinity --step-id ` | `tools/specdev_tools/cli.py` (`trinity` subcommand); `tools/specdev_tools/trinity_runtime.py::run_trinity`; `tests/integration/test_trinity_runtime_orchestration.py::test_run_trinity_full_runtime_vertical_slice` | +| TRI-RUN-002 | Milestone targeting is explicit-first with deterministic fallback when omitted | `tools/specdev_tools/trinity_runtime.py::_pick_step_id`; `tests/integration/test_trinity_runtime_orchestration.py::test_run_trinity_fallback_step_selection` | +| TRI-RUN-003 | L1 orchestration enforces `16a -> 16b -> 16c` and planner-first remediation under retry caps | `tools/specdev_tools/trinity_runtime.py::run`; `tools/specdev_tools/trinity_runtime.py::_spawn_phase`; `tests/integration/test_trinity_runtime_orchestration.py::test_run_trinity_full_runtime_vertical_slice` | +| TRI-RUN-004 | Runtime child handoff writes and validates `context_pack/task_input/task_result` before ingestion | `tools/specdev_tools/trinity_runtime.py::_spawn_phase`; `tools/specdev_tools/trinity_runtime_validate.py`; `tests/integration/test_trinity_runtime_validation.py`; `tests/integration/test_trinity_runtime_orchestration.py` | +| TRI-RUN-005 | Verified closure syncs roadmap milestone status and validates session log integrity | `tools/specdev_tools/trinity_runtime.py::_update_roadmap_status`; `tools/specdev_tools/trinity_runtime.py::run`; `tests/integration/test_trinity_runtime_orchestration.py::test_run_trinity_full_runtime_vertical_slice` | +| TRI-RUN-006 | Runtime supports deterministic resume from latest persisted `session_state` | `tools/specdev_tools/cli.py` (`trinity --resume`); `tools/specdev_tools/trinity_runtime.py::_load_resume_state`; `tests/integration/test_trinity_runtime_orchestration.py::test_run_trinity_resume_from_latest_session_state` | +| TRI-RUN-007 | Builder executes checklist action contracts and Verifier audits per-checklist evidence linkage | `tools/specdev_tools/trinity_runtime.py::_builder_handler`; `tools/specdev_tools/trinity_runtime.py::_verifier_handler`; `schema/16_impl_context.schema.json` | +| TRI-RUN-008 | Runtime execution mode is explicit and configurable (`llm` default for OpenAI-compatible endpoints, `deterministic` compatibility mode for fixtures/offline runs) | `tools/specdev_tools/trinity_runtime.py` (`TrinityConfig.execution_mode`, `_llm_phase_handler`, deterministic handlers); `tools/specdev_tools/cli.py` (`trinity --mode`); `tests/integration/test_trinity_runtime_orchestration.py::test_run_trinity_llm_mode_openai_compatible_endpoint` | +| TRI-RPLY-001 | Replay strict mode default is fail-on-warning | `tools/specdev_tools/cli.py` (`trinity-replay` defaults); `tools/specdev_tools/trinity_replay.py`; `tests/integration/test_trinity_eval_replay.py` | +| TRI-RMED-001 | Remediation missing-source policy defaults to hard fail | `tools/specdev_tools/cli.py` (`trinity-remediate --missing-resume-source-policy hard`); `tools/specdev_tools/trinity_remediation.py`; `tests/integration/test_trinity_eval_replay.py::test_remediation_missing_resume_source_hard_policy` | +| TRI-RMED-002 | Planner-first remediation policy (no direct Builder remediation path) | `docs/designs/trinity_spec.md` remediation transitions and flow; Step 16 prompts (`prompt_16b_impl_coder.md`, `prompt_16c_impl_reviewer.md`) | +| TRI-EVAL-001 | Eval export rows follow schema and preserve lineage/hash fields | `schema/trinity/eval_export_row.schema.json`; `tools/specdev_tools/trinity_eval_export.py`; `tests/integration/test_trinity_runtime_validation.py::test_validate_runtime_eval_export_row_schema`; `tests/integration/test_trinity_eval_replay.py` | + +## Utility Prompt Contract Mapping + +| Rule ID | Requirement | Source | +|---|---|---| +| TRI-UTIL-001 | Utility roles use explicit input contract, output schema, stop conditions, prohibited behavior | `prompts/trinity/70_researcher.md`; `prompts/trinity/80_tool_usage.md`; `prompts/trinity/90_summarizer.md`; `prompts/trinity/99_auditor.md` | +| TRI-UTIL-002 | Utility outputs must be evidence-grounded and assumption-free | Same files as above (`Non-Negotiable Rules`, `Self-Check` sections) | + +## Gaps Explicitly Deferred + +| Deferred Item | Status | +|---|---| +| Secret-safety detector expansion beyond regex classes | Deferred by product decision (current metadata/content enforcement retained) | +| Broad multi-scenario runtime golden fixtures (beyond deterministic one-milestone vertical slice) | Deferred by product decision (single deterministic full-run integration fixture retained) | diff --git a/docs/designs/trinity_spec.md b/docs/designs/trinity_spec.md new file mode 100644 index 00000000..ed569cad --- /dev/null +++ b/docs/designs/trinity_spec.md @@ -0,0 +1,1161 @@ +# Trinity Automation System Specification + +## 1. System Overview +Trinity is a recursive, checklist-driven AI agent harness that automates the Step 16 implementation loop (`16a` planner, `16b` coder, `16c` reviewer) for a single DevSpec roadmap milestone at a time. + +Trinity uses filesystem artifacts as the authoritative shared state, strict parent-child process boundaries, and deterministic tool-call contracts. It is designed for unattended AI-driven runs (including local OpenAI-compatible LLM endpoints), while preserving reproducibility, validation, and auditability. + +Current maturity status: +- Trinity runtime orchestration (`specdev trinity`) supports LLM-driven state execution (`16a/16b/16c`) via OpenAI-compatible chat endpoints. +- Deterministic state handlers remain available as a compatibility strategy for offline testing, CI fixtures, and controlled replay scenarios. +- Prompt files are operational inputs in LLM strategy and governance contracts in both strategies. +- This document defines the normative target behavior and the compatibility surface. + +### 1.1 Problem Statement +In long-running agentic implementation loops, the main failure modes are: +1. Context loss from window limits. +2. Spec drift from seeded constraints. +3. Hallucinated APIs/files/contracts. +4. Infinite repair loops without stop conditions. +5. Unverified code changes without evidence-bound tests. + +### 1.2 System Goals +1. Spec authority: if it is not in governed spec artifacts, it does not exist. +2. Lossless persistence: any phase can hibernate/resume from disk. +3. Strict hierarchy: parent agents consume child artifacts, not child chats. +4. Self-correction: syntax, semantic, and policy checks run before human escalation. +5. Verified atomic units: checklist items close only with evidence-backed verification. + +### 1.3 Core Operating Philosophy +1. Single active agent process at a time. +2. Parent-only spawning and mediation. +3. Filesystem artifacts are the execution truth. +4. Fractal loops at three levels: + - L1: milestone orchestration. + - L2: persona work-unit loop. + - L3: atomic Draft -> Audit -> Refine -> Publish loop. + +### 1.4 Confirmed Operating Decisions +- Execution engine: `specdev` CLI harness (non-interactive). Chat UI is optional. +- Base branch: `main`. +- Milestone branch naming: `trinity/{step_id}`. +- Invocation scope: exactly one milestone per `specdev trinity` invocation. +- Milestone selection: explicit `step_id`; fallback only if omitted. +- Canonical artifact IO contract: disk-first two-phase contract (questions + filesystem artifact). +- Tool interaction protocol: structured tool calls only; no freeform “paste code in chat” workflow. +- Governance: run governance checks on every incremental commit. +- Logging policy: single eval-grade structured logging strategy (no minimal strategy); logs must capture deterministic replay metadata, validation gates, and Step 16 evidence bindings. + +--- + +## 2. Architecture + +### 2.1 Three-Level Fractal Model + +#### Level 1: Macro Loop (Orchestrator) +```mermaid +graph TD + subgraph "L1: Milestone Lifecycle" + Orch[Orchestrator] -- "16a Spawn" --> Planner + Orch -- "16b Spawn" --> Builder + Orch -- "16c Spawn" --> Verifier + + Verifier -- "Findings and Verdict" --> Orch + Orch -- "Remediation Spawn (16b)" --> Builder + Orch -- "Close Milestone" --> Done + end +``` + +#### Level 2: Persona Loop (Example: Trinity Builder 16B) +```mermaid +graph TD + subgraph "L2: Checklist Work Unit" + Persona[Active Persona] -- "1 Execute Unit" --> Worker + Worker -- "Unit Result" --> Persona + + Persona -- "2 Collective Audit" --> CollectiveAudit[Collective Auditor] + CollectiveAudit -- "Findings" --> Persona + + Persona -- "3 Derive Remediation Findings" --> RemediationQueue[Remediation Scope Queue] + RemediationQueue -- "Next Unit" --> Persona + end +``` + +#### Level 3: Atomic Loop (Worker) +```mermaid +graph TD + subgraph "L3: Atomic Action" + Input --> Exec[Active Worker] + + Exec -- "1 Draft" --> Worker + Worker -- "Write" --> DraftArtifact[Draft Artifact] + + Exec -- "2 Audit" --> Auditor + Auditor -- "Read" --> DraftArtifact + Auditor -- "Findings" --> Report[Audit Findings] + + Exec -- "3 Refine" --> Logic{Pass?} + Logic -- "No" --> Worker + Logic -- "Yes" --> Publisher[Publish & Signal Parent] + end +``` + +### 2.2 Persona and Prompt Map +Core Step personas map to existing versioned Step 16 prompts. Utility personas are guided by a dedicated Trinity utility prompt library. + +| Persona | Responsibility | Prompt Source | +| :--- | :--- | :--- | +| Orchestrator | Anchor lifecycle and state transitions | `devspec_toolkit/prompts/prompt_16_impl_context.md` + harness transition rules | +| Trinity Planner 16A | Produce/update milestone plan sections | `devspec_toolkit/prompts/prompt_16a_impl_planner.md` | +| Trinity Builder 16B | Execute checklist item implementations | `devspec_toolkit/prompts/prompt_16b_impl_coder.md` | +| Trinity Verifier 16C | Review implementation and evidence; emit findings/verdict | `devspec_toolkit/prompts/prompt_16c_impl_reviewer.md` | +| Auditor | Generic critique pass for L3 loops | `devspec_toolkit/prompts/trinity/99_auditor.md` | +| Summarizer | Extraction-only evidence snippet support for long outputs | `devspec_toolkit/prompts/trinity/90_summarizer.md` | +| ToolUser | Deterministic tool-call behavior | `devspec_toolkit/prompts/trinity/80_tool_usage.md` | +| Researcher | Optional bounded context discovery | `devspec_toolkit/prompts/trinity/70_researcher.md` | + +### 2.3 Utility Prompt Library (Normative Contract) +Utility prompt library location: +- `devspec_toolkit/prompts/trinity/` + +Required prompt files: +1. `70_researcher.md` +2. `80_tool_usage.md` +3. `90_summarizer.md` +4. `99_auditor.md` + +Guardrails: +- Each file must include a version header (e.g., `Version: 1.2`) and a change log section listing dated changes. +- Each file should define explicit input contract, output schema, stop conditions, and prohibited behavior. +- Utility prompts are part of the normative Trinity contract for utility-persona execution. +- If a state invokes a utility persona and the corresponding prompt file is unavailable, that branch must fail fast as blocked. +- Utility prompts must enforce assumption-free behavior: unresolved context must be surfaced as explicit questions or blocked findings, never guessed. + +Utility Fast-Path: +- For utility invocations where the input is a single artifact and the expected output is a single structured response, the runtime may use an inline utility protocol that skips file-based `task_input.json`/`task_result.json` IO. The inline result must still be schema-validated and logged as a session event. The full spawn protocol remains required for multi-artifact or multi-turn utility sessions. + +--- + +## 3. Artifact Model and Lifecycle + +### 3.1 Authoritative Artifacts +Trinity uses two Step 16 artifacts with different responsibilities: +1. Milestone state machine (authoritative execution file): `spec/impl_context/{step_id}.json`. +2. Anchor roll-up (derived union/summary): `spec/16_impl_context.json`. + +### 3.2 Milestone vs Anchor Rules +- The per-milestone file is the source of truth for Plan/Execute/Review lifecycle. +- The anchor is derived from active milestone context files and is never the canonical source for per-milestone execution details. +- Trinity Planner 16A updates the milestone file for the active `step_id`; it does not treat the anchor as the primary editable state. + +### 3.3 Anchor Regeneration Policy +- Regenerate `spec/16_impl_context.json` at milestone start. +- Regenerate again after each successful L1 state transition (`16a`, `16b`, `16c`). +- Context pack must be regenerated alongside the anchor at every state transition. +- On resume, if the anchor timestamp is older than the most recent milestone context timestamp, regenerate the anchor before re-entering the active state. +- Preserve `extensions` and explicitly manual notes across regeneration. +- Only update fields required by the anchor prompt and schema contract. +- Anchor regeneration must not independently mutate roadmap/progress completion status; milestone closure state is owned by Verifier verdict + Orchestrator transition logic. + +--- + +## 4. Context Governance (Seed-Manifest Authority) + +### 4.1 Spec Authority Set +Trinity must resolve context from governed seeds, not from hard-coded assumptions. + +Required authority set: +1. `spec/common/seed_manifest.json`. +2. `step_requirements["16a"|"16b"|"16c"]` from the seed manifest. +3. Core Step artifacts (as required by seed/state), commonly including: + - `spec/04_fr_list.json` + - `spec/05_interface_contracts.json` + - `spec/06_invariants.json` + - `spec/07_nfrs.json` + - `spec/08_fixtures.json` + - `spec/09_impl_plan.json` + - `spec/10_governance.json` + - `spec/11_redteam.json` + - `spec/12_ci_gates.json` + - `spec/13_extension_manifest.json` + - `spec/13a_completeness_assessment.json` + - `spec/14_roadmap.json` + - `spec/15_scaffold.json` (when relevant) + +No Trinity rule may bypass seed-manifest governance. + +### 4.2 Deterministic Context Resolver +`ContextResolver(step_id, state)` must return: +1. Ordered seed files to load (exact seed-manifest order). +2. Allowed read paths. +3. Allowed write paths from `target_file_patterns` and docs policy. +4. Required spec artifacts for `spec_ref` traceability. + +### 4.3 Spec Reference Resolver +`SpecRefResolver(spec_type, id)` must resolve deterministic provenance: +- `path` +- `line_range` (`Lx-Ly`) +- `commit_hash` (40-char SHA) + +The harness must never emit `spec_ref` records without these fields. +Grounding checks are mandatory: +- `spec_ref.id` must exist in authority artifacts resolved by `ContextResolver`. +- `spec_ref.commit_hash` must exist in git history. +- `spec_ref.line_range` must map to the referenced file content at that commit. +- Staleness check: if the content at `line_range` differs between `commit_hash` and HEAD, emit a `drift` warning in the session log and in `plan.drift` if applicable. + +### 4.5 Context Pack Budget +Context pack total token estimate must not exceed `limits.hard_token_limit`. Budget enforcement rules: +- If estimated tokens for `context_pack.json` contents exceed `limits.soft_token_limit`, lower-priority context items (by reverse `global_seed_order`) are truncated with a `context_budget_truncation` event in the session log. +- The token estimate uses a configurable approximation ratio (`runtime.token_estimation_ratio`, default: 4 chars per token). +- Truncated items retain their pointer/reference but have `content: null` with `truncation_reason: "budget"` in the context pack. + +### 4.4 Need-to-Know Context Handover +Parent-child communication is strictly artifact-based. + +Parent -> Child input contract (`.trinity/runtime/spawns//task_input.json`): +- `protocol_version` +- `child_id` +- `parent_id` +- `role` (mapped to Persona) +- `phase` (mapped to State ID) +- `step_id` +- `task_description` +- `expected_output_schema` +- `context_pack_ref` +- `target_files` (paths only; child loads file content as needed) +- `spec_refs` (IDs/pointers; child resolves through governed artifacts) +- `role_metadata` (prompt role metadata for the active state) + +Child -> Parent output contract (`.trinity/runtime/spawns//task_result.json`): +- `protocol_version` +- `child_id` +- `role` +- `phase` +- `step_id` +- `status` +- `summary` +- `artifacts` +- optional structured findings/questions + +Rules: +- Parent never consumes child raw conversation transcript. +- Child starts with no inherited chat history and relies on explicit artifacts + governed context. +- Handover should pass pointers first, not bulk content, unless explicitly required by phase validation. + +### 4.5 Seed Mutation Ownership +- Seed/manifest mutation is owned by Orchestrator + Trinity Planner 16A states only. +- Trinity Builder 16B and Verifier 16C must not mutate `spec/common/seed_manifest.json` or seed-required authority sets. +- If Builder/Verifier are blocked by missing seed context, they emit explicit ambiguity/findings and return control to Orchestrator. +- Orchestrator may trigger a controlled re-plan cycle (`16a`) after a new spec baseline commit. + +--- + +## 5. Artifact Exchange Contract (Disk-First Two-Phase IO) +Trinity follows a disk-first two-phase contract: + +1. Phase A: questions only. + - Persona emits only clarifying questions when blocked. + - Harness pauses for user input; no guessing. + +2. Phase B: artifact only. + - Persona writes/updates the artifact file at the expected `spec/...` path. + - Persona returns a concise status message with artifact path and validation outcome. + - Harness immediately runs schema + deep validation gates. + +This is the only documented artifact exchange mechanism. + +### 5.1 Prompt Contract Conformance Rules +- All Step 16 prompts used by Trinity (`prompt_16_impl_context.md`, `prompt_16a_impl_planner.md`, `prompt_16b_impl_coder.md`, `prompt_16c_impl_reviewer.md`) must conform to Section 5 disk-first IO behavior. +- Prompt examples must be schema-valid JSON artifacts (no inline JSON comments, no schema-invalid placeholder fields). +- Example artifacts should be sourced from validated fixture artifacts to prevent prompt/schema drift. + +--- + +## 6. Trinity Loop Protocol (Checklist-First State Machine) + +Machine-checkable lifecycle reference: +- `docs/designs/trinity_state_machine.json` is the normative state/transition artifact for L1 orchestration behavior. + +### 6.1 Level 1: Macro Loop (Orchestrator) +Definition: milestone lifecycle over `spec/impl_context/{step_id}.json`. + +1. Spawn Planner (`16a`) and validate plan. +2. Spawn Builder (`16b`) and execute checklist implementations. +3. Spawn Verifier (`16c`) and audit evidence + outcomes. +4. If verifier returns blocking findings, Orchestrator routes to Planner for re-plan before any further Builder run. +5. If verified, sync roadmap/progress artifacts and close the milestone run. + +### 6.2 Level 2: Persona Loop +Definition: persona-specific work unit loop. + +Common structure: +`Execute -> Collective Audit -> Derive Remediation Units -> Repeat` + +#### 6.2.1 Trinity Planner 16A +- Work unit: planning sections inside `spec/impl_context/{step_id}.json`. +- Produces checklist-first plan in schema-grounded fields. +- Validates traceability against seeded artifacts. + +#### 6.2.2 Trinity Builder 16B +- Work unit: checklist items from `plan.spec_alignment.checklist[]`. +- Iterates checklist units and runs L3 loops per unit. +- Updates execution/implementation state for existing checklist IDs only. +- Must not create/reorder checklist items; remediation item creation is owned by Planner/Verifier handoff. +- Proposed additions: when Builder encounters a requirement gap that is clearly in-scope but absent from the plan, it may emit a `proposed_additions[]` array in `execution.emergent_ambiguities` with severity `informational`. These are not published to the checklist — they are routed to Planner via the Remediation path. + +#### 6.2.3 Trinity Verifier 16C +- Work unit: review sections in milestone context. +- Runs verification actions and evaluates evidence bindings, docs gates, and CI gates. +- Emits `review.findings[]` and final `review.verdict`. + +### 6.3 Level 3: Atomic Loop (Worker Lifecycle) +Definition: one atomic checklist implementation or verification action. + +Normative loop rule: +- L3 is a true loop, not a single pass. Each atomic unit repeats `Draft/Execute -> Audit/Review -> Refine` until pass criteria are met or retry caps are exceeded. + +States: +1. Optional Research + - Goal: gather missing bounded context. + - Output: structured research fragment under `extensions`. +2. Draft/Execute + - Goal: satisfy `checklist_item.implementation`. + - Output fragment: `implementation.{status,files_touched,actions}`. +3. Verify + - Goal: run required commands and collect evidence-bound results. + - Output fragment: `execution.execution_results[]`. +4. Audit + - Goal: audit correctness, completeness, drift, tests, docs, and scope. + - Output fragment: `review.{findings,verdict,...}`. +5. Publish + - Parent merges validated fragment into milestone artifact and checkpoints state. + +Audit scope boundary: +- L3 Audit scope is one checklist item. L2 Collective Audit scope is the full state output. When both invoke `99_auditor.md`, they differ in `audit_scope.checklist_ids` — L3 uses a single ID; L2 uses all IDs touched in the current state. + +### 6.4 Checklist-First Semantics +- Trinity must not invent or depend on a `plan.tasks` field. +- Work queue is a derived view of checklist state. +- Ordering is inferred from: + - `plan.solution.sequence_of_concerns` + - file overlap/conflict heuristics +- No new untyped task array is written into Step 16 artifacts. + +### 6.5 Retry Caps and Stop Conditions +Retry caps are configured in `.trinity/trinity.yaml` under `runtime.retry_caps`. + +Default caps: +- `16a` planner retries: 10. +- `16b` builder retries: 10. +- `16c` verifier retries: 10. +- Global milestone loop cap (`runtime.retry_caps.milestone`): 10. + +Cap interaction semantics: +- The milestone retry cap counts L1 loop iterations (each being one full `16a→16b→16c` pass or remediation cycle). +- Per-state caps count retries *within* a single L1 iteration. +- Thus a milestone cap of 10 allows up to 10 full cycles, each cycle allowing up to 10 retries per state. + +On blocker conditions (missing seed, out-of-scope write, failed tests after max retries): +- Stop the active branch immediately and mark milestone deferred/blocked with explicit findings and ambiguities. +- Route remediation through Planner-first re-plan before resuming Builder/Verifier. + +### 6.6 Verdict Contract +- `review.verdict` values are restricted to Step 16 enum values only: `verified | deferred | rejected`. +- Trinity orchestration must not branch on any non-schema verdict value. + +### 6.7 Planner Scope Budgets +Configurable scope rails prevent over-scoping: +- `runtime.scope_budget.max_checklist_items` (default: 30). +- `runtime.scope_budget.max_target_files` (default: 50). +- `runtime.scope_budget.max_seed_additions` (default: 5). + +If Planner exceeds any budget, the state gate emits a `scope_budget_exceeded` finding and blocks until overridden via user input or replanned within budget. + +--- + +## 7. Tool Protocol and Scope Enforcement +Trinity harness must expose strict typed tool calls with deterministic contracts. + +### 7.1 Required Tool Capabilities +1. `read_file(path, start_line?, end_line?)` +2. `write_file(path, content)` (atomic) +3. `edit_file(...)` and/or deterministic `apply_patch(...)` +4. `list_dir(path)` and `glob_match(path, patterns[])` +5. `search_text(pattern, paths[])` with line numbers +6. Git utilities: `git_head`, `git_show`, `git_diff` +7. `exec_cmd(command, mode)` with structured result metadata +8. Validation gate: `specdev validate` / `validate_json` +9. Checkpoint utilities: deterministic git checkpoint operations for branch creation/switch and commit checkpoints required by Section 10. + +Wire-contract requirement: +- Tool invocations and results must be representable as typed envelopes validated against Trinity tool protocol schemas (request/result), not ad-hoc freeform structures. +- `tool_name` must deterministically select a typed payload contract for request `args` and result `result`. +- Unknown tool names, unknown payload fields, or missing required tool payload fields are schema-level failures. + +### 7.2 Write-Path Guardrails +- Allowlist writes from `plan.summary.target_file_patterns` + docs policy. +- Out-of-scope writes are blocked at the tool layer. +- Tooling enforces path checks before applying file changes. + +### 7.3 Command Capture Contract +For every command invocation, record at minimum: +- command +- exit_code +- duration_ms +- timestamp +- working_dir +- pointer to updated Step 16 artifact + +### 7.4 Execution Strategies +`exec_cmd` supports two strategies: +1. Standard strategy + - For short outputs (discovery and quick checks). + - Returns bounded output directly. +2. Summarized strategy + - For long outputs (tests/builds/lints). + - Uses extraction-oriented summarizer flow and returns concise extracted evidence context. + +Selection rule: +- Use summarized strategy when output length is expected to exceed bounded output thresholds or when evidence extraction is required. + +Deterministic pre-scan: +- For `summarized` strategy, a deterministic pre-scan state runs before LLM summarization. The pre-scan extracts lines matching configured marker patterns (`PASSED`, `FAILED`, `ERROR`, `exit code`). +- If the pre-scan finds definitive pass/fail markers, the LLM Summarizer is skipped and the deterministic result is used. +- LLM Summarizer runs only when markers are ambiguous or missing. + +### 7.5 Tool Argument Safety +- Before executing write operations, the tool layer must verify the target path exists (for edits) or the parent directory exists (for new files). Non-existent path writes must be propagated as `tool_error` with a descriptive message, not silently handled. +- LLM output JSON extraction uses a three-tier parser: (1) direct `json.loads`, (2) fenced code block extraction, (3) brace-balanced extraction with depth limit of 50. If tier 3 is used, the extracted payload must pass schema validation before acceptance. If validation fails, the response is treated as malformed and retried. + +### 7.6 Runtime Protocol Schemas +Runtime protocol artifacts are schema-governed: +- `task_input.json` -> `devspec_toolkit/schema/trinity/task_input.schema.json` +- `context_pack.json` -> `devspec_toolkit/schema/trinity/context_pack.schema.json` +- `task_result.json` -> `devspec_toolkit/schema/trinity/task_result.schema.json` +- `.trinity/runtime/tools/tool_call_request.json` -> `devspec_toolkit/schema/trinity/tool_call_request.schema.json` +- `.trinity/runtime/tools/tool_call_result.json` -> `devspec_toolkit/schema/trinity/tool_call_result.schema.json` +- `.trinity/sessions/*.jsonl` -> `devspec_toolkit/schema/trinity/session_event.schema.json` +- `.trinity/logging/log_capture_policy.json` -> `devspec_toolkit/schema/trinity/log_capture_policy.schema.json` +- `.trinity/runtime/scratchpads/scratchpad_*.json` -> `devspec_toolkit/schema/trinity/scratchpad_state.schema.json` +- `.trinity/runtime/session_state_*.json` -> `devspec_toolkit/schema/trinity/session_state.schema.json` +- `.trinity/runtime/spawn_log.json` -> `devspec_toolkit/schema/trinity/spawn_log.schema.json` + +Validation rules: +- Harness validates each runtime protocol file before child spawn and before parent ingestion. +- Schema violations are blocking errors and must stop affected branches. +- Deep validation is also required for relational constraints that schemas cannot fully express (seed order consistency, allowlist/write-scope checks, task-input/context-pack consistency checks, required_spec_refs git grounding checks, and phase-gate completeness checks for successful `16a/16b/16c` outputs). +- Session log deep validation must enforce transaction-boundary closure for child handoffs in canonical order: canonical `SPAWN`, pass `VALIDATION` for task input, pass `VALIDATION` for task result, then canonical `TERMINATE` for the same child span. + +### 7.6 Prompt-Side Tool Schema Budget Policy (Normative) +Goal: +- Prevent token-window inflation from repeatedly injecting full per-tool schemas while preserving strict deterministic contracts. + +Required behavior: +1. Catalog-first prompt contract: + - Default prompt context includes only compact tool catalog entries (`tool_name`, short description, required argument keys, critical constraints). + - Full JSON schemas are not injected by default. +2. On-demand full schema expansion: + - Full tool schema is injected only for tool(s) selected for planning/execution in the current action scope. + - Expansion must be minimal and limited to those tool names. +3. Schema identity references: + - Prompt context should prefer schema references (`schema_uri`, `schema_version`, `schema_sha256`) over repeated inline schema bodies when unchanged. + - Runtime metadata should preserve these references for reproducibility and audit. +4. Hard runtime enforcement remains authoritative: + - Prompt-side compaction is an optimization only. + - Final correctness is always decided by runtime schema + deep validation gates. +5. Fail-safe escalation: + - If compact catalog is insufficient to disambiguate an argument/result shape, the role must request on-demand full schema instead of guessing. +6. Machine-enforced observability: + - `TOOL_CALL` and `TOOL_RESULT` session events must include `metadata.tool_schema_context` (`strategy`, `request_schema_uri`, `request_schema_sha256`, `result_schema_uri`, `result_schema_sha256`, and `expanded_tool_names`; plus `catalog_ref`/`catalog_sha256` when catalog strategies are used). + - Runtime deep validation must enforce context consistency (for example, on-demand strategy must name expanded tools rather than leaving expansion implicit). + +--- + +## 8. Evidence Binding, Logging, and Secret Safety + +### 8.1 Step 16 Evidence Requirements +For passed execution results: +- `execution.execution_results[].evidence` contains a verbatim excerpt with pass markers. +- `execution.execution_results[].evidence_binding.sha256` is the SHA-256 hash of the exact `evidence` string. +- `execution.execution_results[].evidence_ref` format: `sha256:`. + +Summaries are UI-only. Evidence fields in Step 16 artifacts are never paraphrased. + +### 8.2 Long Output Strategy +For long command outputs in unattended runs: +- Use extraction-oriented summarizer behavior to select concise verbatim lines containing pass/fail markers and minimal surrounding context. +- If no compliant evidence excerpt can be produced, treat the checklist item as blocked. + +### 8.3 Logging Strategy and Event Contract +Trinity uses one logging strategy: + +1. Eval strategy (default and only strategy) + - Persist structured event stream with deterministic replay fields. + - MVP baseline: apply deterministic redaction profile metadata per event and enforce command allow/deny policy for common secret-dumping patterns. + - Post-MVP hardening target: enforce pre-persist secret scanning/redaction for all persisted prompt/response artifacts so raw secrets are never written to disk. + - Persist Step 16 evidence excerpts and hash bindings exactly as produced. + - Persist validation gate outcomes and lineage keys (`tool_call_id`, `result_id`, `artifact_ref`, `artifact_sha256`, `diff_ref`). + - Apply capture policy tuning for prompt/response full-capture sampling via policy file. + - Operate in logging-first strategy by default: session/eval artifacts are persisted locally and in CI artifacts, with no external eval backend required. + - External export/publish is optional and must not be required for core Trinity execution correctness. + +Session log contract: +- Format: JSONL. +- Path: `.trinity/sessions/_.jsonl`. +- Aggregation: one root session file containing child events linked by IDs. +- Event types: `SPAWN`, `MESSAGE`, `TOOL_CALL`, `TOOL_RESULT`, `VALIDATION`, `TERMINATE`, `ERROR`. +- Log schema: `devspec_toolkit/schema/trinity/session_event.schema.json`. +- Every event must validate against the log schema before persistence. + +Event schema: +```json +{ + "schema_version": "trinity-session-log-v1", + "timestamp": "ISO-8601", + "event_type": "SPAWN | MESSAGE | TOOL_CALL | TOOL_RESULT | VALIDATION | TERMINATE | ERROR", + "event_id": "uuid", + "event_sequence": 1, + "prev_event_sha256": "previous_event_hash_or_null", + "event_sha256": "current_event_hash", + "run_id": "root_run_id", + "phase_id": "state_identifier", + "loop_id": "loop_identifier", + "agent_id": "unique_session_id", + "parent_id": "calling_agent_id", + "role": "Orchestrator | Planner | Builder | Verifier | Worker | Researcher | Auditor | Summarizer | ToolUser", + "step_id": "current_step_id", + "tool_call_id": "stable_tool_call_id_or_null", + "result_id": "stable_result_id_or_null", + "artifact_ref": "path_or_uri_or_null", + "artifact_sha256": "sha256_or_null", + "diff_ref": "git_diff_ref_or_null", + "model": "gpt-5-or-local", + "content": { + "summary": "...", + "capture_level": "none|summary|full", + "capture_decision_reason": "policy:default|policy:always_full|policy:sampled|policy:capped", + "prompt_artifact_ref": "path_or_null", + "prompt_sha256": "sha256_or_null", + "response_artifact_ref": "path_or_null", + "response_sha256": "sha256_or_null", + "task_input_artifact_ref": "path_or_null", + "task_result_artifact_ref": "path_or_null", + "tool_call": { "name": "...", "args": {} }, + "tool_result": { + "command": "...", + "exit_code": 0, + "duration_ms": 1200, + "working_dir": "...", + "stdout_excerpt": "...", + "stderr_excerpt": "...", + "truncated": false + }, + "validation": { + "schema": "pass|fail|n/a", + "deep_validator": "pass|fail|n/a", + "governance": "pass|fail|n/a", + "seed_lint": "pass|fail|n/a", + "docs_lint": "pass|fail|n/a" + } + }, + "metadata": { + "toolkit_version": "...", + "schema_version": "...", + "git_head": "...", + "prompt_template_id": "prompt_16b_impl_coder", + "prompt_template_sha256": "sha256_of_prompt_template", + "redaction_profile": "eval", + "redaction_applied": false, + "capture_policy_ref": "path_or_null", + "capture_policy_sha256": "sha256_or_null", + "redaction_stats": { + "total_replacements": 0, + "by_class": { "api_key": 0, "token": 0 }, + "classes_detected": [], + "detectors_used": ["secret_scanner_v1"], + "min_confidence": 0.0, + "max_confidence": 0.0 + }, + "decoding": { "temperature": 0.2, "top_p": 0.9, "max_tokens": 4096 }, + "token_usage": { "prompt": 0, "completion": 0, "total": 0 } + } +} +``` + +Traceability rules: +- `SPAWN` records child intent and `child_id`. +- `TERMINATE` records status summary and final artifact pointers for the child scope. +- `parent_id` and `agent_id` reconstruct the full call tree deterministically. +- `TOOL_CALL` and `TOOL_RESULT` are joined by `tool_call_id`. +- Artifact lineage and replay use `artifact_ref` + `artifact_sha256` + `diff_ref`. +- Deterministic replay ordering and tamper detection use `event_sequence` + `prev_event_sha256` + `event_sha256`. +- Full-text capture for eval (when enabled) is referenced via `prompt_artifact_ref`/`response_artifact_ref` and corresponding SHA-256 hashes. +- Capture-level decisions are policy-governed and must be explainable via `capture_decision_reason`. + +Training/export compatibility: +- Preserve native event schema as source-of-truth. +- Provide derived export view mapped to OpenAI-style `messages[]` during dataset generation. +- Export rows should validate against `devspec_toolkit/schema/trinity/eval_export_row.schema.json`. + +### 8.4 Sensitive Data Handling for Logs +MVP scope: +- Continue unattended execution when sensitive output is detected, with best-effort redaction metadata and command allow/deny policy for common secret-dumping commands. +- If compliant non-sensitive verbatim evidence (with required pass markers) cannot be produced, mark the unit blocked and escalate for human input. +- Dataset export pipelines must support deterministic redaction profiles before external sharing. +- Step 16 evidence fields remain verbatim excerpts with pass markers and cannot be paraphrased. + +Post-MVP scope: +- Add mandatory pre-persist secret scanning/redaction for persisted prompt/response/session artifacts. +- Add fail-closed behavior for full-capture events that cannot be safely redacted before write. +- Promote "no raw secret persistence on disk" from aspirational constraint to runtime-enforced invariant. + +--- + +## 9. Session and Resume Model + +### 9.1 Single Active Agent + Disk Call Stack +When parent spawns child: +1. Parent writes `session_state_.json` and pending spawn entry. +2. Parent terminates. +3. Child executes from `.trinity/runtime/spawns//task_input.json` and writes `.trinity/runtime/spawns//task_result.json`. +4. Parent resumes, ingests child summary/artifacts, and updates spawn status. + +Contracts: +- `session_state_.json` must validate against `schema/trinity/session_state.schema.json`. + +### 9.2 Loop Detection +Parent tracks repeated spawn intents in `spawn_log`. +- If identical purpose exceeds configured retries, parent aborts current branch with explicit blocked status. + +Contracts: +- `spawn_log.json` must validate against `schema/trinity/spawn_log.schema.json`. + +### 9.3 Concurrency and Integrity +- Artifact writes use atomic writes; file locks are required only when multi-process strategy is enabled. +- Parent merge uses deterministic field-level merge for child fragment ingestion and retry replay. +- Conflict detection guards against stale child artifacts (state drift), not only concurrent sessions. +- Merge strategy must be documented and testable. +- Merge precedence is deterministic: latest valid child artifact for the same state/checklist scope wins; out-of-phase fragments are rejected as stale. +- Crash consistency requirement: spawn IO write, validation result, and session-log event append must be atomic as a transaction boundary per state handoff. + +### 9.4 Context Flush and Recovery +When token or state boundaries are reached: +1. Serialize compressed state to `.trinity/runtime/scratchpads/scratchpad_.json`. +2. Store active variables and next action. +3. Reset messages. +4. Resume by reloading scratchpad + current milestone artifact. + +Scratchpad requirements: +- Scratchpad content is structured and schema-validated (no freeform-only recovery state). +- Scratchpad must include phase, checklist scope, last successful validation gate, and pending next action pointer. +- Scratchpad schema: `devspec_toolkit/schema/trinity/scratchpad_state.schema.json`. +- Optional human-readable scratchpad views are derived artifacts and must not be used as execution source-of-truth. + +Scratchpad lifecycle (S-2): +- A scratchpad file is created at the start of an L2/L3 loop and persisted on every state boundary. +- On successful state completion, the scratchpad is archived to `.trinity/runtime/scratchpads/archive/`. +- On crash recovery, the most recent scratchpad for the active task_id is loaded. +- Stale scratchpads (from completed or abandoned runs) are cleaned up at the start of a new milestone run. + +Session log rotation (S-3): +- When a session log exceeds 5000 events (configurable via `runtime.session_log_compaction_threshold`), the runtime rotates the active log file and creates a compacted summary event at the start of the new file referencing the archived segment. +- Archived segments are preserved for replay/eval but are not loaded during active execution. + +Workspace cleanup (S-4): +- Intermediate draft/audit versions (`artifacts/v_*`, `reports/audit_v_*`) are preserved until milestone completion. +- On milestone closure, only the final published version is promoted; intermediate versions are archived to `.trinity/workspace//archive/`. +- Clean-up is deferred by default; `runtime.workspace_cleanup: eager` archives immediately after each L3 loop completes. + +### 9.5 Workspace Artifact Versioning +For iterative Draft -> Audit -> Refine loops, Trinity maintains versioned workspace artifacts: + +```text +.trinity/ + workspace/ + / + artifacts/ + reports/ + logs/ +``` + +Versioning rules: +- Drafts: `artifacts/v_` +- Audit reports: `reports/audit_v_.md` +- Child spawn IO: `.trinity/runtime/spawns//task_input.json` and `.trinity/runtime/spawns//task_result.json` +- Atomic workspace IO: `.trinity/workspace//task_input.json` and `.trinity/workspace//task_result.json` +- Session logs: `logs/_session.jsonl` + +Publishing rule: +- Only the passing artifact version is promoted to repository paths. + +### 9.6 Agent Interaction and Artifact Ingestion Flow +This section defines produce/pass/ingest behavior for all L1, L2, and L3 interactions. + +### 9.6.1 Canonical Child Invocation Contract +For any child spawn (Planner, Builder, Verifier, Worker, Auditor, Researcher, Summarizer, ToolUser): +1. Parent creates `.trinity/runtime/spawns//task_input.json`. +2. Parent creates `.trinity/runtime/spawns//context_pack.json`. +3. Child executes and writes `.trinity/runtime/spawns//task_result.json`. +4. Parent ingests `task_result.json`, optionally dereferences artifact pointers, then records a `SPAWN` + `TERMINATE` pair in session logs. + +Required `task_input.json` fields: +- `protocol_version` +- `child_id` +- `parent_id` +- `role` +- `phase` (`16a` | `16b` | `16c` | utility) +- `step_id` +- `task_description` +- `expected_output_schema` +- `context_pack_ref` +- `target_files` +- `spec_refs` +- `role_metadata` + +Required `context_pack.json` fields: +- `protocol_version` +- `phase` +- `step_id` +- `seed_manifest_path` +- `seed_files_ordered` (already resolved by `ContextResolver`) +- `required_spec_refs` (resolved path/line/commit via `SpecRefResolver`) +- `artifact_refs` (milestone context path, anchor path, workspace refs as applicable) +- `allowed_read_paths` +- `allowed_write_paths` +- `target_file_patterns` (if applicable) +- `docs_policy` and `test_contract` (if applicable) + +Required `task_result.json` fields: +- `protocol_version` +- `child_id` +- `role` +- `phase` +- `step_id` +- `status` +- `summary` +- `artifacts` +- `findings` for `blocked|failed`, `questions` for `questions` + +Schema gate: +- All runtime artifacts in this contract are validated against the Trinity runtime schemas before use. + +### 9.6.2 L1 Orchestrator Flows +`16a` Planner spawn: +1. Produce: Orchestrator passes roadmap milestone metadata, current milestone file pointer, seed-governed planning context, and spec baseline commit hash context. +2. Child result: disk-updated milestone artifact for `spec/impl_context/{step_id}.json` plan sections. +3. L1 phase-gate audit: Orchestrator runs planning quality gates (schema, deep validator, required traceability/spec refs, docs impact/test contract presence). If gate fails, respawn Planner within retry caps. +4. Ingest: Orchestrator persists validated artifact, updates anchor, and checkpoints commit. + +`16b` Builder spawn: +1. Produce: Orchestrator passes active checklist scope, ordering hints, write allowlist (`target_file_patterns`), and verification contract. +2. Child result: updated implementation/execution sections for checklist items with evidence binding fields. +3. L1 phase-gate audit: Orchestrator validates evidence gates, status transitions, scope adherence, and governance checks. If gate fails, route to Planner-first re-plan within retry caps before any further Builder run. +4. Ingest: Orchestrator merges validated updates and checkpoints commit. + +`16c` Verifier spawn: +1. Produce: Orchestrator passes current implementation state, execution evidence refs, docs impact requirements, and CI/delivery expectations. +2. Child result: review section with findings and verdict. +3. L1 phase-gate audit: Orchestrator validates review artifact integrity (required findings/verdict structure, evidence refs, and gate outcomes). +4. Ingest: Orchestrator branches by verdict: + - `verified`: update roadmap/progress and close milestone. + - `deferred` with blocking findings: route to Planner first. + - direct Builder remediation is not allowed; all remediation re-enters through Planner to prevent scope/seed/spec-ref drift. + - `rejected`: stop milestone by default and require Planner-led re-plan before any further Builder execution. + +### 9.6.3 L2 Persona Flows +Planner L2 (`16a`): +1. Produce: Planner creates L3 atomic tasks for plan authoring, traceability checks, and schema conformance checks. +2. Pass: Each L3 task gets only relevant seed/spec refs and target JSON sections. +3. Review/Refine loop: Planner runs internal review of draft plan fragments and refines until planning gates pass or retry caps are reached. +4. Ingest: Planner merges validated child outputs into a single milestone artifact update on disk. + +Builder L2 (`16b`): +1. Produce: Builder selects next checklist unit from `plan.spec_alignment.checklist[]` and creates L3 task input. +2. Pass: Builder includes checklist implementation text, related spec refs, allowed file patterns, and linked test expectations. +3. Review/Refine loop: Builder iterates draft implementation, audits findings, and remediation until checklist-item pass conditions are satisfied or retry caps are hit. +4. Ingest: Builder merges validated Worker/Auditor outputs and updates implementation/execution status for existing checklist IDs on disk. + +Verifier L2 (`16c`): +1. Produce: Verifier creates L3 tasks for evidence audits, docs gate checks, CI gate checks, and delivery checks. +2. Pass: Verifier includes evidence refs and review requirements from milestone context. +3. Review/Refine loop: Verifier iterates on finding quality and gate completeness until review quality criteria are met or retry caps are reached. +4. Ingest: Verifier consolidates validated findings and outputs one review artifact with deterministic verdict. + +### 9.6.4 L3 Atomic Flows +Worker path: +1. Produce: Parent persona spawns worker via `.trinity/runtime/spawns//task_input.json`, with `task_workspace=.trinity/workspace//`. +2. Pass: Context pack includes minimal spec refs, target files, and allowed write paths; workspace-local task file may be created as a mirror for audit traceability. +3. Loop: Worker executes iterative `Draft -> Review/Audit -> Refine` until pass or retry cap. +4. Ingest: Parent ingests `.trinity/runtime/spawns//task_result.json`, validates changed files vs allowlist, and stores draft artifacts `artifacts/v_*`. + +Auditor path: +1. Produce: Parent spawns auditor with candidate draft pointer and validation checklist. +2. Pass: Context pack includes required constraints for correctness/completeness/drift/tests/docs/scope. +3. Loop: Auditor executes iterative `Draft Findings -> Review Severity/Traceability -> Refine Findings` before publish. +4. Ingest: Parent ingests `.trinity/runtime/spawns//task_result.json` plus audit report `reports/audit_v_*.md`, then decides refine vs publish. + +Publish path: +1. Produce: Parent selects passing version only. +2. Pass: Parent uses deterministic patch/write tools under allowlist constraints. +3. Ingest: Parent updates milestone JSON fragments and emits phase-level artifact upstream. + +### 9.6.5 Utility Sub-Agent Flows +Researcher: +1. Trigger: missing context that cannot be resolved by direct dependency reads. +2. Pass: bounded search scope and expected structured output format. +3. Draft->Review->Refine: Researcher drafts context findings, self-reviews for relevance/completeness/no-hallucination, and refines before publish. +4. Ingest: parent consumes `summary`, `relevant_files`, `relevant_specs`, and `relevant_code_ranges`; parent never consumes researcher chat transcript. + +ToolUser: +1. Trigger: when deterministic tool-call planning is needed for non-trivial edits/exec. +2. Pass: objective, scope constraints, and available tool capability list. +3. Ingest: parent consumes structured tool plan or tool-call sequence only. +4. Loop policy: exempt from Draft->Review->Refine; this is a deterministic tool/terminal helper path. + +Summarizer: +1. Trigger: long command output requiring evidence extraction. +2. Pass: raw command output pointer + extraction constraints (must include pass/fail markers verbatim). +3. Draft->Review->Refine: Summarizer drafts extraction, reviews for verbatim marker compliance, and refines until evidence gates pass. +4. Ingest: parent consumes extracted verbatim snippet and evidence metadata; rejects paraphrased summaries for Step 16 evidence fields. + +Auditor (utility strategy outside persona L3): +1. Trigger: cross-cutting quality audit before final publish. +2. Pass: artifact pointers and policy checklist. +3. Draft->Review->Refine: Auditor drafts findings, reviews severity/traceability, and refines before publish. +4. Ingest: parent consumes structured findings and decides remediation scope. + +### 9.6.6 Blocked and Question Paths +Phase A question path: +1. Child emits questions only. +2. Parent pauses execution and requests user input. +3. Parent resumes by writing updated `.trinity/runtime/spawns//task_input.json` and preserving lineage in the same spawn directory. + +Blocked path: +1. Child returns blocked status with explicit ambiguity/findings. +2. Parent applies retry policy from `runtime.retry_caps` (`16a`, `16b`, `16c`, `milestone`). +3. On cap exceeded, parent marks deferred/blocked state in milestone artifact and stops affected branches. + +### 9.6.7 Context Passing and Ingestion Rules (Normative) +1. Context is always passed as pointers first; bulk content only when required for deterministic validation. +2. Child receives only governed context resolved through `seed_manifest` and step requirements. +3. Parent ingests only: + - child status and structured fields needed for state transitions, + - artifact pointers plus required validation reads, + - evidence refs and hashes needed for Step 16 compliance. +4. Parent never ingests child internal reasoning/chat logs. +5. Any write outside `allowed_write_paths` or `target_file_patterns` is blocked and recorded as a finding. + +### 9.6.8 Draft->Review->Refine Applicability Matrix +Mandatory loop (`Draft -> Review -> Refine`) applies to: +- Planner +- Builder +- Verifier +- Worker +- Auditor (L3 and utility mode) +- Researcher +- Summarizer + +Exempt (deterministic helper paths only): +- ToolUser +- Terminal command runner helper used by `exec_cmd` transport + +Enforcement: +- Harness must enforce loop presence for mandatory roles by requiring an explicit review artifact/checkpoint before allowing publish/ingest. + +--- + +## 10. Branching, Commits, and Governance + +### 10.1 Branching Model +- Start from clean working tree. +- Create `trinity/{step_id}` from `main`. +- Execute milestone run on that branch only. + +### 10.2 Spec Baseline Commit Policy +Before running `16a`, if seed/spec changes are needed: +1. Apply seed/spec changes. +2. Validate. +3. Commit as a dedicated spec baseline commit. +4. Use that commit SHA for `spec_ref.commit_hash` values. + +If seed/spec changes are required mid-run, Builder/Verifier must return blocked ambiguity; Orchestrator re-enters a controlled `16a` planning cycle, creates a new spec baseline commit, and updates downstream `spec_ref` hashes accordingly. + +### 10.3 Incremental Commit Checkpoints +At minimum: +1. Commit after valid `16a` plan artifact. +2. Commit after each verified checklist item. +3. Final closure commit after verified `16c` + roadmap sync. + +### 10.4 Governance Gate +Run governance checks on every incremental commit, not only at the end. + +--- + +## 11. Roadmap Semantics and Milestone Selection + +### 11.1 Roadmap Authority +- Milestone source of truth: `spec/14_roadmap.json`. +- Roadmap dependencies: top-level `dependencies[]` is authoritative for gating. +- `milestones[].source_milestones[]` is provenance mapping, not execution dependency edges. +- `milestones[].tasks[]` is decomposition text, not cross-milestone dependency control. + +### 11.2 Execution Ordering +- Roadmap execution policy is sequential in listed milestone order. +- Since invocation scope is one milestone per run, scheduler behavior is mostly validation guardrails. + +### 11.3 Milestone Targeting +- Primary mode: user provides `step_id` explicitly. +- Fallback auto-pick (only when missing): first non-`done` milestone in listed order whose dependencies are satisfied; otherwise stop. + +--- + +## 12. Configuration and CLI + +### 12.1 Trinity Config +User-authored `.trinity/trinity.yaml` is authoritative input. + +Example: +```yaml +llm: + api_base: "http://localhost:1234/v1" + model: "input-model" + timeout: 300 + api_key_env: "OPENAI_API_KEY" + temperature: 0.2 + top_p: 0.9 + max_tokens: 4096 + +limits: + soft_token_limit: 60000 + hard_token_limit: 80000 + max_loops: 10 + +runtime: + execution_strategy: "llm" # "llm" (default) or "deterministic" + max_child_turns: 12 + child_timeout_seconds: 21600 # 6h default for local LLM latency + child_timeout_by_state: + 16a: 7200 + 16b: 21600 + 16c: 10800 + utility: 3600 + allow_dirty: false + checkpoint_commits: true + allow_bootstrap_authority_fallback: false + allow_anchor_conflicts: false + retry_caps: + planner: 10 + builder: 10 + verifier: 10 + milestone: 10 +``` + +### 12.2 CLI Entry +```bash +specdev trinity --step-id m1-core-foundation +specdev trinity --resume --resume-run-id +``` + +Lifecycle: +1. Load config and validate shape. +2. Validate milestone selection and dependency guardrails. +3. Initialize status dashboard and session logs. +4. Spawn orchestrator and execute the three-level loop. + +Note: +- The command above is the target runtime interface. +- Default runtime strategy executes LLM-driven state handlers against an OpenAI-compatible endpoint defined in `.trinity/trinity.yaml`. +- Deterministic strategy remains available (`runtime.execution_strategy: deterministic` or CLI `--strategy deterministic`) for fixtures/offline verification. +- For local/self-hosted LLMs, tune `runtime.child_timeout_seconds` and `runtime.child_timeout_by_state` to avoid false blocked states on long generations. +- Set timeout to `0` to disable timeout enforcement for a state (use with care). + +Minimum runtime conformance target: +- A one-milestone vertical slice (LLM or deterministic strategy) must support `16a -> validate -> 16b -> validate -> 16c -> validate` with blocking stops on schema/deep/governance failures. +- Conformance mappings from normative rules to schema/validator/tests are tracked in `docs/designs/trinity_conformance_checklist.md`. + +Supporting tooling (available pre-runtime): +- `specdev-tools trinity-export-eval --out ` for dataset row generation. +- `specdev-tools trinity-replay ` for standalone replay/integrity analysis (strict-by-default; use `--allow-warnings` only for local triage). +- `specdev-tools trinity-publish-eval --rows-glob ... --replay-glob ... [--endpoint-env TRINITY_EVAL_EXPORT_ENDPOINT]` for optional CI/eval dashboard bundle export. + +External publish defaults: +- If no publish endpoint/token is configured, export remains local-only and publish is skipped. +- Local-only capture (`.trinity/sessions/*.jsonl`, `.trinity/eval/*`, CI uploaded artifacts) is the baseline archival contract for future fine-tuning/eval ingestion. +- Strict publish gating is opt-in (`require_eval_publish=true`) and should only be used when an external sink is intentionally provisioned. + +Capture policy tuning for 60k-80k token windows: +- `context_window_token_target` and `max_full_capture_context_fraction` define a derived full-capture budget. +- `full_capture_token_budget_per_run` provides an explicit hard cap. +- `max_full_prompt_tokens_per_event` and `max_full_completion_tokens_per_event` guard per-event spikes. +- When caps are exceeded, runtime validation expects fallback behavior via `oversize_fallback` (for example `summary`). + +--- + +## 13. Terminal Reporting (Concise Live Dashboard) +A three-panel terminal dashboard is recommended: + +1. Header panel + - Trinity version, active `step_id`, progress counters. +2. Live event stream + - High-level events only (`SPAWN`, `WRITE`, `VALIDATE`, `VERIFY`, `ERROR`). +3. Footer metrics + - Current phase, elapsed time, token usage, estimated cost. + +Dashboard must not stream raw model token output by default. + +--- + +Out-of-scope for this document: +- Product-specific implementation logic executed by Trinity against arbitrary repositories. +- Implementing schema/test/runtime changes in tooling modules. +- Post-MVP secret-hardening implementation details (for example, pre-persist full-capture secret scanning/redaction internals). + +--- + +## 14. Milestone Closure and Escalation + +### 14.1 Milestone Closure Contract (G-1) +On `review.verdict == "verified"`, the orchestrator must: +1. Sync roadmap progress: set `milestones[step_id].status = "done"` in `spec/14_roadmap.json`. +2. Archive workspace: move `.trinity/workspace//` intermediate artifacts per Section 9.5 cleanup rules. +3. Final checkpoint commit with message format: `trinity: close [verified]`. +4. Close session log with a `TERMINATE` event containing `closure_reason: "verified"`. +5. Emit terminal dashboard summary. + +On `review.verdict == "deferred"` or `"rejected"`: +1. Do NOT update roadmap progress. +2. Record findings snapshot in `.trinity/runtime/blocked//`. +3. If within milestone retry cap, route through Planner-first remediation. +4. If cap exceeded, emit `BLOCKED` terminal event and stop. + +### 14.2 Human Escalation Protocol (G-2) +When Trinity encounters a situation it cannot resolve autonomously: +1. Write an escalation artifact to `.trinity/runtime/escalations/_.json` containing: + - `reason` (enum: `ambiguity`, `scope_budget_exceeded`, `security_concern`, `dependency_missing`, `retry_cap_exceeded`) + - `context` (affected checklist IDs, phase, current retry count) + - `suggested_actions` (proposed resolution paths) + - `blocking` (boolean) +2. Transition state machine to `QUESTIONS_PENDING`. +3. Display escalation summary on terminal dashboard. +4. On user response, inject answers into the originating state context and resume. + +Escalation triggers: +- State A questions from any child persona. +- Scope budget exceeded (Section 6.7). +- Retry cap exceeded with no progress. +- Out-of-scope write attempt that cannot be resolved by narrowing. +- Security-sensitive operations detected (command denylist hit). + +### 14.3 Trinity vs Manual Strategy Divergence (A-2) +In Trinity harness strategy: +- Roadmap/progress sync is automatic on verified verdict. +- Checkpoint commits are automatic. +- Session logging is always active. +- State transitions follow the state machine strictly. + +In manual strategy (user drives agents directly): +- Roadmap/progress updates must be explicitly requested by the user. +- Checkpoint commits are user-initiated. +- Session logging is opt-in. +- State sequencing is advisory; user may invoke states in any order. + +Both strategies: +- Schema validation of Step 16 artifacts is always enforced. +- Prompt contracts remain authoritative. +- Seed governance rules apply. + +--- + +## 15. Cross-Milestone Knowledge Base (AR-4) +Trinity may optionally maintain a cross-milestone knowledge base at `.trinity/knowledge/`: +- `patterns.jsonl`: reusable implementation patterns observed across milestones. +- `decisions.jsonl`: architecture decisions with rationale and evidence refs. +- `issues.jsonl`: recurring issues and their resolutions. + +Knowledge base rules: +- Entries are append-only within a project lifecycle. +- The Planner may read knowledge base entries to inform planning. +- No knowledge base entry may override spec authority. +- Knowledge base is informational context, not normative input. + +--- + +## 16. Dashboard Interaction Capabilities (AR-5) +The terminal dashboard (Section 13) should support these interactive capabilities when running in terminal-attached strategy: +- `[p]ause / [r]esume`: pause autonomous execution at the next state boundary. +- `[s]tatus`: show current state machine state, retry counts, and active child. +- `[e]scalate`: manually trigger escalation to QUESTIONS_PENDING. +- `[q]uit`: graceful shutdown with state persistence. + +In non-interactive strategy (CI/unattended), these are no-ops. + +--- + +## 17. Structured Logging Contracts (L-1 through L-5) + +### 17.1 Outcome Labels (L-1) +Every session event should include an `outcome_label` field in its metadata: +- `success`: the action achieved its intended result. +- `partial`: the action partially succeeded (some items passed, some failed). +- `failure`: the action failed to achieve its intended result. +- `skip`: the action was skipped due to precondition. +- `retry`: the action is being retried. + +These labels enable downstream dataset curation for fine-tuning. + +### 17.2 Interaction Correctness Tags (L-2) +Tool call events should include a `correctness_tag` in metadata: +- `correct`: tool arguments and result were valid. +- `self_corrected`: initial attempt failed but was retried successfully. +- `failed`: tool call resulted in an error that was not recovered. + +These tags are assigned by the runtime after observing tool call outcomes. + +### 17.3 Custom Redaction Patterns (L-3) +Beyond the built-in secret scanners, users may configure domain-specific redaction rules in `.trinity/trinity.yaml`: +```yaml +logging: + custom_redaction_patterns: + - name: "internal_api_keys" + pattern: "INTERNAL-[A-Z0-9]{32}" + replacement: "[REDACTED:internal_api_key]" + - name: "customer_ids" + pattern: "CUST-[0-9]{8}" + replacement: "[REDACTED:customer_id]" +``` +Custom patterns are applied in order after built-in patterns. + +### 17.4 Quality Metrics (L-4) +On session close, the runtime should compute and persist aggregate quality metrics: +- `total_events`: count of all session events. +- `total_retries`: count of retry events. +- `evidence_binding_rate`: fraction of checklist items with valid evidence. +- `drift_warning_count`: count of spec_ref staleness warnings. +- `mean_state_duration_ms`: average wall-clock time per state. +- `scope_violations`: count of out-of-scope write attempts. + +Metrics are appended to the session close event. + +### 17.5 Multi-Format Export Adapters (L-5) +Session logs can be exported via adapter pipelines: +- `jsonl` (default): raw event stream. +- `eval-rows`: one row per tool interaction, for fine-tuning dataset curation (via `trinity-export-eval`). +- `summary`: human-readable markdown summary of the session. + +Additional adapters (e.g., OpenTelemetry spans, external eval dashboards) are extension points documented here for future implementation. + +--- + +## 18. Tool Capability Gaps (T-1 through T-5) +The following tool capabilities are not yet implemented but are identified as future extension points: + +| ID | Gap | Impact | Extension Point | +|----|-----|--------|----------------| +| T-1 | Tree-sitter symbol navigation | No structured code navigation; agents rely on text search | Add `code_symbols(path)` tool returning symbol table | +| T-2 | Semantic diff | No semantic understanding of code changes; diffs are textual | Add `semantic_diff(base, head)` tool returning AST-level changes | +| T-3 | Dependency graph resolution | No programmatic access to dependency graphs | Add `dep_graph(path)` tool for import/package dependency trees | +| T-4 | Test isolation runner | No isolated single-test execution; relies on full test suite | Add `run_test(test_id)` tool with isolated env + structured result | +| T-5 | Artifact pinning and comparison | No structured comparison of artifact versions | Add `compare_artifacts(v1, v2)` tool for field-level diff | + +These gaps do not block current Trinity execution but limit efficiency. Implementations should conform to the typed tool protocol (Section 7). diff --git a/docs/designs/trinity_state_machine.json b/docs/designs/trinity_state_machine.json new file mode 100644 index 00000000..2aa8be0e --- /dev/null +++ b/docs/designs/trinity_state_machine.json @@ -0,0 +1,301 @@ +{ + "version": "1.1.0", + "id": "trinity-l1-state-machine", + "description": "Normative L1 orchestrator lifecycle for one milestone run.", + "states": [ + { + "id": "INIT", + "terminal": false, + "description": "Config and milestone selection complete; no child has started." + }, + { + "id": "PLAN_16A", + "terminal": false, + "description": "Planner child is active; plan artifact/gates are in progress." + }, + { + "id": "BUILD_16B", + "terminal": false, + "description": "Builder child is active; checklist execution is in progress." + }, + { + "id": "REVIEW_16C", + "terminal": false, + "description": "Verifier child is active; review findings and verdict are in progress." + }, + { + "id": "REMEDIATE_16A", + "terminal": false, + "description": "Planner-first remediation state after non-verified review outcomes." + }, + { + "id": "BLOCKED", + "terminal": true, + "description": "Run is stopped due to blocking contract violations or unresolved ambiguity." + }, + { + "id": "COMPLETED", + "terminal": true, + "description": "Milestone verified and closure sync completed." + }, + { + "id": "QUESTIONS_PENDING", + "terminal": false, + "description": "Autonomous execution paused; waiting for user response to Phase A questions or escalation.", + "resume_key": "originating_phase" + }, + { + "id": "ERROR", + "terminal": true, + "description": "Run stopped due to infrastructure failure (LLM unreachable, disk full, timeout). Distinct from BLOCKED (contract violations)." + } + ], + "phase_gate_requirements": { + "16a": [ + "step16_schema_valid", + "step16_deep_valid", + "checklist_non_empty", + "required_spec_refs_grounded", + "docs_impact_present" + ], + "16b": [ + "step16_schema_valid", + "step16_deep_valid", + "execution_results_present", + "scope_guard_pass", + "evidence_binding_present" + ], + "16c": [ + "step16_schema_valid", + "step16_deep_valid", + "review_section_present", + "review_verdict_enum_valid", + "findings_contract_valid" + ] + }, + "transaction_boundary_contract": { + "description": "Every child handoff must be transaction-closed in session logs.", + "required_sequence": [ + "SPAWN(task_input_artifact_ref canonical)", + "VALIDATION(task_input_artifact_ref schema=pass deep_validator=pass)", + "VALIDATION(task_result_artifact_ref schema=pass deep_validator=pass)", + "TERMINATE(task_result_artifact_ref canonical)" + ], + "pairing_key": "child_id extracted from canonical spawn/task_result refs", + "failure_mode": "branch_blocked" + }, + "transitions": [ + { + "id": "t-init-to-16a", + "from": "INIT", + "to": "PLAN_16A", + "event": "spawn_planner", + "guards": [], + "actions": [ + "emit_task_input", + "emit_context_pack", + "append_spawn_event" + ] + }, + { + "id": "t-16a-pass-to-16b", + "from": "PLAN_16A", + "to": "BUILD_16B", + "event": "phase_gate_pass", + "guards": [ + "phase=16a", + "all phase_gate_requirements.16a pass" + ], + "actions": [ + "ingest_child_result", + "checkpoint_commit", + "regenerate_anchor" + ] + }, + { + "id": "t-16a-fail-to-blocked", + "from": "PLAN_16A", + "to": "BLOCKED", + "event": "phase_gate_fail", + "guards": [ + "phase=16a" + ], + "actions": [ + "record_blocking_finding", + "stop_branch" + ] + }, + { + "id": "t-16b-pass-to-16c", + "from": "BUILD_16B", + "to": "REVIEW_16C", + "event": "phase_gate_pass", + "guards": [ + "phase=16b", + "all phase_gate_requirements.16b pass" + ], + "actions": [ + "ingest_child_result", + "checkpoint_commit", + "regenerate_anchor" + ] + }, + { + "id": "t-16b-fail-to-remediate", + "from": "BUILD_16B", + "to": "REMEDIATE_16A", + "event": "phase_gate_fail", + "guards": [ + "phase=16b" + ], + "actions": [ + "record_findings", + "enqueue_planner_first_remediation" + ] + }, + { + "id": "t-16c-verified-to-completed", + "from": "REVIEW_16C", + "to": "COMPLETED", + "event": "review_verdict", + "guards": [ + "verdict=verified", + "all phase_gate_requirements.16c pass" + ], + "actions": [ + "sync_roadmap_progress", + "checkpoint_commit", + "close_session" + ] + }, + { + "id": "t-16c-nonverified-to-remediate", + "from": "REVIEW_16C", + "to": "REMEDIATE_16A", + "event": "review_verdict", + "guards": [ + "verdict in {deferred,rejected}" + ], + "actions": [ + "record_findings", + "enqueue_planner_first_remediation" + ] + }, + { + "id": "t-remediate-to-16b", + "from": "REMEDIATE_16A", + "to": "BUILD_16B", + "event": "phase_gate_pass", + "guards": [ + "phase=16a", + "all phase_gate_requirements.16a pass" + ], + "actions": [ + "ingest_replan", + "checkpoint_commit" + ] + }, + { + "id": "t-remediate-fail-to-blocked", + "from": "REMEDIATE_16A", + "to": "BLOCKED", + "event": "phase_gate_fail", + "guards": [ + "phase=16a" + ], + "actions": [ + "record_blocking_finding", + "stop_branch" + ] + }, + { + "id": "t-16a-retry", + "from": "PLAN_16A", + "to": "PLAN_16A", + "event": "phase_retry", + "guards": [ + "retry_count < retry_caps.planner" + ], + "actions": [ + "increment_retry_count", + "log_retry_reason" + ] + }, + { + "id": "t-16b-retry", + "from": "BUILD_16B", + "to": "BUILD_16B", + "event": "phase_retry", + "guards": [ + "retry_count < retry_caps.builder" + ], + "actions": [ + "increment_retry_count", + "log_retry_reason" + ] + }, + { + "id": "t-16c-retry", + "from": "REVIEW_16C", + "to": "REVIEW_16C", + "event": "phase_retry", + "guards": [ + "retry_count < retry_caps.verifier" + ], + "actions": [ + "increment_retry_count", + "log_retry_reason" + ] + }, + { + "id": "t-phase-to-questions", + "from": [ + "PLAN_16A", + "BUILD_16B", + "REVIEW_16C" + ], + "to": "QUESTIONS_PENDING", + "event": "questions_emitted", + "guards": [ + "child_result.status == 'questions'" + ], + "actions": [ + "persist_originating_phase", + "write_escalation_artifact", + "pause_autonomous_execution" + ] + }, + { + "id": "t-questions-to-phase", + "from": "QUESTIONS_PENDING", + "to": "$originating_phase", + "event": "user_response_received", + "guards": [ + "originating_phase is valid" + ], + "actions": [ + "inject_user_answers", + "resume_phase_with_context" + ] + }, + { + "id": "t-any-to-error", + "from": [ + "INIT", + "PLAN_16A", + "BUILD_16B", + "REVIEW_16C", + "REMEDIATE_16A", + "QUESTIONS_PENDING" + ], + "to": "ERROR", + "event": "infrastructure_failure", + "guards": [], + "actions": [ + "save_session_state", + "record_error_details", + "stop_branch" + ] + } + ] +} \ No newline at end of file diff --git a/docs/developers/design/migration_system_spec.md b/docs/developers/design/migration_system_spec.md index 27c5a5f1..e0ab60c9 100644 --- a/docs/developers/design/migration_system_spec.md +++ b/docs/developers/design/migration_system_spec.md @@ -597,9 +597,11 @@ Before returning your output, you MUST verify: ## Output Contract -Return exactly one fenced code block with language `json`. -Do NOT include explanatory text outside the code block. -The JSON must be valid and complete. +Use a disk-first contract: +- Write/update the target artifact file at `{{TARGET_FILE}}` on disk. +- Return concise status only (updated path + validation result). +- Do not emit full JSON payloads in chat. +- The artifact content must be valid and complete. ``` ### Full Template Example: `template_prose_to_json.md` @@ -757,17 +759,20 @@ Before returning your output, you MUST verify each item: ## Output Contract -Return exactly one fenced code block with language `json`. +Use a disk-first contract: +- Write/update the target artifact file at `{{TARGET_FILE}}` on disk. +- Return concise status only (updated path + validation result). +- Do not emit full JSON payloads in chat. The JSON must: 1. Be valid, parseable JSON 2. Conform to the target schema -3. Contain NO content outside the code block +3. Be persisted to `{{TARGET_FILE}}` without truncation 4. Include `_migration_notes` object if any data couldn't be mapped ```json { - // Your complete output here + // Reference structure only; write complete output to disk } ``` ``` diff --git a/docs/developers/getting_started.md b/docs/developers/getting_started.md index 4062f03d..6703a326 100644 --- a/docs/developers/getting_started.md +++ b/docs/developers/getting_started.md @@ -5,7 +5,7 @@ This guide onboards developers to the end-to-end spec workflow and shows how to ## Prerequisites - Python 3.10+ for running the CLI and validation commands - (Optional) Node.js for exercising generated scaffolds -- Access to an AI assistant that can emit valid JSON +- Access to an AI assistant that can follow disk-first artifact updates - Familiarity with Git and basic JSON editing ## 0. Initialize the Project @@ -101,9 +101,9 @@ Before writing formal specs, you must define the "Seed" of your project using th 2. Read the prompt to internalise the Definition of Ready and dependencies. 3. Run the matching prompt from [./devspec_toolkit/prompts/prompt_NN_name.md](../../prompts/) using the two‑phase flow: - Phase A — Clarify: the assistant reads the prompt’s “Context To Ingest” and “Operating Flow”, applies the “Self‑Audit Gate”, and outputs only a short bulleted list of targeted questions if critical info is missing. - - Phase B — Emit: after answering questions, rerun to emit exactly one fenced `json` block. -4. Paste the single fenced `json` block into `spec/NN_name.json` in your host repo. -5. Validate the artifact using the [core validation commands](reference.md#core-validation-commands). + - Phase B — Write: after answering questions, rerun and have the assistant write/update `spec/NN_name.json` on disk, then return concise status (artifact path + validation result). +4. Validate the artifact using the [core validation commands](reference.md#core-validation-commands). +5. Confirm traceability and required fields before moving to the next step. 6. Keep traceability up to date; run the same command set after each change with the `--repo-root` flag. ### Phase II · Spec → Implementation (Steps 13–16c) @@ -122,9 +122,9 @@ Keep [reference.md](reference.md) handy for the complete command catalogue, flag - Copy the prompt exactly as stored under [./devspec_toolkit/prompts/](../../prompts/). - Use the two‑phase flow: - Phase A — Clarify: if the prompt’s “Self‑Audit Gate” is not satisfied, the assistant should output only a concise, grouped list of Gap Questions. Answer them. - - Phase B — Emit: the assistant then emits **exactly one** fenced `json` block that validates against the embedded schema. + - Phase B — Write: the assistant writes/updates the target artifact on disk and returns concise status (artifact path + validation result). - Clarify responses: short, bulleted questions grouped by topic; no JSON, no code fences, no speculative answers; prioritize gating items (trace/owners/units/methods/security) and stop after asking until you respond. -- If validation fails, consult the guide, address errors, and re-run the emission. +- If validation fails, consult the guide, address errors, and re-run the write/validate cycle. - Need a quick reminder of the workflow for a given step? Run `./tools/run_specdev.sh ai-help --step NN`. Automation protocol and runner tips live in [../agents/manifest.json](../agents/manifest.json) and [../agents/agents.md](../agents/agents.md). diff --git a/docs/developers/index.md b/docs/developers/index.md index caa26c45..49812e88 100644 --- a/docs/developers/index.md +++ b/docs/developers/index.md @@ -18,6 +18,7 @@ Use this index to locate the developer-facing material while working through the - [`tooling/coverage_matrix.md`](tooling/coverage_matrix.md) — traceability mechanics and enforcement. - [`tooling/gap_hunter_checklist.md`](tooling/gap_hunter_checklist.md) — repeatable gap-hunting process. - [`tools/changelog_parser.md`](tools/changelog_parser.md) — changelog YAML parser for migration system. +- [`tools/trinity_observability.md`](tools/trinity_observability.md) — replay/export/remediation workflows and CI dashboard integration. ## Step Guides Each spec step ships with two authoritative files in `spec/`: diff --git a/docs/developers/reference.md b/docs/developers/reference.md index b52c9809..7b2577e7 100644 --- a/docs/developers/reference.md +++ b/docs/developers/reference.md @@ -54,6 +54,7 @@ python3 devspec_toolkit/scripts/init_project.py --target . --strict ./tools/run_specdev.sh changelog --list --repo-root ./devspec_toolkit ./tools/run_specdev.sh changelog --version 0.1.0 --repo-root ./devspec_toolkit ./tools/run_specdev.sh changelog --validate 0.1.0 --repo-root ./devspec_toolkit +``` ### Alignment & Migration ```bash @@ -73,7 +74,31 @@ specdev align apply --spec-dir spec --auto specdev align prompts --spec-dir spec --output prompts/migration/ --mode upgrade ``` +### Trinity Observability Commands +```bash +# Validate runtime artifacts +./tools/run_specdev.sh validate-runtime .trinity/sessions/.jsonl --type session_event --repo-root ./devspec_toolkit + +# Export eval rows from a session log +./tools/run_specdev.sh trinity-export-eval .trinity/sessions/.jsonl --repo-root ./devspec_toolkit --out .trinity/eval/_rows.jsonl + +# Replay and verify artifact/hash lineage +./tools/run_specdev.sh trinity-replay .trinity/sessions/.jsonl --repo-root ./devspec_toolkit --out .trinity/eval/_replay.json + +# Aggregate dashboard summary from exported rows + replay reports +./tools/run_specdev.sh trinity-dashboard --rows-glob ".trinity/eval/*_rows.jsonl" --replay-glob ".trinity/eval/*_replay.json" --out-json .trinity/eval/dashboard.json --out-md .trinity/eval/dashboard.md +# Generate remediation/resume action plan from replay findings +./tools/run_specdev.sh trinity-remediate .trinity/eval/_replay.json --repo-root ./devspec_toolkit --out .trinity/eval/_remediation.json + +# Strict mode for missing resume source artifacts +./tools/run_specdev.sh trinity-remediate .trinity/eval/_replay.json --repo-root ./devspec_toolkit --session-log .trinity/sessions/.jsonl --emit-session-state .trinity/runtime/session_state_resume.json --emit-task-input .trinity/runtime/task_input_resume.json --missing-resume-source-policy hard --out .trinity/eval/_remediation.json + +# Bundle and publish eval artifacts to external dashboard endpoint (optional) +./tools/run_specdev.sh trinity-publish-eval --rows-glob ".trinity/eval/*_rows.jsonl" --replay-glob ".trinity/eval/*_replay.json" --dashboard-json .trinity/eval/dashboard.json --out .trinity/eval/export_bundle.json --endpoint-env TRINITY_EVAL_EXPORT_ENDPOINT --auth-token-env TRINITY_EVAL_EXPORT_TOKEN +``` + +See [`tools/trinity_observability.md`](tools/trinity_observability.md) for CI wiring and resume-output examples. ### Step-Specific Verification For deep validation of specific steps (DAGs, cycles, logic), use the dedicated scripts: @@ -82,7 +107,6 @@ python devspec_toolkit/tests/integration/test_step_02.py spec/02_system_sketch.j python devspec_toolkit/tests/integration/test_step_12.py tests/fixtures/step_12/valid_dag.json python devspec_toolkit/tests/integration/test_step_15.py tests/fixtures/step_15/valid_full.json ``` -``` For Step 13a, generate `spec/13a_completeness_assessment.json` via `prompts/prompt_13a_completeness_assessment.md` and validate it like any other artifact: ```bash @@ -92,7 +116,7 @@ For Step 13a, generate `spec/13a_completeness_assessment.json` via `prompts/prom Invoke commands from the root of your host repository so relative paths to `spec/` and [./devspec_toolkit/](../../) resolve cleanly. ## Two-Phase AI Runner Mode -- Prompts support a two-phase flow: Clarify (questions only) → Emit (single fenced `json`). +- Prompts support a two-phase flow: Clarify (questions only) → Emit (disk-first artifact write + concise status). - Agents read each prompt’s “Context To Ingest”, follow the “Operating Flow”, apply the “Self‑Audit Gate”, and ask targeted questions if gating items are missing. - Runners should honor the manifest interaction hints: see `docs/agents/manifest.json` (`interaction_mode: two_phase`). - Operational guidance for agents and runner tips: `docs/agents/agents.md`. diff --git a/docs/developers/tools/trinity_observability.md b/docs/developers/tools/trinity_observability.md new file mode 100644 index 00000000..385d7353 --- /dev/null +++ b/docs/developers/tools/trinity_observability.md @@ -0,0 +1,168 @@ +# Trinity Observability Tooling + +This page documents the runtime observability helpers for Trinity session logs. + +## Commands + +### Run runtime orchestration +Execute the full Trinity `16a -> 16b -> 16c` lifecycle for a single roadmap milestone: + +```bash +./tools/run_specdev.sh trinity --step-id --repo-root ./devspec_toolkit +``` + +Use JSON output for machine-readable automation: + +```bash +./tools/run_specdev.sh trinity --step-id --repo-root ./devspec_toolkit --json +``` + +Resume from the latest persisted runtime session state: + +```bash +./tools/run_specdev.sh trinity --resume --repo-root ./devspec_toolkit --json +``` + +Disambiguate resume target when multiple session states exist: + +```bash +./tools/run_specdev.sh trinity --resume --resume-run-id --repo-root ./devspec_toolkit --json +``` + +### Export eval rows +Convert session-event JSONL into normalized eval rows: + +```bash +./tools/run_specdev.sh trinity-export-eval .trinity/sessions/.jsonl --repo-root ./devspec_toolkit --out .trinity/eval/_rows.jsonl +``` + +### Replay verification +Reconstruct timeline and verify artifact/hash lineage: + +```bash +./tools/run_specdev.sh trinity-replay .trinity/sessions/.jsonl --repo-root ./devspec_toolkit --out .trinity/eval/_replay.json +``` + +Use strict mode to fail on warnings: + +```bash +./tools/run_specdev.sh trinity-replay .trinity/sessions/.jsonl --repo-root ./devspec_toolkit --strict +``` + +### Dashboard summary +Aggregate exported rows and replay reports: + +```bash +./tools/run_specdev.sh trinity-dashboard --rows-glob ".trinity/eval/*_rows.jsonl" --replay-glob ".trinity/eval/*_replay.json" --out-json .trinity/eval/dashboard.json --out-md .trinity/eval/dashboard.md +``` + +### Remediation plan +Generate remediation actions from replay findings: + +```bash +./tools/run_specdev.sh trinity-remediate .trinity/eval/_replay.json --repo-root ./devspec_toolkit --out .trinity/eval/_remediation.json +``` + +`trinity-remediate` defaults to `--missing-resume-source-policy hard`, so missing source artifacts fail fast by default. Use `--missing-resume-source-policy soft` only for local triage workflows. + +Optional resume artifact generation: + +```bash +./tools/run_specdev.sh trinity-remediate .trinity/eval/_replay.json \ + --repo-root ./devspec_toolkit \ + --session-log .trinity/sessions/.jsonl \ + --emit-session-state .trinity/runtime/session_state_resume.json \ + --emit-task-input .trinity/runtime/task_input_resume.json \ + --missing-resume-source-policy hard \ + --out .trinity/eval/_remediation.json +``` + +### Eval dashboard export +Bundle eval rows/replay summaries/dashboard and optionally publish to an external HTTP endpoint: + +```bash +./tools/run_specdev.sh trinity-publish-eval \ + --rows-glob ".trinity/eval/*_rows.jsonl" \ + --replay-glob ".trinity/eval/*_replay.json" \ + --dashboard-json ".trinity/eval/dashboard.json" \ + --out ".trinity/eval/export_bundle.json" \ + --endpoint-env TRINITY_EVAL_EXPORT_ENDPOINT \ + --auth-token-env TRINITY_EVAL_EXPORT_TOKEN +``` + +## CI Hook + +`.github/workflows/ci.yml` includes a `trinity-observability` job that: +1. Discovers `.trinity/sessions/*.jsonl`. +2. Runs export + replay + remediation per session log. +3. Builds dashboard summary files under `.trinity/eval/`. +4. Bundles eval dashboard export payloads (`.trinity/eval/export_bundle.json`) and optionally pushes them to `TRINITY_EVAL_EXPORT_ENDPOINT`. +5. Uploads `trinity-observability` artifacts and writes dashboard markdown to the GitHub Actions job summary. + +### Real CI verification run +Use manual dispatch to run a strict verification pass against real logs: +1. Open Actions → `SpecDev CI` → `Run workflow`. +2. Set `require_trinity_logs=true` (job fails if no `.trinity/sessions/*.jsonl` exist). +3. Optional: set `trinity_logs_glob` when logs are in non-standard paths. +4. Optional: set `require_eval_publish=true` to fail if external export is not configured or publish fails. + +To enable external dashboard publishing, configure repository secrets: +- `TRINITY_EVAL_EXPORT_ENDPOINT` +- `TRINITY_EVAL_EXPORT_TOKEN` + +## Capture Policy Tuning (60k–80k Window) +`schema/trinity/log_capture_policy.schema.json` supports token-budget controls: +- `context_window_token_target`: intended context window (for example `80000`). +- `max_full_capture_context_fraction`: max fraction of the window for full-capture events (for example `0.2`). +- `full_capture_token_budget_per_run`: explicit hard budget for total full-capture tokens per run. +- `max_full_prompt_tokens_per_event`: per-event prompt token cap before fallback. +- `max_full_completion_tokens_per_event`: per-event completion token cap before fallback. +- `operating_profile`: profile/tier/budget-tier contract (`eval_default|eval_extended|cost_guarded`). +- `budgets`: normalized budget block used by runtime completeness/fallback checks. +- `retention`: retention windows for `session_log_days`, `capture_artifact_days`, and `eval_export_days`. + +Runtime validation applies these controls and expects fallback capture levels (`oversize_fallback`) when budgets are exceeded. +Session events also capture policy-fallback telemetry in metadata: +- `capture_policy_profile` +- `capture_policy_fallback_applied` +- `capture_policy_fallback_reasons` + +## Child Timeout Tuning (Local LLMs) +Trinity child processes now support YAML-configured timeouts: +- `runtime.child_timeout_seconds`: default timeout applied to all phases. +- `runtime.child_timeout_by_phase`: optional overrides for `16a`, `16b`, `16c`, `utility`. + +Example: + +```yaml +runtime: + child_timeout_seconds: 21600 + child_timeout_by_phase: + 16a: 7200 + 16b: 21600 + 16c: 10800 + utility: 3600 +``` + +Use larger values for local/self-hosted LLM endpoints to prevent false timeout blocks during long completions. +Set a timeout to `0` to disable timeout enforcement for that scope. + +## Utility Schema Validation +Utility orchestration now validates structured utility payloads with: +- `schema/trinity/utility_call.schema.json` +- `schema/trinity/utility_result.schema.json` + +Runtime emits explicit `VALIDATION` events for utility payload schema pass/fail before utility output is ingested by parent phases. + +## Anchor Union Telemetry +Anchor regeneration emits `VALIDATION` events with `metadata.anchor_union_metrics` so replay/eval pipelines can track: +- active context count +- merged checklist size +- checklist conflict count and conflict IDs +- scope/docs/test-command union counts + +## Bootstrap Ref Explainability +Planner bootstrap context packs may include `bootstrap_ref_trace` entries that document: +- which roadmap field produced each bootstrap candidate (`selected_from`) +- whether it came from `structured`, `tokenized`, or `authority_fallback` selection +- the grounded spec path and line range used for the final reference diff --git a/docs/developers/workflows/discovery.md b/docs/developers/workflows/discovery.md index 6e89b404..fc6d2a9a 100644 --- a/docs/developers/workflows/discovery.md +++ b/docs/developers/workflows/discovery.md @@ -35,7 +35,7 @@ Consult the matching `spec/NN_name.guide.md` before running the prompt for each ## AI Assist Flow Prompts are designed for a two‑phase interaction to reduce rework: - Phase A — Clarify: the assistant ingests the step’s context and asks targeted Gap Questions when the “Self‑Audit Gate” is not satisfied. -- Phase B — Emit: once answers are provided, the assistant emits a single fenced `json` block that validates against the schema. +- Phase B — Write: once answers are provided, the assistant writes/updates the target `spec/NN_*.json` artifact on disk and returns concise status (artifact path + validation result). Clarify responses should be short, bulleted questions grouped by topic (no JSON, no code fences), prioritizing gating items; the assistant stops until answers are provided. ## Validation Cadence diff --git a/docs/developers/workflows/spec_to_impl.md b/docs/developers/workflows/spec_to_impl.md index 11218438..1d938ab8 100644 --- a/docs/developers/workflows/spec_to_impl.md +++ b/docs/developers/workflows/spec_to_impl.md @@ -7,7 +7,7 @@ All commands in this guide assume you run them from repo root with the toolkit a ## AI Assist Flow Where prompts apply in this phase (e.g., Scaffold updates, Trinity Plan/Review, Drift), use the two‑phase flow: - Phase A — Clarify: ask targeted questions based on the prompt’s “Self‑Audit Gate”. -- Phase B — Emit: output exactly one fenced `json` block for the step’s artifact. +- Phase B — Emit: write/update the artifact file at the step path and return concise status (path + validation result). Clarify responses should be short, bulleted questions grouped by topic (no JSON, no code fences), prioritizing gating items; pause emission until those answers are provided. ## Step Progression @@ -17,9 +17,9 @@ Clarify responses should be short, bulleted questions grouped by topic (no JSON, | 13a — Completeness | Gate implementation on quality | Verify all specs are complete and actionable via `spec/13a_completeness_assessment.json`. | | 14 — Roadmap | Sequence the work (Core + Extensions) | Merge Step 09 baseline + Step 13 extensions into a tactical JIT execution plan. | | 15 — Scaffold | Generate compile-clean skeleton | Implement manually or via framework CLI. | -| 16a — Plan (Trinity) | Detailed Task & Sec/Ops Planning | Define tasks, security fixtures, dashboards, docs impact, and drift checks in `spec/impl_context/{step_id}.json`. | +| 16a — Plan (Trinity) | Checklist & Sec/Ops Planning | Define `summary`, `docs_impact`, `spec_alignment`, `review_requirements`, security fixtures, dashboards, alerts, and drift checks in `spec/impl_context/{step_id}.json`. | | 16b — Build (Trinity) | Implement & Config | Write Code, Configs, and update Docs. | -| 16c — Review (Trinity) | Audit & Gate | Verify Code/Sec/Ops, run full tests, emit Fixture Status default. | +| 16c — Review (Trinity) | Audit & Gate | Verify Code/Sec/Ops, run full tests, and emit structured `delivery_status` evidence (`deployments`, `dashboards_verified`, `alerts_verified`) when delivery is planned. | ## Step 16 Artifact Layout Step 16 uses **two levels** of artifacts: @@ -57,3 +57,4 @@ Ensure the generated `.github/workflows/spec_validation.yml` reflects these jobs - Current artifacts under `spec/13*` through `spec/16*` - Implemented scaffold or runtime referencing Step 05 contracts - Updated monitoring bindings and drift schedules guaranteeing the spec remains the single source of truth +- Review artifacts that include verifiable delivery evidence when `plan.delivery.status == planned` diff --git a/prompts/prompt_00_project_charter.md b/prompts/prompt_00_project_charter.md index 28cb7a73..8bbf59e6 100644 --- a/prompts/prompt_00_project_charter.md +++ b/prompts/prompt_00_project_charter.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 0 · Project Charter** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 0 · Project Charter** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -57,7 +57,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Owner reflects accountability for charter maintenance. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/00_charter.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_01_capabilities.md b/prompts/prompt_01_capabilities.md index b0d916a0..0ca72d6b 100644 --- a/prompts/prompt_01_capabilities.md +++ b/prompts/prompt_01_capabilities.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 1 · Capabilities** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 1 · Capabilities** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -54,7 +54,7 @@ You are a senior specification author and validator. Your job is to emit a singl - No duplicate or overlapping capabilities (glossary-normalized). # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/01_capabilities.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_02_system_sketch.md b/prompts/prompt_02_system_sketch.md index ed658ce0..abb122c9 100644 --- a/prompts/prompt_02_system_sketch.md +++ b/prompts/prompt_02_system_sketch.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 2 · System Sketch** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 2 · System Sketch** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -62,7 +62,7 @@ You are a senior specification author and validator. Your job is to emit a singl - External systems are identified with clear boundaries and owners. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/02_system_sketch.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_02a_delivery_baseline.md b/prompts/prompt_02a_delivery_baseline.md index 42739e47..d70ce83a 100644 --- a/prompts/prompt_02a_delivery_baseline.md +++ b/prompts/prompt_02a_delivery_baseline.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 02a · Delivery Baseline** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 02a · Delivery Baseline** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -54,7 +54,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Compliance labels reflect real obligations (or explicitly none). # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/02a_delivery_baseline.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_03_glossary.md b/prompts/prompt_03_glossary.md index 63762675..3aa34054 100644 --- a/prompts/prompt_03_glossary.md +++ b/prompts/prompt_03_glossary.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 3 · Glossary** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 3 · Glossary** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -53,7 +53,7 @@ You are a senior specification author and validator. Your job is to emit a singl - No duplicates/synonyms remain unresolved. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/03_glossary.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_04_functional_requirements.md b/prompts/prompt_04_functional_requirements.md index f7aee454..73d0f80a 100644 --- a/prompts/prompt_04_functional_requirements.md +++ b/prompts/prompt_04_functional_requirements.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 4 · Functional Requirements** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 4 · Functional Requirements** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -53,7 +53,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Traces to capability and (if known) API/NFR; IDs are kebab-case and stable. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/04_fr_list.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_05_interface_contracts.md b/prompts/prompt_05_interface_contracts.md index 68cf5b44..f9223803 100644 --- a/prompts/prompt_05_interface_contracts.md +++ b/prompts/prompt_05_interface_contracts.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 5 · Interface Contracts** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 5 · Interface Contracts** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -53,7 +53,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Security explicitly chosen and justified; owner set; traces to FRs/capabilities present. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/05_interface_contracts.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_06_invariants.md b/prompts/prompt_06_invariants.md index 4898e45b..f459989e 100644 --- a/prompts/prompt_06_invariants.md +++ b/prompts/prompt_06_invariants.md @@ -15,7 +15,7 @@ To verify your invariants logic, verify against a sample data file: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 6 · Invariants & Rules** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 6 · Invariants & Rules** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -56,7 +56,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Expressions are syntactically valid and reference existing fields; scope defined for each rule; severity set. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/06_invariants.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_07_nfrs.md b/prompts/prompt_07_nfrs.md index 35de9945..e8d65cf9 100644 --- a/prompts/prompt_07_nfrs.md +++ b/prompts/prompt_07_nfrs.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 7 · Non‑Functional Requirements** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 7 · Non‑Functional Requirements** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -51,7 +51,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Names/units align with glossary; traces connect to relevant FRs/APIs/components. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/07_nfrs.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete numbers and metrics; avoid "fast" or "secure". Every NFR must be measurable via specific metric. diff --git a/prompts/prompt_08_fixtures.md b/prompts/prompt_08_fixtures.md index ac490448..7b9f93b6 100644 --- a/prompts/prompt_08_fixtures.md +++ b/prompts/prompt_08_fixtures.md @@ -15,7 +15,7 @@ Then lint the fixtures for completeness: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 8 · Test Plan & Fixtures** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 8 · Test Plan & Fixtures** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -56,7 +56,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Inputs/expected align with schemas; targets list correct IDs; tags present for CI gating where needed. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/08_fixtures.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_09_impl_plan.md b/prompts/prompt_09_impl_plan.md index fd49a59f..820877c4 100644 --- a/prompts/prompt_09_impl_plan.md +++ b/prompts/prompt_09_impl_plan.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 9 · Implementation Plan** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 9 · Implementation Plan** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -53,7 +53,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Dependencies listed for external teams/systems; plan aligns with governance/CI expectations. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/09_impl_plan.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_10_governance.md b/prompts/prompt_10_governance.md index b8b6e37e..3c0a8810 100644 --- a/prompts/prompt_10_governance.md +++ b/prompts/prompt_10_governance.md @@ -16,7 +16,7 @@ To enforce the governance policies defined here (specifically commit messages), Failures here should block the merge. # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 10 · Governance & Change Control** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 10 · Governance & Change Control** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -56,7 +56,7 @@ You are a senior specification author and validator. Your job is to emit a singl - PR rules list core validations; reviewers cover necessary disciplines. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/10_governance.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_11_redteam.md b/prompts/prompt_11_redteam.md index c7fe127b..2a891efa 100644 --- a/prompts/prompt_11_redteam.md +++ b/prompts/prompt_11_redteam.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior security architect and "Red Team" specialist. Your job is to emit a single JSON artifact for **Step 11 · Red‑Team / Failure Modes** that is machine-checkable. You must identify specific threats against the defined interfaces and system sketch, not generic security platitudes. You must think like an attacker who knows the system internals. +You are a senior security architect and "Red Team" specialist. Your job is to emit a single JSON artifact for **Step 11 · Red‑Team / Failure Modes** that is machine-checkable to the file system. You must identify specific threats against the defined interfaces and system sketch, not generic security platitudes. You must think like an attacker who knows the system internals. ## Philosophy: "Shift Left" We are not looking for generic "OWASP Top 10" lists. We are looking for **specific failure modes** in *this* architecture. @@ -72,8 +72,7 @@ Use the `category` field to classify threats precisely: - [ ] Are `edge_cases` structured with IDs? # Output Rules -1. Return exactly one fenced code block with language `json`. -2. **NO** prose before or after the JSON. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/11_redteam.json` using the file creation tool. 3. Follow the **Embedded Schema** exactly. 4. `trace`: Include a root trace to `step-11` or relevant governance ticket. 5. `target_ids`: MUST be populated for every threat. diff --git a/prompts/prompt_12_ci_gates.md b/prompts/prompt_12_ci_gates.md index 0f3aec61..4b20783e 100644 --- a/prompts/prompt_12_ci_gates.md +++ b/prompts/prompt_12_ci_gates.md @@ -10,7 +10,7 @@ Validate the generated JSON: ``` # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 12 · CI Gates** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 12 · CI Gates** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. ## Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -73,7 +73,7 @@ Available CLI tools include: - `./tools/run_specdev.sh docs-lint --repo-root ./devspec_toolkit` - Enforce docs policy ## Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/12_ci_gates.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_13_extension_generator.md b/prompts/prompt_13_extension_generator.md index c8f44958..133a44a7 100644 --- a/prompts/prompt_13_extension_generator.md +++ b/prompts/prompt_13_extension_generator.md @@ -56,7 +56,7 @@ You are a Principal Software Architect and Technical Program Manager. Your goal - If no complex domains are found, return empty array. Do NOT invent trivial extensions. # Output Rules -1. Returns exactly one fenced code block with language `json`. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/13_extension_manifest.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. The `extensions` array must be sorted by `extension_id` (ext-01, ext-02...). diff --git a/prompts/prompt_13a_completeness_assessment.md b/prompts/prompt_13a_completeness_assessment.md index 0e21f1a7..e4835c15 100644 --- a/prompts/prompt_13a_completeness_assessment.md +++ b/prompts/prompt_13a_completeness_assessment.md @@ -16,7 +16,7 @@ To ensure full suite consistency and generate a traceability matrix for analysis ``` # Role -You are a senior specification auditor and quality control expert. Your job is to emit a single JSON artifact for **Step 13a · Completeness Assessment** that evaluates the state of the Discovery Phase (Steps 00-12) and identifies any gaps preventing implementation readiness. +You are a senior specification auditor and quality control expert. Your job is to emit a single JSON artifact for **Step 13a · Completeness Assessment** that evaluates the state of the Discovery Phase (Steps 00-12) and identifies any gaps preventing implementation readiness. You only write the canonical JSON to the file system. # Task - **Input context:** all existing spec artifacts (`00_charter.json` through `12_ci_gates.json`) and their corresponding guides, plus the extension manifest (`13_extension_manifest.json`). @@ -88,7 +88,7 @@ You are a senior specification auditor and quality control expert. Your job is t # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/13a_completeness_assessment.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. `completeness_rating.target` should always be 10. diff --git a/prompts/prompt_14_roadmap.md b/prompts/prompt_14_roadmap.md index e58bb28b..23a31547 100644 --- a/prompts/prompt_14_roadmap.md +++ b/prompts/prompt_14_roadmap.md @@ -15,7 +15,7 @@ Verify that the entire spec suite is consistent before finalizing the roadmap: ``` # Role -You are a senior program manager and architect. Your job is to emit a single JSON artifact for **Step 14 · Roadmap** that aggregates all discovery specs (Core 00-12 and Extensions) into a cohesive implementation plan. +You are a senior program manager and architect. Your job is to emit a single JSON artifact for **Step 14 · Roadmap** that aggregates all discovery specs (Core 00-12 and Extensions) into a cohesive implementation plan to the file system. # Task - **Input context:** Completed Phase 1 specs (`00_charter.json` through `12_ci_gates.json`) AND any Phase 2 Custom Extensions. @@ -124,7 +124,7 @@ You are a senior program manager and architect. Your job is to emit a single JSO - Use traceRef objects to cite upstream specs that shape the roadmap. # Output Rules -1. Return exactly one fenced code block with language `json`. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/14_roadmap.json` using the file creation tool. 2. The JSON (excluding `$schema`) must validate against the Embedded Schema (specifically `schema/14_roadmap.schema.json`). 3. Include a top-level `$schema` field that matches the schema URI. 4. All milestones must have `target_date`, `deliverables`, and `source_milestones`. diff --git a/prompts/prompt_15_scaffold.md b/prompts/prompt_15_scaffold.md index 3ca2b734..d9bf2844 100644 --- a/prompts/prompt_15_scaffold.md +++ b/prompts/prompt_15_scaffold.md @@ -12,7 +12,7 @@ Validate the generated JSON: After generating the JSON artifact, implement the scaffold manually or using your preferred generator/framework CLI. Ensure the generated routes match `05_interface_contracts.json`. # Role -You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 15 · Scaffold Generation** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only output the canonical JSON that matches the schema. +You are a senior specification author and validator. Your job is to emit a single JSON artifact for **Step 15 · Scaffold Generation** that is machine-checkable and immediately consumable by CI and generators. You do not write examples, tutorials, or comments. You only write the canonical JSON to the file system. # Task - **Input context:** previously authored spec artifacts (Charter, Capabilities, Glossary, FRs, etc.) available to you in the workspace; organizational constraints; known IDs for cross-references. @@ -51,7 +51,7 @@ You are a senior specification author and validator. Your job is to emit a singl - Service skeleton sufficient to run a minimal service; validators listed. # Output Rules -1. Return exactly one fenced code block with language `json`. No prose before or after. +1. Do not output the JSON in the chat. Write the final JSON artifact to `spec/15_scaffold.json` using the file creation tool. 2. The JSON must validate against the Embedded Schema below. 3. All IDs must be unique kebab-case strings. 4. Use concrete verbs and measurable outcomes; avoid adjectives that are not testable. diff --git a/prompts/prompt_16_impl_context.md b/prompts/prompt_16_impl_context.md index af679a99..b5724f80 100644 --- a/prompts/prompt_16_impl_context.md +++ b/prompts/prompt_16_impl_context.md @@ -21,9 +21,29 @@ Validate the generated JSON: # Role You are a senior software architect producing the Step 16 **Trinity Anchor**. -Generate a **machine‑checkable JSON artifact** that captures the plan, +Generate a **machine‑checkable JSON artifact** to the file system that captures the plan, implementation checklist, and review expectations for the *current* execution cycle. +## Output Mode (Compatibility) +- **Trinity harness mode (canonical):** + - Phase A: questions only (if blocked). + - Phase B: write/update artifact file on disk and return concise status (artifact path + validation result). +- **Manual coding-agent mode (Codex-style default):** + - Write/update `spec/16_impl_context.json` directly. + - Return a short confirmation with validation outcome. + - Do not emit fenced JSON in chat. + +## Zero-Assumption Protocol (Mandatory) +You must treat this step as evidence compilation, not creative generation. + +1. Every non-trivial field must be backed by a concrete source artifact read in this run. +2. If source evidence is missing, return questions-only output; do not fill with defaults except schema-safe empty values explicitly allowed by prompt rules. +3. Do not infer step scope, checklist content, or file patterns from prior chats. Use only current governed artifacts. +4. Do not invent `spec_ref` IDs, `commit_hash` values, file paths, or test commands. +5. If two sources conflict, record an ambiguity and block final emission until resolved. +6. Replace vague language ("standard", "common", "obvious", "etc.") with explicit statements or remove it. +7. Before emit, run a hallucination scrub: verify every identifier in checklist/spec refs appears in source files. + # Seed Order & Mandatory Sources - Read `spec/common/seed_manifest.json` first. - Use `step_requirements["16"]` if present. If missing, use the union of `16a/16b/16c`. @@ -45,10 +65,9 @@ implementation checklist, and review expectations for the *current* execution cy 4. **Drift Check**: Verify that the Anchor (Step 16) does not conflict with Milestone contexts (16a/b/c). 5. **Checklist**: Convert relevant spec requirements into atomic checklist items. 6. **Docs Impact**: Decide whether docs updates are required and list impacted docs. -7. **Roadmap Sync**: If you identify that milestones are fully completed based on the ingested context, you MUST update: - - `spec/14_roadmap.json`: Statuses to `done`. - - `spec/09_impl_plan.json`: Statuses to `done`. -8. **Emit**: Write `spec/16_impl_context.json`. +7. **Emit**: Produce the anchor artifact. + +Roadmap/progress status mutation is owned by Verifier + Orchestrator closure logic, not by anchor generation. # FORBIDDEN ACTIONS (Immediate Rejection) 1. **NEVER** hallucinate `step_id` or use loose references. @@ -57,6 +76,9 @@ implementation checklist, and review expectations for the *current* execution cy 4. **NEVER** use `plan.tasks` or `metadata`. # Field Definitions & Rules (MANDATORY) + +> **Schema Authority**: The schema (`schema/16_impl_context.schema.json`) is the single source of truth for field types, ranges, and required properties. The rules below are behavioral guidelines for how to populate them. When in conflict, the schema wins. + **Crucial**: Use the following exact definitions to ensure compliance: ## 1. `plan.summary` (The Step Summary) @@ -117,7 +139,7 @@ implementation checklist, and review expectations for the *current* execution cy ## 8. `plan.context` (Existing Codebase Context) * `existing_structures`: Array of known code or non-code structures. * *Rule*: Use strings for non-code artifacts, objects for code signatures. - * For code objects: `{ signature, source_file, line_range }` are required. + * For code objects: `{ signature, source_file }` are required; `line_range` is strongly recommended for traceability. * `coding_examples`: Optional array of illustrative code snippets. ## 9. `plan.security` (Security Considerations) @@ -158,6 +180,7 @@ Before emitting `spec/16_impl_context.json`, verify: - [ ] Every checklist item with `checklist_status: active` has an `implementation` block. - [ ] `target_file_patterns` are explicit (no `**/*` unless deferred). - [ ] If `docs_impact.status` is `required`, `docs_touched` has at least one entry. +- [ ] `plan.review_requirements` exists and contains test commands for active plans. - [ ] If `plan.status` is `deferred`, `deferred_reason` is provided. - [ ] No active Milestone Contexts (16a/b/c) conflict with this Anchor. @@ -177,7 +200,7 @@ Before emitting `spec/16_impl_context.json`, verify: | `plan.ambiguities` | array | no | Risk management (blocking/non_blocking issues) | | `plan.solution` | object | no | Architecture sketch and sequence of concerns | | `plan.context` | object | no | Existing codebase structures and coding examples | -| `plan.review_requirements` | object | no | Verification plan (test_commands, guidelines) | +| `plan.review_requirements` | object | yes | Verification plan (test_commands, guidelines) | | `plan.docs_impact` | object | yes | Documentation impact assessment | | `plan.security` | object | no | Security fixtures and spec mutations | | `plan.delivery` | object | no | Observability (dashboards, alerts) | @@ -190,1766 +213,84 @@ Before emitting `spec/16_impl_context.json`, verify: * **Anchor Drift**: Producing a Step 16 context that conflicts with the specific Milestone contexts (16a/b/c). *Fix*: Anchor must be the union/root, not a distinct implementation plan. * **Lazy Scope**: Leaving `target_file_patterns` empty or using broad `**/*` patterns. *Fix*: Must be explicit glob patterns based on `spec/impl_context/*.json`. * **Hidden Dependencies**: Introducing code changes that require new env vars or secrets without documenting them in `docs_impact`. *Fix*: Check `env` usage. -* **JSON dumps**: Dumping the JSON in the chat output. *Fix*: Only write the file. -* **Schema Hallucination**: Using fields like `plan.tasks` (deprecated) or `metadata` (untyped). *Fix*: Strict adherence to Embedded Schema. +* **Contract Drift**: Returning chat artifact dumps instead of disk-first artifact writes. *Fix*: write artifact on disk and return concise status only. +* **Schema Hallucination**: Using fields like `plan.tasks` (deprecated) or `metadata` (untyped). *Fix*: Strict adherence to `devspec_toolkit/schema/16_impl_context.schema.json`. # Clarification Questions - "Are there any active Milestone Contexts (16a/16b/16c) I should merge?" - "Does this Step 16 Anchor require specific documentation updates beyond the standard set?" - "Are there specific file patterns that should be strictly OUT of scope?" -# Embedded Schema -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://specdev.local/schema/16_impl_context.schema.json", - "title": "16_impl_context", - "description": "Unified artifact for the implementation loop (Plan -> Code -> Review). Enforces Checklist-Driven Implementation with evidence binding.", - "type": "object", - "additionalProperties": false, - "$defs": { - "specRef": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "fr", - "api", - "nfr", - "inv", - "fixture", - "doc", - "code" - ] - }, - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "note": { - "type": "string" - }, - "line_range": { - "type": "string", - "pattern": "^L\\d+-L\\d+$" - }, - "commit_hash": { - "type": "string", - "pattern": "^[0-9a-f]{40}$", - "not": { - "pattern": "^0{40}$" - } - } - }, - "required": [ - "type", - "id", - "line_range", - "commit_hash" - ] - }, - "severityLevel": { - "type": "string", - "enum": [ - "low", - "medium", - "high", - "critical" - ] - }, - "executionStatus": { - "type": "string", - "enum": [ - "passed", - "failed", - "blocked", - "partial" - ] - }, - "evidenceObject": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "log", - "snippet", - "screenshot" - ] - }, - "content": { - "type": "string", - "minLength": 20, - "pattern": "\\S" - }, - "evidence_ref": { - "type": "string" - } - }, - "required": [ - "type", - "content" - ] - } - }, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId", - "description": "The Step ID from the Roadmap (e.g., step-api-core)." - }, - "owner": { - "$ref": "https://specdev.local/schema/core/atoms/1#owner" - }, - "created_at": { - "$ref": "https://specdev.local/schema/core/atoms/1#timestamp" - }, - "seed_refs": { - "$ref": "https://specdev.local/schema/core/collections/1#seedRefArray" - }, - "extensions": { - "type": "object", - "description": "Structured extensions for domain-specific data.", - "additionalProperties": false, - "properties": { - "review_state": { - "type": "object", - "additionalProperties": false, - "properties": { - "outcome": { - "type": "string" - }, - "verified_by": { - "type": "string" - } - }, - "required": [ - "outcome" - ] - }, - "execution_context": { - "type": "object", - "additionalProperties": false, - "properties": { - "command_overrides": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - } - } - }, - "plan": { - "type": "object", - "additionalProperties": false, - "description": "Trinity loop plan (scope, checklist, review requirements, and documentation impact).", - "properties": { - "status": { - "type": "string", - "enum": [ - "active", - "deferred" - ] - }, - "deferred_reason": { - "type": "string" - }, - "summary": { - "type": "object", - "additionalProperties": false, - "properties": { - "functional_summary": { - "type": "string" - }, - "scope_in": { - "type": "array", - "items": { - "type": "string" - } - }, - "scope_out": { - "type": "array", - "items": { - "type": "string" - } - }, - "target_file_patterns": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Explicit list of files/directories to modify or create." - } - }, - "required": [ - "functional_summary", - "scope_in", - "scope_out", - "target_file_patterns" - ] - }, - "docs_impact": { - "type": "object", - "additionalProperties": false, - "description": "Documentation impact assessment. Required when any non-doc file is modified.", - "properties": { - "status": { - "type": "string", - "enum": [ - "required", - "not_required" - ] - }, - "rationale": { - "type": "string", - "minLength": 10 - }, - "docs_touched": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "status", - "rationale" - ], - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "required" - } - } - }, - "then": { - "required": [ - "docs_touched" - ], - "properties": { - "docs_touched": { - "minItems": 1 - } - } - } - } - ] - }, - "spec_alignment": { - "type": "object", - "additionalProperties": false, - "properties": { - "requirements_summary": { - "type": "array", - "description": "Thematic grouping of requirements.", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "theme": { - "type": "string" - }, - "summary": { - "type": "string" - }, - "spec_refs": { - "type": "array", - "items": { - "$ref": "#/$defs/specRef" - } - } - }, - "required": [ - "theme", - "summary" - ] - } - }, - "checklist": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#screamingSnakeId" - }, - "spec_ref": { - "$ref": "#/$defs/specRef" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "behavior", - "constraint", - "validation", - "metadata", - "perf", - "logging", - "docs" - ] - }, - "layer": { - "type": "string", - "enum": [ - "db", - "model", - "service", - "api", - "integration", - "tests", - "docs", - "config" - ] - }, - "checklist_status": { - "type": "string", - "enum": [ - "active", - "deferred" - ], - "default": "active" - }, - "linked_test_expectation": { - "oneOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - } - ] - }, - "implementation": { - "type": "object", - "description": "Atomic work definition for this specific requirement.", - "additionalProperties": false, - "properties": { - "status": { - "type": "string", - "enum": [ - "pending", - "in_progress", - "verified", - "deferred" - ] - }, - "files_touched": { - "type": "array", - "items": { - "type": "string" - } - }, - "actions": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "file_create", - "file_edit", - "run_command", - "manual_verification" - ] - }, - "description": { - "type": "string" - }, - "target": { - "type": "string" - }, - "command": { - "type": "string" - }, - "evidence": { - "$ref": "#/$defs/evidenceObject" - } - }, - "required": [ - "type", - "description" - ], - "allOf": [ - { - "if": { - "properties": { - "type": { - "enum": [ - "file_create", - "file_edit" - ] - } - } - }, - "then": { - "required": [ - "target" - ], - "properties": { - "target": { - "minLength": 1 - } - } - } - }, - { - "if": { - "properties": { - "type": { - "const": "run_command" - } - } - }, - "then": { - "required": [ - "command" - ], - "properties": { - "command": { - "minLength": 1 - } - } - } - } - ] - } - } - }, - "required": [ - "status", - "actions" - ], - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "verified" - } - } - }, - "then": { - "properties": { - "actions": { - "items": { - "required": [ - "evidence" - ] - } - } - } - } - } - ] - } - }, - "required": [ - "id", - "spec_ref", - "description", - "linked_test_expectation" - ], - "allOf": [ - { - "if": { - "not": { - "properties": { - "checklist_status": { - "const": "deferred" - } - } - } - }, - "then": { - "required": [ - "implementation" - ] - } - } - ] - } - } - }, - "required": [ - "checklist" - ] - }, - "ambiguities": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "description": { - "type": "string" - }, - "source": { - "type": "string", - "enum": [ - "spec", - "code", - "plan", - "mixed", - "review" - ] - }, - "severity": { - "type": "string", - "enum": [ - "blocking", - "non_blocking" - ] - }, - "impact": { - "type": "array", - "items": { - "type": "string" - } - }, - "proposed_assumption": { - "type": "string" - }, - "mitigation": { - "type": "string", - "minLength": 10 - }, - "status": { - "type": "string", - "enum": [ - "resolved", - "tracking", - "deferred", - "blocked" - ] - } - }, - "required": [ - "id", - "description", - "severity" - ], - "allOf": [ - { - "if": { - "properties": { - "severity": { - "const": "non_blocking" - } - } - }, - "then": { - "required": [ - "mitigation" - ] - } - } - ] - } - }, - "solution": { - "type": "object", - "additionalProperties": false, - "properties": { - "architecture_sketch": { - "type": "string" - }, - "sequence_of_concerns": { - "type": "array", - "items": { - "type": "string" - } - }, - "risks": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "architecture_sketch" - ] - }, - "context": { - "type": "object", - "additionalProperties": false, - "properties": { - "existing_structures": { - "type": "array", - "description": "Known code or non-code structures. Strings may reference non-code artifacts; objects must cite real code signatures.", - "items": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "signature": { - "type": "string" - }, - "source_file": { - "type": "string", - "pattern": "^[^/].*\\.(py|ts|js|go|rs)$" - }, - "line_range": { - "type": "string", - "pattern": "^L\\d+-L\\d+$" - } - }, - "required": [ - "signature", - "source_file" - ] - } - ] - } - }, - "coding_examples": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "title", - "code" - ] - } - } - } - }, - "review_requirements": { - "type": "object", - "additionalProperties": false, - "properties": { - "guidelines": { - "type": "string" - }, - "test_commands": { - "type": "array", - "items": { - "oneOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "command": { - "type": "string", - "minLength": 1 - }, - "expected_exit_code": { - "type": "integer", - "default": 0 - }, - "timeout_seconds": { - "type": "integer", - "minimum": 1, - "maximum": 3600 - }, - "description": { - "type": "string" - } - }, - "required": [ - "command" - ] - } - ] - } - } - }, - "required": [ - "test_commands" - ] - }, - "docs": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "required_updates": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "path": { - "type": "string" - }, - "update_summary": { - "type": "string" - } - }, - "required": [ - "path", - "update_summary" - ] - } - } - }, - "required": [ - "status", - "required_updates" - ] - } - ] - }, - "security": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "new_fixtures": { - "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" - }, - "spec_mutations": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "ref": { - "$ref": "https://specdev.local/schema/core/collections/1#traceRef" - }, - "change": { - "type": "string" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "ref", - "change", - "reason" - ] - } - } - }, - "required": [ - "status" - ] - } - ] - }, - "delivery": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "dashboards": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "dashboard_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "nfr_refs": { - "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" - }, - "url": { - "type": "string" - } - }, - "required": [ - "dashboard_id" - ] - } - }, - "alerts": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "alert_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "nfr_ref": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "rule": { - "type": "string" - }, - "severity": { - "$ref": "#/$defs/severityLevel" - } - }, - "required": [ - "alert_id", - "rule" - ] - } - } - }, - "required": [ - "status" - ] - } - ] - }, - "drift": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "checks": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "check_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "target": { - "type": "string", - "enum": [ - "api", - "schema", - "nfr", - "invariant", - "fixture", - "config" - ] - }, - "method": { - "type": "string", - "enum": [ - "runtime-sample", - "log-diff", - "schema-diff", - "trace-replay" - ] - }, - "schedule": { - "type": "string", - "pattern": "^(hourly|daily|weekly|monthly|@(annually|monthly|weekly|daily|hourly)|([0-9*/,-]+ ){4}[0-9*/,-]+)$", - "description": "Named interval (hourly/daily/weekly/monthly) or cron expression" - }, - "severity": { - "$ref": "#/$defs/severityLevel" - }, - "remediation_policy": { - "type": "string" - } - }, - "required": [ - "check_id", - "target", - "method" - ] - } - } - }, - "required": [ - "status" - ] - } - ] - }, - "coverage_status": { - "type": "object", - "additionalProperties": false, - "properties": { - "total": { - "type": "integer", - "minimum": 0 - }, - "verified": { - "type": "integer", - "minimum": 0 - }, - "deferred": { - "type": "integer", - "minimum": 0 - }, - "pending": { - "type": "integer", - "minimum": 0 - } - }, - "required": [ - "total", - "verified", - "deferred", - "pending" - ] - }, - "scope_validation": { - "type": "object", - "additionalProperties": false, - "properties": { - "in_scope": { - "type": "array", - "items": { - "type": "string" - } - }, - "out_of_scope": { - "type": "array", - "items": { - "type": "string" - } - }, - "acknowledged": { - "type": "boolean" - } - }, - "allOf": [ - { - "if": { - "properties": { - "out_of_scope": { - "minItems": 1 - } - } - }, - "then": { - "required": [ - "acknowledged" - ], - "properties": { - "acknowledged": { - "const": true - } - } - } - } - ] - } - }, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "deferred" - } - } - }, - "then": { - "required": [ - "deferred_reason" - ], - "properties": { - "summary": { - "properties": { - "target_file_patterns": { - "maxItems": 0 - } - } - }, - "review_requirements": { - "properties": { - "test_commands": { - "maxItems": 0 - } - } - } - } - }, - "else": { - "properties": { - "summary": { - "properties": { - "target_file_patterns": { - "minItems": 1 - } - } - }, - "review_requirements": { - "properties": { - "test_commands": { - "minItems": 1 - } - } - } - } - } - } - ] - }, - "execution": { - "type": "object", - "description": "Global execution summary.", - "additionalProperties": false, - "properties": { - "files_touched": { - "type": "array", - "items": { - "type": "string" - } - }, - "execution_results": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "$ref": "#/$defs/executionStatus" - }, - "outcome_description": { - "type": "string" - }, - "reasoning": { - "type": "string" - }, - "command": { - "type": "string" - }, - "evidence": { - "type": "string", - "minLength": 20 - }, - "evidence_ref": { - "type": "string" - }, - "evidence_binding": { - "type": "object", - "additionalProperties": false, - "properties": { - "timestamp": { - "type": "string", - "format": "date-time" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "exit_code": { - "type": "integer", - "minimum": 0, - "maximum": 255 - }, - "command": { - "type": "string" - } - }, - "required": [ - "timestamp", - "sha256", - "exit_code" - ] - } - }, - "required": [ - "status", - "outcome_description", - "reasoning", - "command", - "evidence" - ], - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "passed" - } - } - }, - "then": { - "required": [ - "evidence_ref", - "evidence_binding" - ], - "properties": { - "evidence": { - "pattern": "(PASSED|passed|OK|SUCCESS|✓|0 (errors|failures?|failed)|\\d+ passed)" - } - } - } - } - ] - } - }, - "critical_evidence": { - "type": "object", - "additionalProperties": false, - "properties": { - "satisfied_checklist_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "passed_test_commands": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "config_validation": { - "type": "object", - "additionalProperties": false, - "properties": { - "dashboard_links_valid": { - "type": "boolean" - }, - "alert_rules_valid": { - "type": "boolean" - }, - "drift_schedules_valid": { - "type": "boolean" - }, - "notes": { - "type": "string" - } - } - }, - "emergent_ambiguities": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "description": { - "type": "string" - }, - "severity": { - "type": "string" - }, - "impact": { - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "type": "string" - } - }, - "required": [ - "id", - "description", - "severity" - ] - } - }, - "final_status": { - "type": "object", - "additionalProperties": false, - "properties": { - "test_results": { - "type": "array", - "items": { - "type": "object" - } - }, - "ci_status": { - "type": "string", - "enum": [ - "green", - "red" - ] - } - } - } - } - }, - "review": { - "type": "object", - "additionalProperties": false, - "properties": { - "findings": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "type": { - "type": "string", - "enum": [ - "bug", - "gap", - "scope_creep", - "style", - "design", - "tests", - "docs" - ] - }, - "severity": { - "type": "string", - "enum": [ - "blocking", - "major", - "minor", - "nit" - ] - }, - "spec_ref": { - "$ref": "#/$defs/specRef" - }, - "description": { - "type": "string" - }, - "related_checklist_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "remediation_task": { - "type": "object", - "additionalProperties": false, - "properties": { - "task_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "summary": { - "type": "string" - }, - "files_to_touch": { - "type": "array", - "items": { - "type": "string" - } - }, - "checklist_ids": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "task_id", - "summary", - "files_to_touch", - "checklist_ids" - ] - }, - "metadata": { - "type": "object", - "additionalProperties": false, - "properties": { - "source": { - "type": "string" - }, - "impact": { - "type": "string" - } - }, - "required": [ - "source", - "impact" - ] - } - }, - "required": [ - "id", - "type", - "severity", - "spec_ref", - "description", - "metadata" - ], - "allOf": [ - { - "if": { - "properties": { - "severity": { - "enum": [ - "blocking", - "major" - ] - } - } - }, - "then": { - "required": [ - "remediation_task" - ] - } - } - ] - } - }, - "ratings": { - "type": "object", - "additionalProperties": false, - "properties": { - "spec_completeness": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "code_quality": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "tests_completeness": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "docs_completeness": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "metadata_usage": { - "type": "integer", - "minimum": 0, - "maximum": 5 - } - }, - "required": [ - "spec_completeness", - "code_quality", - "tests_completeness", - "docs_completeness", - "metadata_usage" - ] - }, - "verdict": { - "type": "string", - "enum": [ - "verified", - "deferred", - "rejected" - ] - }, - "next_actions": { - "type": "string" - }, - "fixture_status": { - "type": "object", - "additionalProperties": false, - "properties": { - "implemented_endpoints": { - "type": "array", - "items": { - "$ref": "https://specdev.local/schema/core/collections/1#traceId" - } - }, - "test_results": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "fixture_ref": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "status": { - "type": "string", - "enum": [ - "pass", - "fail", - "skip" - ] - }, - "notes": { - "type": "string" - } - }, - "required": [ - "fixture_ref", - "status" - ] - } - }, - "ci_status": { - "type": "string", - "enum": [ - "green", - "red" - ] - } - }, - "required": [ - "implemented_endpoints", - "test_results", - "ci_status" - ] - }, - "security_status": { - "type": "string", - "enum": [ - "green", - "red" - ] - }, - "delivery_status": { - "type": "object", - "additionalProperties": false, - "properties": { - "deployments": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "env": { - "type": "string", - "enum": [ - "dev", - "staging", - "prod" - ] - }, - "build_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "status": { - "type": "string", - "enum": [ - "pending", - "success", - "failed" - ] - } - }, - "required": [ - "env", - "build_id" - ] - } - } - } - } - }, - "allOf": [ - { - "if": { - "required": [ - "verdict" - ], - "properties": { - "verdict": { - "const": "verified" - } - } - }, - "then": { - "required": [ - "fixture_status" - ], - "properties": { - "fixture_status": { - "properties": { - "ci_status": { - "const": "green" - } - }, - "required": [ - "ci_status" - ] - } - } - } - } - ] - } - }, - "required": [ - "id", - "owner", - "created_at", - "seed_refs", - "plan" - ] -} -``` +# Canonical Schema Reference +- Use `devspec_toolkit/schema/16_impl_context.schema.json` as the only schema source of truth. +- Do not rely on copied or embedded schema fragments in prompts. +- Validate generated artifacts with `./tools/run_specdev.sh validate --repo-root ./devspec_toolkit`. + +# Output Contract (Schema-Valid Example) +Manual mode note: +- In manual coding-agent workflow, writing the file directly plus concise confirmation is valid. +- This JSON block is reference-only; do not emit it in chat. +- For richer schema-valid examples, reuse fixtures under `tests/fixtures/step_16/`. -# Output Contract ```json { - "id": "step-16-example", - "owner": "system", - "created_at": "2026-02-08T00:00:00Z", + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-anchor-minimal", + "owner": "api", + "created_at": "2024-01-01T00:00:00Z", "seed_refs": [ - { "seed_id": "seed-overview", "path": "docs/seed/seed_overview.md" } + { "seed_id": "seed-overview" }, + { "seed_id": "seed-tech-stack" } ], "plan": { "status": "active", "summary": { - "functional_summary": "Implement Core Authentication flow.", - "scope_in": ["Login", "Logout", "Session Management"], - "scope_out": ["OAuth", "MFA"], - "target_file_patterns": ["src/auth/*.py", "tests/auth/*.py"] - }, - "docs_impact": { - "status": "required", - "rationale": "New auth module requires API documentation updates.", - "docs_touched": ["docs/api/auth.md"] + "functional_summary": "Minimal anchor for active implementation cycle.", + "scope_in": ["core"], + "scope_out": ["extras"], + "target_file_patterns": ["src/main.py", "src/auth.py"] }, "spec_alignment": { "requirements_summary": [ - { "theme": "Security", "summary": "Implement JWT handling" } + { "theme": "Core Logic", "summary": "Implement core business logic" } ], "checklist": [ { - "id": "CHK_AUTH_01", + "id": "REQ_CORE_001", "spec_ref": { "type": "fr", - "id": "fr-auth-login", + "id": "fr-core-login", "line_range": "L10-L20", - "commit_hash": "a1b2c3d4e5f67890a1b2c3d4e5f67890a1b2c3d4" + "commit_hash": "a1b2c3d4e5f61234567890123456789012345678" }, - "description": "User can login with valid credentials.", - "linked_test_expectation": "pytest tests/auth/test_login.py::test_login_success", - "checklist_status": "active", + "description": "Implement login function", + "type": "behavior", + "layer": "service", + "linked_test_expectation": "pytest tests/auth/test_login.py::test_login_success -q", + "nfr_refs": ["nfr-availability-uptime"], + "fixture_ref": "fixture-login-success", "implementation": { - "status": "pending", - "files_touched": ["src/auth/login.py"], + "status": "in_progress", + "files_touched": ["src/auth.py"], "actions": [ { "type": "file_create", - "description": "Create login handler", - "target": "src/auth/login.py" + "target": "src/auth.py", + "description": "Create auth module" } ] } } ] }, - "ambiguities": [ - { - "id": "amb-token-storage", - "description": "Token storage mechanism not specified (in-memory vs Redis)", - "source": "spec", - "severity": "non_blocking", - "mitigation": "Default to in-memory for MVP, Redis for production", - "impact": ["session-management"], - "status": "resolved" - } - ], - "solution": { - "architecture_sketch": "Flask Blueprint with JWT extended.", - "sequence_of_concerns": ["Models", "Views", "Tests"], - "risks": ["Token leakage in logs"] - }, - "context": { - "existing_structures": [ - { "signature": "class User(db.Model)", "source_file": "src/models.py", "line_range": "L1-L50" } - ] + "docs_impact": { + "status": "required", + "rationale": "Code changes require documentation updates for traceability.", + "docs_touched": ["README.md"] }, "review_requirements": { - "test_commands": ["pytest tests/auth"] - }, - "security": { - "status": "planned", - "new_fixtures": ["fix-auth-token-leak"], - "spec_mutations": [ - { - "ref": { "type": "nfr", "id": "nfr-sec-01" }, - "change": "Add token rotation requirement", - "reason": "Mitigate token replay attacks" - } - ] - }, - "delivery": { - "status": "not_applicable", - "reason": "No observability changes required for initial implementation" - }, - "drift": { - "status": "planned", - "checks": [ - { - "check_id": "drift-auth-api", - "target": "api", - "method": "runtime-sample", - "schedule": "daily", - "remediation_policy": "Regenerate API fixtures from live endpoints" - } - ] - } - }, - "execution": { - "files_touched": ["src/auth/login.py", "tests/auth/test_login.py"], - "execution_results": [ - { - "status": "passed", - "outcome_description": "Login test passed with valid credentials", - "reasoning": "Implemented JWT token generation and validation", - "command": "pytest tests/auth/test_login.py::test_login_success", - "evidence": "tests/auth/test_login.py::test_login_success PASSED", - "evidence_ref": "artifacts/test_run_2026_02_08.log", - "evidence_binding": { - "timestamp": "2026-02-08T03:00:00Z", - "sha256": "abc123def456...", - "exit_code": 0, - "command": "pytest tests/auth/test_login.py::test_login_success" - } - } - ], - "critical_evidence": { - "satisfied_checklist_ids": ["CHK_AUTH_01"], - "passed_test_commands": ["pytest tests/auth"] - } - }, - "review": { - "findings": [ - { - "id": "rev-auth-01", - "type": "docs", - "severity": "minor", - "spec_ref": { - "type": "doc", - "id": "doc-api-auth", - "line_range": "L1-L10", - "commit_hash": "b1c2d3e4f5a67890b1c2d3e4f5a67890b1c2d3e4" - }, - "description": "API documentation missing error response codes", - "related_checklist_ids": ["CHK_AUTH_01"], - "metadata": { - "source": "reviewer", - "impact": "Documentation completeness" - } - } - ], - "ratings": { - "spec_completeness": 5, - "code_quality": 5, - "tests_completeness": 5, - "docs_completeness": 4, - "metadata_usage": 5 - }, - "verdict": "verified", - "next_actions": "Update API documentation with error codes", - "fixture_status": { - "implemented_endpoints": ["POST /auth/login"], - "test_results": [ - { - "fixture_ref": "fix-auth-login-success", - "status": "pass", - "notes": "All assertions passed" - } - ], - "ci_status": "green" + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": ["pytest tests/auth/test_login.py::test_login_success -q"] } } } diff --git a/prompts/prompt_16a_impl_planner.md b/prompts/prompt_16a_impl_planner.md index 4d110812..7d9ba47c 100644 --- a/prompts/prompt_16a_impl_planner.md +++ b/prompts/prompt_16a_impl_planner.md @@ -22,7 +22,27 @@ Validate the generated JSON: # Role You are a senior software architect and planning assistant. Your job is to generate the **Implementation Context** for a single Roadmap Step (Step 16a). -Instead of prose, you must **create or update the artifact file on disk** (`spec/impl_context/{step_id}.json`) with a machine-checkable **JSON artifact** that defines the plan, checklist, and tasks for the coding agent. +You must write a machine-checkable **JSON artifact** to the file system for `spec/impl_context/{step_id}.json` that defines the plan and checklist contract for implementation. + +## Output Mode (Compatibility) +- **Trinity harness mode (canonical):** + - Phase A: questions only (if blocked). + - Phase B: write/update artifact file on disk and return concise status (artifact path + validation result). +- **Manual coding-agent mode (Codex-style default):** + - Write/update `spec/impl_context/{step_id}.json` directly. + - Return a short confirmation with validation outcome. + - Do not emit fenced JSON in chat. + +## Zero-Assumption Protocol (Mandatory) +Planning output must be completely grounded and reproducible. + +1. Every checklist item must map to an observed governed requirement. If you cannot locate source lines, do not emit the item. +2. Every implementation action must reference a real file path from repository inspection; never infer filenames. +3. Every test expectation must be executable and explicit; never use placeholders or umbrella commands unless they are exactly what the step requires. +4. If step requirements are incomplete, emit `plan.ambiguities` + questions and stop; never silently patch gaps with guessed tasks. +5. If commit hashes cannot be resolved from git, stop and report blocker; never fabricate hashes. +6. For each emitted identifier (`checklist.id`, `spec_ref.id`, fixture refs), verify existence in source artifacts before finalizing. +7. If any statement cannot be traced to evidence, remove it or convert it into a blocking ambiguity. # Seed Order & Mandatory Sources - Read `spec/common/seed_manifest.json` first; follow `global_seed_order` and `step_requirements["16a"]`. @@ -31,6 +51,7 @@ Instead of prose, you must **create or update the artifact file on disk** (`spec - If a required seed is missing or stale, stop and request it before proceeding. - You must evaluate whether additional context (README maps, tooling docs, architecture guides, ops runbooks) is required for this step. If so, add new seeds to the manifest and update `step_requirements["16a"]` before proceeding. - When you add or change seeds, you MUST also plan documentation updates (see `plan.docs_impact`). + - **Seed expansion cap**: Limit to a maximum of **5 new seeds** per planning cycle. Additions beyond this require explicit user approval via Phase A questions. # Context To Ingest - **Roadmap**: Use the Step ID and description from `spec/14_roadmap.json` to scope the work. @@ -71,11 +92,13 @@ Instead of prose, you must **create or update the artifact file on disk** (`spec ### Atomicity Violations 1. **NEVER** group multiple behaviors in one checklist item -2. **NEVER** create checklist item that spans multiple files -3. **NEVER** create implementation action that requires >2 file edits +2. **PREFER** checklist items that can be validated with minimal file surface; multi-file items are allowed when the requirement is inherently cross-cutting. +3. **PREFER** small implementation actions, but do not impose hard file-count limits that conflict with real requirement boundaries. # Field Definitions & Rules (MANDATORY) +> **Schema Authority**: The schema (`schema/16_impl_context.schema.json`) is the single source of truth for field types, ranges, and required properties. The rules below are behavioral guidelines for how to populate them. When in conflict, the schema wins. + You must populate the JSON fields according to these specific definitions and expectations, derived from the rigorous DevSpec standard. ## 1. `plan.summary` (The Step Summary) @@ -132,6 +155,7 @@ You must populate the JSON fields according to these specific definitions and ex 1) **String form** for non-code or mixed structures (e.g., shell/nginx/workflow context) including a concrete file path in the text. 2) **Object form** for code signatures only: `{ "signature": "...", "source_file": "...", "line_range": "Lx-Ly" }`. * For object form, `source_file` must be a repo-relative path ending in `.py`, `.ts`, `.js`, `.go`, or `.rs`. + * `line_range` is strongly recommended for traceability but optional per schema. * *Rule*: Do NOT hallucinate. If you can't see the file, do not list it. ## 6. `plan.tasks` (DELETED) @@ -199,171 +223,87 @@ Use these fields to capture high-fidelity context that doesn't fit into standard * **Verification Gap**: Emitting a plan without explicitly verifying that it covers *all* requirements. *Fix*: **Verify-First** heuristic. # Output Rules -1. **Write/update** the artifact file at `spec/impl_context/{step_id}.json` with the full JSON output. -2. **Do not dump the JSON in the chat thread.** Instead, respond with a short confirmation that the file was updated and validation succeeded (or failed). -3. The JSON must validate against `schema/16_impl_context.schema.json`. -4. Populate the `plan` object fully. Leave `execution` and `review` objects empty. -5. If spec/seed drift was detected, update the relevant files under `spec/` (and `docs/seed` if applicable) as part of the same operation, and include those files in `plan.summary.target_file_patterns` and `plan.docs_impact.docs_touched`. +1. Canonical contract is disk-first two-phase: Phase A questions-only, Phase B writes artifact on disk and returns concise status. +2. The JSON must validate against `schema/16_impl_context.schema.json`. +3. Populate the `plan` object fully. `execution` and `review` may be omitted or left as empty objects. +4. In manual coding-agent mode, direct file write plus concise confirmation is the default behavior. +5. If spec/seed drift was detected, update relevant `spec/` files in-scope and include them in `plan.summary.target_file_patterns` and `plan.docs_impact.docs_touched`. # Clarification Questions - Which spec version covers this step? - Are there any ambiguous requirements that need resolution before coding? - Do we have existing tests we can extend, or must we create new ones? -# Embedded Schema -# Embedded Schema -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://specdev.local/schema/16_impl_context.schema.json", - "title": "16_impl_context", - "type": "object", - "additionalProperties": false, - "$defs": { - "specRef": { - "type": "object", - "required": ["type", "id", "line_range", "commit_hash"], - "properties": { - "type": { "enum": ["fr", "api", "nfr", "inv", "fixture", "doc", "code"] }, - "id": { "type": "string" }, - "line_range": { "type": "string" }, - "commit_hash": { "type": "string", "pattern": "^[0-9a-f]{40}$" } - } - } - }, - "properties": { - "id": { "type": "string" }, - "owner": { "type": "string" }, - "created_at": { "type": "string" }, - "extensions": { "type": "object" }, - "plan": { - "type": "object", - "required": ["summary", "spec_alignment", "review_requirements"], - "properties": { - "summary": { - "type": "object", - "required": ["functional_summary", "scope_in", "target_file_patterns"], - "properties": { - "functional_summary": { "type": "string" }, - "scope_in": { "type": "array", "items": { "type": "string" } }, - "scope_out": { "type": "array", "items": { "type": "string" } }, - "target_file_patterns": { "type": "array", "items": { "type": "string" } } - } - }, - "spec_alignment": { - "type": "object", - "required": ["checklist"], - "properties": { - "requirements_summary": { - "type": "array", - "items": { "type": "object", "required": ["theme", "summary"], "properties": { "theme": { "type": "string" }, "summary": { "type": "string" } } } - }, - "checklist": { - "type": "array", - "items": { - "type": "object", - "required": ["id", "spec_ref", "description", "linked_test_expectation"], - "properties": { - "id": { "type": "string" }, - "spec_ref": { "$ref": "#/$defs/specRef" }, - "description": { "type": "string" }, - "linked_test_expectation": { "type": "string" }, - "checklist_status": { "enum": ["active", "deferred"] }, - "implementation": { - "type": "object", - "required": ["status", "actions"], - "properties": { - "status": { "enum": ["pending", "in_progress", "verified", "deferred"] }, - "files_touched": { "type": "array", "items": { "type": "string" } }, - "actions": { - "type": "array", - "items": { - "type": "object", - "required": ["type", "description"], - "properties": { - "type": { "enum": ["file_create", "file_edit", "run_command", "manual_verification"] }, - "description": { "type": "string" }, - "target": { "type": "string" }, - "command": { "type": "string" }, - "evidence": { - "type": "object", - "required": ["type", "content"], - "properties": { - "type": { "enum": ["log", "snippet", "screenshot"] }, - "content": { "type": "string" } - } - } - } - } - } - } - } - } - } - } - } - }, - "review_requirements": { - "type": "object", - "required": ["test_commands"], - "properties": { - "test_commands": { "type": "array", "items": { "type": "string" } } - } - } - } - } - }, - "required": ["id", "plan"] -} -``` +# Canonical Schema Reference +- Use `devspec_toolkit/schema/16_impl_context.schema.json` as the only schema source of truth. +- Do not rely on copied or embedded schema fragments in prompts. +- Validate generated artifacts with `./tools/run_specdev.sh validate --repo-root ./devspec_toolkit`. + +# Output Contract (Schema-Valid Example) +Manual mode note: +- In manual coding-agent workflow, writing the file directly plus concise confirmation is valid. +- This JSON block is reference-only; do not emit it in chat. +- For richer schema-valid examples, reuse fixtures under `tests/fixtures/step_16/`. -# Output Contract ```json { - "id": "step-api-core", + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-minimal", "owner": "api", - "created_at": "2025-01-01T00:00:00Z", + "created_at": "2024-01-01T00:00:00Z", + "seed_refs": [ + { "seed_id": "seed-overview" }, + { "seed_id": "seed-tech-stack" } + ], "plan": { + "status": "active", "summary": { - "functional_summary": "Implement core API login", - "scope_in": ["Login", "Logout"], - "target_file_patterns": ["src/auth/*.py"] + "functional_summary": "Minimal implementation.", + "scope_in": ["core"], + "scope_out": ["extras"], + "target_file_patterns": ["src/main.py", "src/auth.py"] }, "spec_alignment": { "requirements_summary": [ - { "theme": "Auth", "summary": "Implement JWT-based Login/Logout" } + { "theme": "Core Logic", "summary": "Implement core business logic" } ], "checklist": [ { - "id": "CHK_AUTH_01", + "id": "REQ_CORE_001", "spec_ref": { - "type": "api", - "id": "api-auth-login", - "line_range": "L12-L15", - "commit_hash": "a1b2c3d4e5f67890a1b2c3d4e5f67890a1b2c3d4" + "type": "fr", + "id": "fr-core-login", + "line_range": "L10-L20", + "commit_hash": "a1b2c3d4e5f61234567890123456789012345678" }, - "description": "POST /login returns JWT", - "linked_test_expectation": "pytest tests/auth/test_login.py::test_jwt", - "checklist_status": "active", + "description": "Implement login function", + "type": "behavior", + "layer": "service", + "linked_test_expectation": "pytest tests/auth/test_login.py::test_login_success -q", + "nfr_refs": ["nfr-availability-uptime"], + "fixture_ref": "fixture-login-success", "implementation": { - "status": "pending", - "actions": [ - { - "type": "file_create", - "target": "src/auth/routes.py", - "description": "Create login endpoint" - } - ] + "status": "in_progress", + "files_touched": ["src/auth.py"], + "actions": [ + { + "type": "file_create", + "target": "src/auth.py", + "description": "Create auth module" + } + ] } } ] }, - "context": { - "existing_structures": [{ "signature": "class User", "source_file": "src/models.py" }] + "docs_impact": { + "status": "required", + "rationale": "Code changes require documentation updates for traceability.", + "docs_touched": ["README.md"] }, - // plan.tasks REMOVED "review_requirements": { - "test_commands": ["pytest tests/auth/"] + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": ["pytest tests/auth/test_login.py::test_login_success -q"] } } } diff --git a/prompts/prompt_16b_impl_coder.md b/prompts/prompt_16b_impl_coder.md index cee75de9..10f430c3 100644 --- a/prompts/prompt_16b_impl_coder.md +++ b/prompts/prompt_16b_impl_coder.md @@ -20,13 +20,33 @@ Instead of outputting code directly to the user, you: 1. **Write Code Files** (using tool calls). 2. **Update the Artifact** (`spec/impl_context/{step_id}.json`) to record your execution results. +## Output Mode (Compatibility) +- **Trinity harness mode (canonical):** + - Phase A: questions only (if blocked). + - Phase B: write/update artifact file on disk and return concise status (artifact path + validation result). +- **Manual coding-agent mode (Codex-style default):** + - Edit files and artifact directly. + - Return a short confirmation with validation outcome. + - Do not emit fenced JSON in chat. + +## Zero-Assumption Protocol (Mandatory) +Implementation is execution-only against explicit plan contracts. + +1. Do not implement any behavior without a corresponding checklist action. +2. Do not create new files/functions/classes unless explicitly required by checklist actions. +3. Do not infer command outcomes; command status must come from real execution output. +4. Do not infer pass/fail from partial logs; if output is incomplete, mark blocked and capture ambiguity. +5. Do not infer dependency presence; verify with concrete inspection commands. +6. If any required input is missing (spec ref, target path, test command), stop and record `execution.emergent_ambiguities`. +7. If an edit conflicts with scope constraints, abort immediately and return blocked status. + # Seed Order & Mandatory Sources - Read `spec/common/seed_manifest.json` first; follow `global_seed_order` and `step_requirements["16b"]`. - Ingest required seeds in order before any other context. - Populate `seed_refs` with the seeds actually used. - If a required seed is missing or stale, stop and request it before proceeding. -- You must evaluate whether additional context (README maps, tooling docs, architecture guides, ops runbooks) is required for this step. If so, add new seeds to the manifest and update `step_requirements["16b"]` before proceeding. - - If the plan does not include required seed changes in `plan.summary.target_file_patterns`, log an `emergent_ambiguity` and STOP. +- In Step 16b, you must **not** mutate `seed_manifest` or `step_requirements`. +- If context is missing, log `execution.emergent_ambiguities` and stop; escalation is handled by Planner/Orchestrator. # Task - **Input context:** `spec/impl_context/{step_id}.json` (The Plan). @@ -35,6 +55,8 @@ Instead of outputting code directly to the user, you: # Field Definitions & Rules (MANDATORY) +> **Schema Authority**: The schema (`schema/16_impl_context.schema.json`) is the single source of truth for field types, ranges, and required properties. The rules below are behavioral guidelines for how to populate them. When in conflict, the schema wins. + You must populate the `execution` JSON object according to these specific definitions and expectations. ## 1. `execution.files_touched` (Scope Control) @@ -53,7 +75,8 @@ You must populate the `execution` JSON object according to these specific defini * `status`: `passed`, `failed`, `blocked`, or `partial`. * `outcome_description`: Brief summary of what ran (e.g. "Ran Auth Tests"). * `reasoning`: Why did it pass/fail? (e.g., "All 5 tests passed"). - * `evidence`: **Verbatim** stdout/stderr snippet (max 20 lines) OR structured object. + * `evidence`: **Verbatim** stdout/stderr snippet as a string (schema-min length applies). + * `command`: Required command string. * **CRITICAL: EVIDENCE BINDING** * For `run_command` actions, you MUST emit `evidence` as a **String**: ```json @@ -63,6 +86,7 @@ You must populate the `execution` JSON object according to these specific defini * *Rule*: Do NOT say "not run" without a concrete blocker explanation. * *Rule*: **Verbatim Output**: Copy exact stdout/stderr. Do NOT paraphrase. * *Rule*: **Success Markers**: Output MUST contain `PASSED`, `OK`, `SUCCESS`, or exit code 0. +* *Rule*: When `status == "passed"`, both `evidence_ref` and `evidence_binding` are mandatory. ## 3. `checklist[].implementation.actions[].evidence` (Object Binding) * **MANDATORY**: Before marking an action as `verified`, you **MUST** populate its `evidence` field. @@ -70,39 +94,31 @@ You must populate the `execution` JSON object according to these specific defini ```json "evidence": { "type": "log", - "content": "pytest tests/auth/test_login.py ... [100%] PASSED" - } - ``` - ```json - "evidence": { - "type": "reference", - "content": "See docs/ops/environment_data_and_secrets.md for environment variables", - "path": "docs/ops/environment_data_and_secrets.md", - "section": "email-config" + "content": "pytest tests/auth/test_login.py::test_login_success ... [100%] PASSED" } ``` * *Rule*: The `content` must be a verbatim copy of the output captured in `execution_results`. -## 3. `execution.critical_evidence` (Traceability) +## 4. `execution.critical_evidence` (Traceability) * `satisfied_checklist_ids`: List of IDs that are now fully implemented and verified. * *Rule*: Only include an ID here if its `linked_test_expectation` command passed. * `passed_test_commands`: List of the specific test commands that passed. * *Expectation*: This allows the reviewer to trace each requirement to a passed test. -## 4. `execution.emergent_ambiguities` (Blockers) +## 5. `execution.emergent_ambiguities` (Blockers) * Log any blockers or spec issues you discovered. - * `id`: `AMB-NEW-X`. + * `id`: kebab-case (e.g., `amb-missing-seed-readme-map`). * `description`: The issue. * `severity`: `blocking` or `non_blocking`. * `impact`: Which checklist IDs are affected? -## 5. `execution.config_validation` (Ops Rigor) +## 6. `execution.config_validation` (Ops Rigor) * Applies when implementing `plan.delivery`, `plan.drift`, or `plan.security`. * *Rule*: **Dashboard Links**: Dashboard URLs must be valid/reachable or follow the known URI pattern. * *Rule*: **Alert Logic**: Alert rules must be syntactically valid for the monitoring system. * *Rule*: **Drift Schedules**: Schedules must be valid cron strings or ISO 8601 intervals. -## 6. Advanced Schema Fields (Consumption Rules) +## 7. Advanced Schema Fields (Consumption Rules) You must **READ** and **ACT** on these fields to ensure high-fidelity implementation. * **`checklist[].implementation`**: * **Requirement-First**: You are strictly implementing the actions defined in `checklist[].implementation`. @@ -135,6 +151,7 @@ Read Checklist → For Each Requirement → Fill Implementation Slots → Verify a. Any action fails → Set `status: blocked`, log `emergent_ambiguity` b. Plan contains `blocking` ambiguity → STOP immediately c. Required file outside `target_file_patterns` → STOP, log scope violation + d. On any blocked outcome, hand off to Orchestrator for **Planner-first** remediation; do not self-replan. 4. **Log**: Populate `execution` fields as defined above. 5. **Emit**: Save the updated JSON. @@ -170,35 +187,17 @@ Read Checklist → For Each Requirement → Fill Implementation Slots → Verify 4. **NEVER** modify `plan` outside `checklist[].implementation` evidence/status updates # Output Rules -1. Return exactly one fenced code block with language `json`. +1. Canonical contract is disk-first two-phase: Phase A questions-only, Phase B writes artifact on disk and returns concise status. 2. The JSON must validate against `schema/16_impl_context.schema.json`. 3. Do NOT modify `plan` outside `checklist[].implementation` evidence/status updates. Update `review` only when a checklist action explicitly targets review fields. +4. In manual coding-agent mode, direct file updates plus concise confirmation is the default behavior. -# Output Contract (Update Logic) -*Input*: -```json -{ "plan": { ... }, "execution": {} } -``` +# Canonical Schema Reference +- Use `devspec_toolkit/schema/16_impl_context.schema.json` as the only schema source of truth. +- Do not rely on copied or embedded schema fragments in prompts. +- Validate generated artifacts with `./tools/run_specdev.sh validate --repo-root ./devspec_toolkit`. -*Output*: -```json -{ - "plan": { ... }, - "execution": { - "files_touched": ["src/auth/routes.py"], - "execution_results": [ - { - "status": "passed", - "outcome_description": "Executed Auth Tests", - "reasoning": "All 5 tests passed, verifying JWT generation.", - "evidence": "pytest tests/auth/test_login.py ... [100%] PASSED" - } - ], - "critical_evidence": { - "satisfied_checklist_ids": ["CHK_AUTH_01"], - "passed_test_commands": ["pytest tests/auth/test_login.py"] - }, - "emergent_ambiguities": [] - } -} -``` +# Output Contract (Disk-First) +- In manual coding-agent workflow, writing files/artifacts directly plus concise confirmation is valid. +- Do not emit full artifact JSON in chat. +- For schema-valid examples, reuse fixtures under `tests/fixtures/step_16/`. diff --git a/prompts/prompt_16c_impl_reviewer.md b/prompts/prompt_16c_impl_reviewer.md index 0f252c09..c96a8472 100644 --- a/prompts/prompt_16c_impl_reviewer.md +++ b/prompts/prompt_16c_impl_reviewer.md @@ -12,28 +12,49 @@ After updating the JSON artifact, validate it: # Role You are a senior technical reviewer. Your job is to **Audit** the implementation of a Step by comparing the `plan` and `execution` in the `spec/impl_context/{step_id}.json` artifact against the actual code. -You output the final version of the JSON, populating the `review` section. +You write the final version of the JSON to the file system, populating the `review` section. + +## Output Mode (Compatibility) +- **Trinity harness mode (canonical):** + - Phase A: questions only (if blocked). + - Phase B: write/update artifact file on disk and return concise status (artifact path + validation result). +- **Manual coding-agent mode (Codex-style default):** + - Update the artifact directly. + - Return a short confirmation with validation outcome. + - Do not emit fenced JSON in chat. + +## Zero-Assumption Protocol (Mandatory) +Review output must be evidence-first and claim-minimized. + +1. Every finding must be traceable to concrete artifact lines or command evidence. +2. Never approve based on developer intent or summary prose alone. +3. Never assume omitted evidence implies success; absence of evidence is a failure signal. +4. Never soften severity when impact is unknown; unknown impact defaults to at least `major` until clarified. +5. Never infer checklist completion from partial action logs. +6. If required artifacts are missing, return non-verified verdict with explicit blocker finding. +7. Before final verdict, run contradiction checks: verdict vs ci_status, verdict vs finding severities, verdict vs evidence completeness. # Seed Order & Mandatory Sources - Read `spec/common/seed_manifest.json` first; follow `global_seed_order` and `step_requirements["16c"]`. - Ingest required seeds in order before any other context. - Populate `seed_refs` with the seeds actually used. - If a required seed is missing or stale, stop and request it before proceeding. -- You must evaluate whether additional context (README maps, tooling docs, architecture guides, ops runbooks) is required for this step. If so, add new seeds to the manifest and update `step_requirements["16c"]` before proceeding. +- In Step 16c, you must **not** mutate `seed_manifest` or `step_requirements`. +- If context is missing, return findings/ambiguity with a non-verified verdict and hand off to Planner/Orchestrator. # Task - **Input context:** `spec/impl_context/{step_id}.json` (Plan + Exec), plus the actual Codebase. - **Objective:** Verify correctness. If bugs exist, **spawn new remediation tasks**. - **Output Artifact:** A modified version of the input JSON, sorted into `spec/impl_context/{step_id}.json`. -## Crucial Side Effect (Roadmap Sync) -- If your `verdict` is `verified`, you **MUST** also update: - - `spec/14_roadmap.json`: Set the corresponding milestone's status to `done`. - - `spec/09_impl_plan.json`: Set the corresponding milestone's status to `done`. -- This ensures the high-level roadmap and implementation plan stay in sync with implementation reality. +## Roadmap Sync Ownership +- In Trinity harness mode, roadmap/progress sync is owned by Orchestrator after ingesting reviewer verdict. +- In manual mode, perform roadmap/progress updates only when the user explicitly requests same-turn closure updates. # Field Definitions & Rules (MANDATORY) +> **Schema Authority**: The schema (`schema/16_impl_context.schema.json`) is the single source of truth for field types, ranges, and required properties. The rules below are behavioral guidelines for how to populate them. When in conflict, the schema wins. + You must populate the `review` JSON object according to these specific definitions. ## 1. `review.fixture_status` (The Scoreboard) @@ -56,8 +77,8 @@ You must populate the `review` JSON object according to these specific definitio * `code_quality`: Is the code clean/safe? * `tests_completeness`: Are all paths tested? * `docs_completeness`: Are docs updated? - * `metadata_usage`: Are `metadata` fields used to capture lost context (Source, Impact)? - * **(New)**: `review.ratings` object MUST include `metadata_usage`. + * `context_metadata_usage`: Are structured `metadata` fields on checklist items and findings used to capture contextual provenance (Source, Impact, Decision rationale)? + * *Rule*: `review.ratings` object MUST include `context_metadata_usage`. * **Scale**: * **5**: Exemplary. Verified. (Specs are exhaustive, no "hand-waving"). * **4**: Good (minor nits). Verified. @@ -79,7 +100,7 @@ You must populate the `review` JSON object according to these specific definitio * `spec_ref`: **MANDATORY**. Cite the spec/plan line violated. * *Check*: Does the code match the Spec Version/Commit hash? If mismatch, flag as `gap`. * `description`: Concrete description of the issue. - * `metadata`: Optional map for `source` (e.g. "User Feedback") or `impact` (e.g. "Data Loss"). + * `metadata`: REQUIRED map with `source` and `impact`. * `remediation_task`: **REQUIRED for Blocking/Major items**. * See Section 3 below. @@ -111,8 +132,9 @@ You must populate the `review` JSON object according to these specific definitio * *Verdict*: `green` only if all mitigations verified. * **Delivery (Gate)**: * Verify `deployments` are recorded for all active environments (`dev`, `staging`, `prod`). - * *Check*: Are `dashboards` linked to NFRs? Do links work? - * *Check*: Do `alerts` exist for all Critical NFRs? + * *Check*: Are planned dashboards verified and captured in `delivery_status.dashboards_verified[]` with `evidence_ref`? + * *Check*: Do planned alerts have verified entries in `delivery_status.alerts_verified[]` with `evidence_ref`? + * *Rule*: If `plan.delivery.status == planned`, `review.delivery_status` MUST include at least one non-empty verification entry (`deployments`, `dashboards_verified`, or `alerts_verified`). * *Verdict*: `red` if any Critical NFR is unmonitored. * *Example*: ```json @@ -121,19 +143,19 @@ You must populate the `review` JSON object according to these specific definitio { "env": "dev", "build_id": "b123", "status": "success" }, { "env": "staging", "build_id": "b456", "status": "success" } ], - "dashboards": [ + "dashboards_verified": [ { "dashboard_id": "dashboard-availability", - "nfr_refs": ["nfr-availability-uptime"], - "url": "https://monitoring.example.com/dashboards/availability" + "url": "https://monitoring.example.com/dashboards/availability", + "evidence_ref": "sha256:dashboard-evidence" } ], - "alerts": [ + "alerts_verified": [ { "alert_id": "alert-latency-high", - "nfr_ref": "nfr-latency-page-load", "rule": "p99 > 200ms", - "severity": "critical" + "severity": "critical", + "evidence_ref": "sha256:alert-evidence" } ] } @@ -197,51 +219,16 @@ For each `checklist[]` item: 3. **NEVER** skip `metadata_usage` rating # Output Rule -1. Return exactly one fenced code block with language `json`. +1. Canonical contract is disk-first two-phase: Phase A questions-only, Phase B writes artifact on disk and returns concise status. 2. The JSON must validate against `schema/16_impl_context.schema.json`. +3. In manual coding-agent mode, direct file updates plus concise confirmation is the default behavior. -# Output Contract (Update Logic) -*Input*: -```json -{ "plan": { ... }, "execution": { ... }, "review": {} } -``` +# Canonical Schema Reference +- Use `devspec_toolkit/schema/16_impl_context.schema.json` as the only schema source of truth. +- Do not rely on copied or embedded schema fragments in prompts. +- Validate generated artifacts with `./tools/run_specdev.sh validate --repo-root ./devspec_toolkit`. -*Output*: -```json -{ - "plan": { ... }, - "execution": { ... }, - "review": { - "ratings": { - "spec_completeness": 5, - "code_quality": 4, - "tests_completeness": 5, - "docs_completeness": 3 - }, - "verdict": "deferred", - "findings": [ - { - "id": "finding-auth-01", - "type": "bug", - "description": "Login fails on empty password", - "severity": "major", - "remediation_task": { - "task_id": "rev-auth-01-fix", - "summary": "Add partial implementation for empty password check", - "checklist_ids": ["CHK_AUTH_01"], - "files_to_touch": ["src/auth/routes.py"] - } - } - ], - "fixture_status": { - "implemented_endpoints": ["api-auth-login"], - "test_results": [{ "fixture_ref": "fixture-auth-success", "status": "pass" }], - "ci_status": "green" - }, - "security_status": "green", - "delivery_status": { - "deployments": [{ "env": "dev", "build_id": "b123", "status": "success" }] - } - } -} -``` +# Output Contract (Disk-First) +- In manual coding-agent workflow, writing files/artifacts directly plus concise confirmation is valid. +- Do not emit full artifact JSON in chat. +- For schema-valid examples, reuse fixtures under `tests/fixtures/step_16/`. diff --git a/prompts/trinity/70_researcher.md b/prompts/trinity/70_researcher.md new file mode 100644 index 00000000..42d98cd2 --- /dev/null +++ b/prompts/trinity/70_researcher.md @@ -0,0 +1,133 @@ +# Trinity Utility Prompt · 70 Researcher + +## Purpose +Produce bounded, evidence-grounded context discovery for a specific Trinity task without introducing assumptions. This role exists only to collect and structure verifiable context that other roles can consume. + +## Invocation Preconditions +Run this role only when at least one of the following is true: +1. Required context cannot be found in already-loaded seed-governed artifacts. +2. A checklist item references code/docs paths that are ambiguous or missing. +3. A parent role explicitly requests a targeted context expansion. + +If none are true, return `status: "blocked"` with reason `research_not_required`. + +## Input Contract +The caller must provide all fields below. If any required field is missing, do not infer it. + +```json +{ + "protocol_version": "trinity-runtime-v1", + "role": "Researcher", + "phase": "utility", + "step_id": "m1-core-foundation | null", + "objective": "short statement of what must be discovered", + "input": { + "required_outputs": ["findings", "open_questions", "recommended_spec_refs"], + "bounded_scope": { + "allowed_paths": ["spec/", "src/", "docs/"], + "disallowed_paths": [".env", "secrets/", "node_modules/"], + "max_files": 40, + "max_commands": 20 + } + }, + "context_pack": { "allowed_read_paths": ["spec/", "src/", "docs/"] }, + "milestone_artifact_ref": "spec/impl_context/m1-core-foundation.json", + "tool_catalog_ref": ".trinity/runtime/tools/catalog.json" +} +``` + +## Non-Negotiable Grounding Rules +1. Every factual claim must cite at least one concrete artifact location (`path` + `line_range`). +2. Never claim existence of a file/function/symbol that you did not read. +3. Never use phrases like "likely", "probably", "standard", or "common pattern" as evidence. +4. Never expand beyond `bounded_scope.allowed_paths`. +5. If evidence conflicts, report conflict explicitly and set `status: "blocked"`. +6. If required evidence is missing, return `status: "questions"` and ask for exact missing artifacts. + +## Required Method +1. Restate objective and scope. +2. Enumerate candidate artifacts from allowed paths only. +3. Read only what is needed to satisfy objective. +4. Extract exact evidence snippets and normalized references. +5. Produce structured findings with confidence tied to evidence coverage. +6. Emit unresolved questions for anything not fully grounded. + +## Output Contract +Return only JSON with this shape: + +```json +{ + "status": "ready | questions | blocked", + "objective": "string", + "scope_executed": { + "files_read": ["path"], + "commands_run": ["command"], + "scope_violations": [] + }, + "findings": [ + { + "id": "res-001", + "claim": "verifiable statement", + "evidence": [ + { + "path": "repo-relative path", + "line_range": "Lx-Ly", + "excerpt": "verbatim snippet" + } + ], + "confidence": 0.0, + "confidence_reason": "coverage and consistency explanation" + } + ], + "recommended_spec_refs": [ + { + "type": "fr | api | inv | nfr | fixture", + "id": "spec-id", + "path": "spec/path.json", + "line_range": "Lx-Ly", + "commit_hash": "40-char sha" + } + ], + "open_questions": [ + { + "id": "q-001", + "blocking": true, + "question": "exact clarification request", + "required_artifact": "path or id" + } + ], + "errors": [] +} +``` + +## Runtime Wrapper Contract +When running inside Trinity runtime, return the payload above through the protocol wrapper: + +```json +{ + "action": "final_result", + "summary": "short closure summary", + "loop_checkpoint": { + "draft": "what you drafted", + "review": "what you checked", + "refine": "what you corrected" + }, + "utility_result": { + "...": "use the Output Contract fields above" + } +} +``` + +## Stop Conditions +Return immediately with `status: "blocked"` when: +1. Allowed paths are missing or empty. +2. Required files do not exist. +3. Conflicting evidence cannot be resolved deterministically. +4. Scope budget (`max_files` or `max_commands`) is exhausted before objective is met. + +## Self-Check Before Return +1. Did every claim include at least one concrete evidence reference? +2. Are all evidence excerpts verbatim and attributable? +3. Did any statement depend on intuition instead of observed artifacts? +4. Are unresolved unknowns converted into explicit `open_questions`? +5. Is output JSON schema-valid and assumption-free? diff --git a/prompts/trinity/80_tool_usage.md b/prompts/trinity/80_tool_usage.md new file mode 100644 index 00000000..2cd9f56c --- /dev/null +++ b/prompts/trinity/80_tool_usage.md @@ -0,0 +1,157 @@ +# Trinity Utility Prompt · 80 Tool Usage + +## Purpose +Generate deterministic, schema-valid tool call plans and/or executions with explicit constraints, so Trinity never relies on implicit behavior or freeform guesses. + +## Invocation Preconditions +Use this role when: +1. A task requires two or more tool actions with dependency ordering. +2. Write-path enforcement or command safety constraints must be validated upfront. +3. Parent role requests a deterministic tool sequence artifact. + +If the task is trivial and single-step, return `status: "blocked"` with reason `tool_planner_not_required`. + +## Input Contract +All fields are required unless explicitly nullable. + +```json +{ + "protocol_version": "trinity-runtime-v1", + "role": "ToolUser", + "phase": "utility", + "step_id": "m1-core-foundation | null", + "objective": "what the tool sequence must accomplish", + "input": { + "required_outputs": ["tool_calls", "postflight_checks"], + "available_tools": [ + "read_file", + "write_file", + "edit_file", + "apply_patch", + "list_dir", + "glob_match", + "search_text", + "git_head", + "git_show", + "git_diff", + "exec_cmd", + "validate_json", + "checkpoint_branch", + "checkpoint_commit" + ], + "constraints": { + "allowed_read_paths": ["spec/", "src/", "tests/"], + "allowed_write_paths": ["src/", "tests/", "spec/impl_context/"], + "target_file_patterns": ["src/auth.py", "tests/auth/test_login.py"], + "forbidden_commands": ["cat .env", "printenv", "history"] + } + }, + "context_pack": { "allowed_read_paths": ["spec/", "src/", "tests/"] }, + "execution_mode": "plan_only | execute" +} +``` + +## Non-Negotiable Rules +1. Never reference a tool not listed in `available_tools`. +2. Never schedule writes outside `allowed_write_paths` and `target_file_patterns`. +3. Never emit a command that can dump secrets or bypass scope. +4. Never assume intermediate state; declare every prerequisite explicitly. +5. For every write/edit, include a preceding read/inspect step unless artifact is new. +6. If required tool capability is missing, return `status: "questions"` and ask for explicit enablement. + +## Deterministic Planning Procedure +1. Build preflight checks: path existence, permissions, branch state, baseline file reads. +2. Build execution calls: one objective per call; no overloaded calls. +3. Build postflight checks: schema validation, diff inspection, test command capture. +4. Build rollback/containment plan for failed writes or failed validations. +5. Emit explicit expected artifacts for each call. + +## Output Contract +Return only JSON: + +```json +{ + "status": "ready | questions | blocked", + "objective": "string", + "preflight_checks": [ + { + "id": "pre-001", + "tool_name": "read_file", + "args": { "path": "spec/impl_context/m1-core-foundation.json" }, + "why": "confirm artifact exists before edits" + } + ], + "tool_calls": [ + { + "order": 1, + "tool_name": "edit_file", + "args": { + "path": "src/auth.py", + "edits": [ + { "search": "old", "replace": "new" } + ] + }, + "expected_artifacts": [ + { + "artifact_ref": "src/auth.py", + "validation": ["git_diff", "validate_json"] + } + ], + "failure_policy": "stop | continue_with_warning" + } + ], + "postflight_checks": [ + { + "id": "post-001", + "tool_name": "exec_cmd", + "args": { "command": "pytest tests/auth/test_login.py -q", "mode": "summarized" }, + "pass_markers": ["PASSED", "0 failed"] + } + ], + "rollback_plan": [ + { + "trigger": "schema validation failure", + "actions": ["revert_uncommitted_changes_for_target_files", "emit_blocking_finding"] + } + ], + "open_questions": [], + "errors": [] +} +``` + +## Runtime Wrapper Contract +When running inside Trinity runtime, return the payload above via: + +```json +{ + "action": "final_result", + "summary": "short closure summary", + "loop_checkpoint": { + "draft": "what you drafted", + "review": "what you checked", + "refine": "what you corrected" + }, + "utility_result": { + "...": "use the Output Contract fields above" + } +} +``` + +## Execution Mode Rules +If `execution_mode == "execute"`: +1. Execute exactly in `tool_calls[].order`. +2. Persist each request/result in Trinity tool protocol artifacts. +3. Stop immediately on first `failure_policy: stop` failure. +4. Never auto-replan inside the same run; return failure details to parent. + +## Stop Conditions +Return `status: "blocked"` when: +1. Any planned call violates path constraints. +2. Any required preflight check cannot be represented with available tools. +3. Required validation tools are unavailable. + +## Self-Check Before Return +1. Are all tool calls schema-valid for the declared tool protocol? +2. Is every write operation traceable to scope constraints? +3. Are all assumptions converted to explicit checks? +4. Does the plan include explicit failure handling and rollback? diff --git a/prompts/trinity/90_summarizer.md b/prompts/trinity/90_summarizer.md new file mode 100644 index 00000000..5b22c3a4 --- /dev/null +++ b/prompts/trinity/90_summarizer.md @@ -0,0 +1,114 @@ +# Trinity Utility Prompt · 90 Summarizer + +## Purpose +Extract concise, verbatim evidence from long command outputs without paraphrasing, invention, or omission of critical pass/fail markers. + +## Invocation Preconditions +Use this role only when output is too long for direct inclusion but Step 16 evidence is still required. + +If raw output length is manageable and direct evidence can be preserved without truncation, return `status: "blocked"` with reason `summarizer_not_required`. + +## Input Contract +Caller must provide: + +```json +{ + "protocol_version": "trinity-runtime-v1", + "role": "Summarizer", + "phase": "utility", + "step_id": "m1-core-foundation | null", + "objective": "extract deterministic pass/fail evidence from command output", + "input": { + "required_outputs": ["classification", "evidence_excerpt", "markers_found"], + "command": "pytest tests/auth/test_login.py -q", + "raw_output_ref": ".trinity/workspace//logs/test.log", + "extraction_rules": { + "required_markers": ["PASSED", "FAILED", "ERROR", "exit code"], + "max_lines": 20, + "include_context_lines": 2, + "verbatim_only": true + } + }, + "classification_goal": "pass | fail | blocked", + "context_pack": { "allowed_read_paths": [".trinity/workspace/"] } +} +``` + +## Non-Negotiable Rules +1. Never paraphrase extracted evidence lines. +2. Never rewrite, normalize, or redact pass/fail markers. +3. Never classify as `pass` if required markers are absent. +4. Never claim a command succeeded if exit status is missing or contradictory. +5. If evidence is ambiguous, return `status: "questions"` or `status: "blocked"` with exact reason. + +## Extraction Procedure +1. Load raw output from `raw_output_ref`. +2. Locate required markers and exit-code signals. +3. Select smallest contiguous evidence windows satisfying marker coverage. +4. Keep verbatim text only; preserve case and punctuation. +5. Produce deterministic classification and uncertainty notes. + +## Output Contract +Return only JSON: + +```json +{ + "status": "ready | questions | blocked", + "command": "string", + "raw_output_ref": "path", + "classification": "pass | fail | blocked | unknown", + "evidence_excerpt": "verbatim excerpt containing required markers", + "evidence_windows": [ + { + "start_line": 1, + "end_line": 4, + "lines": [ + "tests/auth/test_login.py::test_success PASSED", + "1 passed in 0.12s" + ] + } + ], + "markers_found": ["PASSED"], + "markers_missing": [], + "confidence": 0.0, + "confidence_reason": "coverage and contradiction analysis", + "open_questions": [], + "errors": [] +} +``` + +## Runtime Wrapper Contract +When running inside Trinity runtime, return the payload above via: + +```json +{ + "action": "final_result", + "summary": "short closure summary", + "loop_checkpoint": { + "draft": "what you drafted", + "review": "what you checked", + "refine": "what you corrected" + }, + "utility_result": { + "...": "use the Output Contract fields above" + } +} +``` + +## Classification Rules +1. `pass`: explicit success markers and no contradictory failure markers. +2. `fail`: explicit failure marker, non-zero exit code, or error traceback. +3. `blocked`: output truncated/missing so decision cannot be made safely. +4. `unknown`: partial signal without deterministic outcome; must include open question. + +## Stop Conditions +Return `status: "blocked"` when: +1. `raw_output_ref` is missing or unreadable. +2. Required markers are absent and no trustworthy fallback exists. +3. Evidence exceeds `max_lines` after applying minimal windows. + +## Self-Check Before Return +1. Is every evidence line verbatim from source? +2. Does the classification match objective markers without assumptions? +3. Are contradictions explicitly reported? +4. Is output compact but sufficient for reviewer traceability? diff --git a/prompts/trinity/99_auditor.md b/prompts/trinity/99_auditor.md new file mode 100644 index 00000000..db3db5fb --- /dev/null +++ b/prompts/trinity/99_auditor.md @@ -0,0 +1,156 @@ +# Trinity Utility Prompt · 99 Auditor + +## Purpose +Run a structured, evidence-bound audit of candidate artifacts before publish/ingest, with zero tolerance for unsupported claims. + +## Invocation Preconditions +Use this role for: +1. Cross-cutting pre-publish quality gate. +2. Independent pass/fail assessment of Builder or Verifier outputs. +3. Policy-focused checks (scope, evidence, traceability, docs, tests). + +If caller cannot supply artifact refs and applicable checklist/spec refs, return `status: "questions"`. + +## Input Contract + +```json +{ + "protocol_version": "trinity-runtime-v1", + "role": "Auditor", + "phase": "utility", + "step_id": "m1-core-foundation | null", + "objective": "independent evidence-bound audit before publish", + "input": { + "required_outputs": ["checks", "findings", "recommendation"], + "artifact_refs": [ + "spec/impl_context/m1-core-foundation.json", + "src/auth.py", + "tests/auth/test_login.py" + ], + "audit_scope": { + "checklist_ids": ["CHK_AUTH_01"], + "required_spec_refs": [ + { "type": "fr", "id": "fr-auth-login" } + ], + "must_check": ["scope", "tests", "evidence", "docs", "security"] + } + }, + "severity_policy": { + "blocking_requires_remediation_task": true, + "major_requires_remediation_task": true + } +} +``` + +## Non-Negotiable Rules +1. Every finding must include concrete evidence (`path`, `line_range`, `excerpt`). +2. Never emit `verified`-style pass conclusion if any blocking control is not checked. +3. Never downgrade severity to avoid remediation. +4. Never create findings from inferred behavior not present in artifacts. +5. Never omit remediation tasks for `blocking` or `major` findings. + +## Audit Procedure +1. Validate artifact presence and parseability. +2. Evaluate scope adherence against target patterns and declared checklist. +3. Evaluate execution evidence integrity and pass marker validity. +4. Evaluate traceability to required spec refs. +5. Evaluate documentation and test contract completeness. +6. Emit deterministic findings and closure recommendation. + +## Output Contract +Return only JSON: + +```json +{ + "status": "ready | questions | blocked", + "audit_summary": { + "artifacts_audited": 0, + "checks_executed": 0, + "checks_failed": 0 + }, + "checks": [ + { + "check_id": "scope-001", + "status": "pass | fail | blocked", + "evidence": [ + { + "path": "src/auth.py", + "line_range": "L10-L18", + "excerpt": "verbatim snippet" + } + ], + "notes": "specific outcome" + } + ], + "findings": [ + { + "id": "aud-001", + "type": "bug | gap | scope_creep | tests | docs | design | security", + "severity": "blocking | major | minor | nit", + "description": "specific and reproducible issue statement", + "spec_ref": { + "type": "fr | api | inv | nfr | fixture", + "id": "spec-id", + "line_range": "Lx-Ly", + "commit_hash": "40-char sha" + }, + "evidence": [ + { + "path": "repo-relative path", + "line_range": "Lx-Ly", + "excerpt": "verbatim snippet" + } + ], + "metadata": { + "source": "manual-audit | test-output | schema-validation", + "impact": "security-risk | functional-failure | maintainability-risk" + }, + "remediation_task": { + "task_id": "rem-001", + "summary": "exact fix direction", + "checklist_ids": ["CHK_AUTH_01"], + "files_to_touch": ["src/auth.py", "tests/auth/test_login.py"] + } + } + ], + "recommendation": "pass | needs_remediation | blocked", + "open_questions": [], + "errors": [] +} +``` + +## Runtime Wrapper Contract +When running inside Trinity runtime, return the payload above via: + +```json +{ + "action": "final_result", + "summary": "short closure summary", + "loop_checkpoint": { + "draft": "what you drafted", + "review": "what you checked", + "refine": "what you corrected" + }, + "utility_result": { + "...": "use the Output Contract fields above" + } +} +``` + +## Severity Assignment Rules +1. `blocking`: correctness/security/data-loss risk; release cannot proceed. +2. `major`: high-confidence gap against required spec/test/docs contracts. +3. `minor`: non-blocking quality issue with straightforward fix. +4. `nit`: style/polish issue with no correctness impact. + +## Stop Conditions +Return `status: "blocked"` when: +1. Required artifacts are missing. +2. Evidence cannot be traced to concrete lines/outputs. +3. Required scope checks cannot be executed deterministically. + +## Self-Check Before Return +1. Do all findings include reproducible evidence? +2. Do blocking/major findings include remediation tasks? +3. Is recommendation consistent with findings severity? +4. Did any conclusion rely on assumptions not supported by artifacts? diff --git a/schema/16_impl_context.schema.json b/schema/16_impl_context.schema.json index b7a662d5..ccff8576 100644 --- a/schema/16_impl_context.schema.json +++ b/schema/16_impl_context.schema.json @@ -1,1661 +1,1822 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://specdev.local/schema/16_impl_context.schema.json", - "title": "16_impl_context", - "description": "Unified artifact for the implementation loop (Plan -> Code -> Review). Enforces Checklist-Driven Implementation with evidence binding.", - "type": "object", - "additionalProperties": false, - "$defs": { - "specRef": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "fr", - "api", - "nfr", - "inv", - "fixture", - "doc", - "code" - ] - }, - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "note": { - "type": "string" - }, - "line_range": { - "type": "string", - "pattern": "^L\\d+-L\\d+$" - }, - "commit_hash": { - "type": "string", - "pattern": "^[0-9a-f]{40}$", - "not": { - "pattern": "^0{40}$" - } - } - }, - "required": [ - "type", - "id", - "line_range", - "commit_hash" - ] + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/16_impl_context.schema.json", + "title": "16_impl_context", + "description": "Unified artifact for the implementation loop (Plan -> Code -> Review). Enforces Checklist-Driven Implementation with evidence binding.", + "type": "object", + "additionalProperties": false, + "$defs": { + "specRef": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "fr", + "api", + "nfr", + "inv", + "fixture" + ] }, - "severityLevel": { - "type": "string", - "enum": [ - "low", - "medium", - "high", - "critical" - ] + "id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" }, - "executionStatus": { - "type": "string", - "enum": [ - "passed", - "failed", - "blocked", - "partial" - ] + "note": { + "type": "string" }, - "evidenceObject": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "log", - "snippet", - "screenshot", - "reference" - ] - }, - "content": { - "type": "string", - "minLength": 20, - "pattern": "\\S" - }, - "evidence_ref": { - "type": "string" - }, - "path": { - "type": "string" - }, - "section": { - "type": "string" - } - }, - "required": [ - "type", - "content" - ] + "line_range": { + "type": "string", + "pattern": "^L\\d+-L\\d+$" + }, + "commit_hash": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "not": { + "pattern": "^0{40}$" + } } + }, + "required": [ + "type", + "id", + "line_range", + "commit_hash" + ] }, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId", - "description": "The Step ID from the Roadmap (e.g., step-api-core)." + "severityLevel": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "critical" + ] + }, + "executionStatus": { + "type": "string", + "enum": [ + "passed", + "failed", + "blocked", + "partial" + ] + }, + "evidenceObject": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "log", + "snippet", + "screenshot", + "reference" + ] }, - "owner": { - "$ref": "https://specdev.local/schema/core/atoms/1#owner" + "content": { + "type": "string", + "minLength": 20, + "pattern": "\\S" }, - "created_at": { - "$ref": "https://specdev.local/schema/core/atoms/1#timestamp" + "evidence_ref": { + "type": "string" }, - "seed_refs": { - "$ref": "https://specdev.local/schema/core/collections/1#seedRefArray" + "path": { + "type": "string" }, - "extensions": { - "type": "object", - "description": "Structured extensions for domain-specific data.", - "additionalProperties": false, - "properties": { - "review_state": { - "type": "object", - "additionalProperties": false, - "properties": { - "outcome": { - "type": "string" - }, - "verified_by": { - "type": "string" - } - }, - "required": [ - "outcome" - ] - }, - "execution_context": { - "type": "object", - "additionalProperties": false, - "properties": { - "command_overrides": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } + "section": { + "type": "string" + } + }, + "required": [ + "type", + "content" + ] + } + }, + "properties": { + "id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId", + "description": "The Step ID from the Roadmap (e.g., step-api-core)." + }, + "owner": { + "$ref": "https://specdev.local/schema/core/atoms/1#owner" + }, + "created_at": { + "$ref": "https://specdev.local/schema/core/atoms/1#timestamp" + }, + "seed_refs": { + "$ref": "https://specdev.local/schema/core/collections/1#seedRefArray" + }, + "extensions": { + "type": "object", + "description": "Structured extensions for domain-specific data.", + "additionalProperties": false, + "properties": { + "review_state": { + "type": "object", + "additionalProperties": false, + "properties": { + "outcome": { + "type": "string" + }, + "verified_by": { + "type": "string" + } + }, + "required": [ + "outcome" + ] + }, + "execution_context": { + "type": "object", + "additionalProperties": false, + "properties": { + "command_overrides": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "plan": { + "type": "object", + "additionalProperties": false, + "description": "Trinity loop plan (scope, checklist, review requirements, and documentation impact).", + "properties": { + "status": { + "type": "string", + "enum": [ + "active", + "deferred" + ] + }, + "deferred_reason": { + "type": "string" + }, + "summary": { + "type": "object", + "additionalProperties": false, + "properties": { + "functional_summary": { + "type": "string" + }, + "scope_in": { + "type": "array", + "items": { + "type": "string" + } + }, + "scope_out": { + "type": "array", + "items": { + "type": "string" + } + }, + "target_file_patterns": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Explicit list of files/directories to modify or create." + } + }, + "required": [ + "functional_summary", + "scope_in", + "scope_out", + "target_file_patterns" + ] + }, + "docs_impact": { + "type": "object", + "additionalProperties": false, + "description": "Documentation impact assessment. Required when any non-doc file is modified.", + "properties": { + "status": { + "type": "string", + "enum": [ + "required", + "not_required" + ] + }, + "rationale": { + "type": "string", + "minLength": 10 + }, + "docs_touched": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "status", + "rationale" + ], + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "required" + } } + }, + "then": { + "required": [ + "docs_touched" + ], + "properties": { + "docs_touched": { + "minItems": 1 + } + } + } } + ] }, - "plan": { - "type": "object", - "additionalProperties": false, - "description": "Trinity loop plan (scope, checklist, review requirements, and documentation impact).", - "properties": { - "status": { + "spec_alignment": { + "type": "object", + "additionalProperties": false, + "properties": { + "requirements_summary": { + "type": "array", + "description": "Thematic grouping of requirements.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "theme": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "spec_refs": { + "type": "array", + "items": { + "$ref": "#/$defs/specRef" + } + } + }, + "required": [ + "theme", + "summary" + ] + } + }, + "checklist": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "$ref": "https://specdev.local/schema/core/atoms/1#screamingSnakeId" + }, + "spec_ref": { + "$ref": "#/$defs/specRef" + }, + "description": { + "type": "string" + }, + "type": { "type": "string", "enum": [ - "active", - "deferred" + "behavior", + "constraint", + "validation", + "metadata", + "perf", + "logging", + "docs", + "security" ] - }, - "deferred_reason": { - "type": "string" - }, - "summary": { - "type": "object", - "additionalProperties": false, - "properties": { - "functional_summary": { - "type": "string" - }, - "scope_in": { - "type": "array", - "items": { - "type": "string" - } - }, - "scope_out": { - "type": "array", - "items": { - "type": "string" - } - }, - "target_file_patterns": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Explicit list of files/directories to modify or create." + }, + "layer": { + "type": "string", + "enum": [ + "db", + "model", + "service", + "api", + "integration", + "tests", + "docs", + "config", + "security" + ] + }, + "checklist_status": { + "type": "string", + "enum": [ + "active", + "deferred" + ], + "default": "active" + }, + "linked_test_expectation": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 } - }, - "required": [ - "functional_summary", - "scope_in", - "scope_out", - "target_file_patterns" + } ] - }, - "docs_impact": { + }, + "nfr_refs": { + "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" + }, + "fixture_ref": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "implementation": { "type": "object", + "description": "Atomic work definition for this specific requirement.", "additionalProperties": false, - "description": "Documentation impact assessment. Required when any non-doc file is modified.", "properties": { - "status": { - "type": "string", - "enum": [ - "required", - "not_required" - ] - }, - "rationale": { - "type": "string", - "minLength": 10 - }, - "docs_touched": { - "type": "array", - "items": { - "type": "string" - } + "status": { + "type": "string", + "enum": [ + "pending", + "in_progress", + "verified", + "deferred" + ] + }, + "files_touched": { + "type": "array", + "items": { + "type": "string" } - }, - "required": [ - "status", - "rationale" - ], - "allOf": [ - { - "if": { + }, + "actions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "file_create", + "file_edit", + "run_command", + "manual_verification" + ] + }, + "description": { + "type": "string" + }, + "target": { + "type": "string" + }, + "command": { + "type": "string" + }, + "evidence": { + "$ref": "#/$defs/evidenceObject" + } + }, + "required": [ + "type", + "description" + ], + "allOf": [ + { + "if": { + "properties": { + "type": { + "enum": [ + "file_create", + "file_edit" + ] + } + } + }, + "then": { + "required": [ + "target" + ], "properties": { - "status": { - "const": "required" - } + "target": { + "minLength": 1 + } } + } }, - "then": { + { + "if": { + "properties": { + "type": { + "const": "run_command" + } + } + }, + "then": { "required": [ - "docs_touched" + "command" ], "properties": { - "docs_touched": { - "minItems": 1 - } + "command": { + "minLength": 1 + } } + } } + ] } - ] - }, - "spec_alignment": { - "type": "object", - "additionalProperties": false, - "properties": { - "requirements_summary": { - "type": "array", - "description": "Thematic grouping of requirements.", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "theme": { - "type": "string" - }, - "summary": { - "type": "string" - }, - "spec_refs": { - "type": "array", - "items": { - "$ref": "#/$defs/specRef" - } - } - }, - "required": [ - "theme", - "summary" - ] + } + }, + "required": [ + "status", + "actions" + ], + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "verified" } + } }, - "checklist": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#screamingSnakeId" - }, - "spec_ref": { - "$ref": "#/$defs/specRef" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "behavior", - "constraint", - "validation", - "metadata", - "perf", - "logging", - "docs", - "security" - ] - }, - "layer": { - "type": "string", - "enum": [ - "db", - "model", - "service", - "api", - "integration", - "tests", - "docs", - "config", - "security" - ] - }, - "checklist_status": { - "type": "string", - "enum": [ - "active", - "deferred" - ], - "default": "active" - }, - "linked_test_expectation": { - "oneOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - } - ] - }, - "nfr_refs": { - "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" - }, - "fixture_ref": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "implementation": { - "type": "object", - "description": "Atomic work definition for this specific requirement.", - "additionalProperties": false, - "properties": { - "status": { - "type": "string", - "enum": [ - "pending", - "in_progress", - "verified", - "deferred" - ] - }, - "files_touched": { - "type": "array", - "items": { - "type": "string" - } - }, - "actions": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string", - "enum": [ - "file_create", - "file_edit", - "run_command", - "manual_verification" - ] - }, - "description": { - "type": "string" - }, - "target": { - "type": "string" - }, - "command": { - "type": "string" - }, - "evidence": { - "$ref": "#/$defs/evidenceObject" - } - }, - "required": [ - "type", - "description" - ], - "allOf": [ - { - "if": { - "properties": { - "type": { - "enum": [ - "file_create", - "file_edit" - ] - } - } - }, - "then": { - "required": [ - "target" - ], - "properties": { - "target": { - "minLength": 1 - } - } - } - }, - { - "if": { - "properties": { - "type": { - "const": "run_command" - } - } - }, - "then": { - "required": [ - "command" - ], - "properties": { - "command": { - "minLength": 1 - } - } - } - } - ] - } - } - }, - "required": [ - "status", - "actions" - ], - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "verified" - } - } - }, - "then": { - "properties": { - "actions": { - "items": { - "required": [ - "evidence" - ] - } - } - } - } - } - ] - } - }, + "then": { + "properties": { + "actions": { + "items": { "required": [ - "id", - "spec_ref", - "description", - "linked_test_expectation" - ], - "allOf": [ - { - "if": { - "not": { - "properties": { - "checklist_status": { - "const": "deferred" - } - } - } - }, - "then": { - "required": [ - "implementation" - ] - } - } + "evidence" ] + } } + } } - }, - "required": [ - "checklist" + } ] + } }, - "ambiguities": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, + "required": [ + "id", + "spec_ref", + "description", + "linked_test_expectation" + ], + "allOf": [ + { + "if": { + "not": { "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "description": { - "type": "string" - }, - "source": { - "type": "string", - "enum": [ - "spec", - "code", - "plan", - "mixed", - "review" - ] - }, - "severity": { - "type": "string", - "enum": [ - "blocking", - "non_blocking" - ] - }, - "impact": { - "type": "array", - "items": { - "type": "string" - } - }, - "proposed_assumption": { - "type": "string" - }, - "mitigation": { - "type": "string", - "minLength": 10 - }, - "status": { - "type": "string", - "enum": [ - "resolved", - "tracking", - "deferred", - "blocked" - ] - }, - "decision": { - "type": "string" - }, - "resolved": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "boolean" - } - ] - } - }, - "required": [ - "id", - "description", - "severity" - ], - "allOf": [ - { - "if": { - "properties": { - "severity": { - "const": "non_blocking" - } - } - }, - "then": { - "required": [ - "mitigation" - ] - } - } - ] + "checklist_status": { + "const": "deferred" + } + } + } + }, + "then": { + "required": [ + "implementation" + ] + } + } + ] + } + } + }, + "required": [ + "checklist" + ] + }, + "ambiguities": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "description": { + "type": "string" + }, + "source": { + "type": "string", + "enum": [ + "spec", + "code", + "plan", + "mixed", + "review" + ] + }, + "severity": { + "type": "string", + "enum": [ + "blocking", + "non_blocking" + ] + }, + "impact": { + "type": "array", + "items": { + "type": "string" + } + }, + "proposed_assumption": { + "type": "string" + }, + "mitigation": { + "type": "string", + "minLength": 10 + }, + "status": { + "type": "string", + "enum": [ + "resolved", + "tracking", + "deferred", + "blocked" + ] + }, + "decision": { + "type": "string" + }, + "resolved": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + } + }, + "required": [ + "id", + "description", + "severity" + ], + "allOf": [ + { + "if": { + "properties": { + "severity": { + "const": "non_blocking" } + } }, - "solution": { + "then": { + "required": [ + "mitigation" + ] + } + } + ] + } + }, + "solution": { + "type": "object", + "additionalProperties": false, + "properties": { + "architecture_sketch": { + "type": "string" + }, + "sequence_of_concerns": { + "type": "array", + "items": { + "type": "string" + } + }, + "risks": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "architecture_sketch" + ] + }, + "context": { + "type": "object", + "additionalProperties": false, + "properties": { + "existing_structures": { + "type": "array", + "description": "Known code or non-code structures. Strings may reference non-code artifacts; objects must cite real code signatures.", + "items": { + "oneOf": [ + { + "type": "string" + }, + { "type": "object", "additionalProperties": false, "properties": { - "architecture_sketch": { - "type": "string" - }, - "sequence_of_concerns": { - "type": "array", - "items": { - "type": "string" - } - }, - "risks": { - "type": "array", - "items": { - "type": "string" - } - } + "signature": { + "type": "string" + }, + "source_file": { + "type": "string", + "pattern": "^[^/].*\\.(py|ts|js|go|rs)$" + }, + "line_range": { + "type": "string", + "pattern": "^L\\d+-L\\d+$" + } }, "required": [ - "architecture_sketch" + "signature", + "source_file" ] + } + ] + } + }, + "coding_examples": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "code": { + "type": "string" + } }, - "context": { - "type": "object", - "additionalProperties": false, - "properties": { - "existing_structures": { - "type": "array", - "description": "Known code or non-code structures. Strings may reference non-code artifacts; objects must cite real code signatures.", - "items": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "signature": { - "type": "string" - }, - "source_file": { - "type": "string", - "pattern": "^[^/].*\\.(py|ts|js|go|rs)$" - }, - "line_range": { - "type": "string", - "pattern": "^L\\d+-L\\d+$" - } - }, - "required": [ - "signature", - "source_file" - ] - } - ] - } - }, - "coding_examples": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "title", - "code" - ] - } - } - } - }, - "review_requirements": { + "required": [ + "title", + "code" + ] + } + } + } + }, + "review_requirements": { + "type": "object", + "additionalProperties": false, + "properties": { + "guidelines": { + "type": "string" + }, + "test_commands": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { "type": "object", "additionalProperties": false, "properties": { - "guidelines": { - "type": "string" - }, - "test_commands": { - "type": "array", - "items": { - "oneOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "command": { - "type": "string", - "minLength": 1 - }, - "expected_exit_code": { - "type": "integer", - "default": 0 - }, - "timeout_seconds": { - "type": "integer", - "minimum": 1, - "maximum": 3600 - }, - "description": { - "type": "string" - } - }, - "required": [ - "command" - ] - } - ] - } - }, - "nfr_measurement_methods": { - "type": "object", - "additionalProperties": false, - "properties": { - "nfr-availability-uptime": { - "type": "object", - "additionalProperties": false, - "properties": { - "command": { - "type": "string" - }, - "expected": { - "type": "string" - }, - "description": { - "type": "string" - } - } - }, - "nfr-privacy-cookie-free": { - "type": "object", - "additionalProperties": false, - "properties": { - "command": { - "type": "string" - }, - "expected": { - "type": "string" - }, - "description": { - "type": "string" - } - } - } - } - }, - "timeout_constants": { - "type": "object", - "additionalProperties": false, - "properties": { - "EMAIL_DELIVERY_TIMEOUT": { - "type": "integer" - }, - "DNS_VERIFICATION_TIMEOUT": { - "type": "integer" - }, - "ANALYTICS_BEACON_TIMEOUT": { - "type": "integer" - } - } - } + "command": { + "type": "string", + "minLength": 1 + }, + "expected_exit_code": { + "type": "integer", + "default": 0 + }, + "timeout_seconds": { + "type": "integer", + "minimum": 1, + "maximum": 3600 + }, + "description": { + "type": "string" + } }, "required": [ - "test_commands" - ] - }, - "docs": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "required_updates": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "path": { - "type": "string" - }, - "update_summary": { - "type": "string" - } - }, - "required": [ - "path", - "update_summary" - ] - } - } - }, - "required": [ - "status", - "required_updates" - ] - } + "command" ] + } + ] + } + }, + "nfr_measurement_methods": { + "type": "object", + "description": "Per-NFR measurement strategy keyed by NFR ID.", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "methodology": { + "type": "string" + }, + "frequency": { + "type": "string" + }, + "thresholds": { + "type": "string" + }, + "command": { + "type": "string" + }, + "expected": { + "type": "string" + }, + "description": { + "type": "string" + } }, - "security": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "new_fixtures": { - "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" - }, - "spec_mutations": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "ref": { - "$ref": "https://specdev.local/schema/core/collections/1#traceRef" - }, - "change": { - "type": "string" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "ref", - "change", - "reason" - ] - } - } - }, - "required": [ - "status" - ] - } - ] + "required": [ + "methodology", + "frequency", + "thresholds" + ] + } + }, + "timeout_constants": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_timeout": { + "type": "integer", + "minimum": 1 }, - "delivery": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "dashboards": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "dashboard_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "nfr_refs": { - "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" - }, - "url": { - "type": "string" - } - }, - "required": [ - "dashboard_id" - ] - } - }, - "alerts": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "alert_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "nfr_ref": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "rule": { - "type": "string" - }, - "severity": { - "$ref": "#/$defs/severityLevel" - } - }, - "required": [ - "alert_id", - "rule" - ] - } - } - }, - "required": [ - "status" - ] - } - ] + "max_timeout": { + "type": "integer", + "minimum": 1 }, - "drift": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "not_applicable" - }, - "reason": { - "type": "string" - } - }, - "required": [ - "status", - "reason" - ] - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "const": "planned" - }, - "checks": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "check_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "target": { - "type": "string", - "enum": [ - "api", - "schema", - "nfr", - "invariant", - "fixture", - "config" - ] - }, - "method": { - "type": "string", - "enum": [ - "runtime-sample", - "log-diff", - "schema-diff", - "trace-replay" - ] - }, - "schedule": { - "type": "string", - "pattern": "^(hourly|daily|weekly|monthly|@(annually|monthly|weekly|daily|hourly)|([0-9*/,-]+ ){4}[0-9*/,-]+)$", - "description": "Named interval (hourly/daily/weekly/monthly) or cron expression" - }, - "severity": { - "$ref": "#/$defs/severityLevel" - }, - "remediation_policy": { - "type": "string" - } - }, - "required": [ - "check_id", - "target", - "method" - ] - } - } - }, - "required": [ - "status" - ] - } + "per_operation": { + "type": "object", + "additionalProperties": { + "type": "integer", + "minimum": 1 + } + } + }, + "required": [ + "default_timeout", + "max_timeout" + ], + "allOf": [ + { + "if": { + "properties": { + "max_timeout": { + "type": "integer" + }, + "default_timeout": { + "type": "integer" + } + }, + "required": [ + "max_timeout", + "default_timeout" ] + }, + "then": { + "properties": { + "max_timeout": { + "minimum": 1 + } + } + } + } + ] + } + }, + "required": [ + "test_commands" + ] + }, + "docs": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "not_applicable" }, - "coverage_status": { + "reason": { + "type": "string" + } + }, + "required": [ + "status", + "reason" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "planned" + }, + "required_updates": { + "type": "array", + "items": { "type": "object", "additionalProperties": false, "properties": { - "total": { - "type": "integer", - "minimum": 0 - }, - "verified": { - "type": "integer", - "minimum": 0 - }, - "deferred": { - "type": "integer", - "minimum": 0 - }, - "pending": { - "type": "integer", - "minimum": 0 - } + "path": { + "type": "string" + }, + "update_summary": { + "type": "string" + } }, "required": [ - "total", - "verified", - "deferred", - "pending" + "path", + "update_summary" ] + } + } + }, + "required": [ + "status", + "required_updates" + ] + } + ] + }, + "security": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "not_applicable" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "status", + "reason" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "planned" + }, + "new_fixtures": { + "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" }, - "scope_validation": { + "spec_mutations": { + "type": "array", + "items": { "type": "object", "additionalProperties": false, "properties": { - "in_scope": { - "type": "array", - "items": { - "type": "string" - } - }, - "out_of_scope": { - "type": "array", - "items": { - "type": "string" - } - }, - "acknowledged": { - "type": "boolean" - } + "ref": { + "$ref": "https://specdev.local/schema/core/collections/1#traceRef" + }, + "change": { + "type": "string" + }, + "reason": { + "type": "string" + } }, - "allOf": [ - { - "if": { - "properties": { - "out_of_scope": { - "minItems": 1 - } - } - }, - "then": { - "required": [ - "acknowledged" - ], - "properties": { - "acknowledged": { - "const": true - } - } - } - } + "required": [ + "ref", + "change", + "reason" ] + } } - }, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "deferred" - } - } - }, - "then": { - "required": [ - "deferred_reason" - ], - "properties": { - "summary": { - "properties": { - "target_file_patterns": { - "maxItems": 0 - } - } - }, - "review_requirements": { - "properties": { - "test_commands": { - "maxItems": 0 - } - } - } - } - }, - "else": { - "properties": { - "summary": { - "properties": { - "target_file_patterns": { - "minItems": 1 - } - } - }, - "review_requirements": { - "properties": { - "test_commands": { - "minItems": 1 - } - } - } - } - } - } - ] + }, + "required": [ + "status" + ] + } + ] }, - "execution": { - "type": "object", - "description": "Global execution summary.", - "additionalProperties": false, - "properties": { - "files_touched": { - "type": "array", - "items": { - "type": "string" - } + "delivery": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "not_applicable" }, - "execution_results": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "status": { - "$ref": "#/$defs/executionStatus" - }, - "outcome_description": { - "type": "string" - }, - "reasoning": { - "type": "string" - }, - "command": { - "type": "string" - }, - "evidence": { - "type": "string", - "minLength": 20 - }, - "evidence_ref": { - "type": "string" - }, - "evidence_binding": { - "type": "object", - "additionalProperties": false, - "properties": { - "timestamp": { - "type": "string", - "format": "date-time" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "exit_code": { - "type": "integer", - "minimum": 0, - "maximum": 255 - }, - "command": { - "type": "string" - } - }, - "required": [ - "timestamp", - "sha256", - "exit_code" - ] - } - }, - "required": [ - "status", - "outcome_description", - "reasoning", - "command", - "evidence" - ], - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "passed" - } - } - }, - "then": { - "required": [ - "evidence_ref", - "evidence_binding" - ], - "properties": { - "evidence": { - "pattern": "(PASSED|passed|OK|SUCCESS|✓|0 (errors|failures?|failed)|\\d+ passed)" - } - } - } - } - ] - } + "reason": { + "type": "string" + } + }, + "required": [ + "status", + "reason" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "planned" }, - "critical_evidence": { + "dashboards": { + "type": "array", + "items": { "type": "object", "additionalProperties": false, "properties": { - "satisfied_checklist_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "passed_test_commands": { - "type": "array", - "items": { - "type": "string" - } - } - } + "dashboard_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "nfr_refs": { + "$ref": "https://specdev.local/schema/core/collections/1#kebabIdArray" + }, + "url": { + "type": "string" + } + }, + "required": [ + "dashboard_id" + ] + } }, - "config_validation": { + "alerts": { + "type": "array", + "items": { "type": "object", "additionalProperties": false, "properties": { - "dashboard_links_valid": { - "type": "boolean" - }, - "alert_rules_valid": { - "type": "boolean" - }, - "drift_schedules_valid": { - "type": "boolean" - }, - "notes": { - "type": "string" - } - } + "alert_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "nfr_ref": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "rule": { + "type": "string" + }, + "severity": { + "$ref": "#/$defs/severityLevel" + } + }, + "required": [ + "alert_id", + "rule" + ] + } + } + }, + "required": [ + "status" + ] + } + ] + }, + "drift": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "not_applicable" }, - "emergent_ambiguities": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "description": { - "type": "string" - }, - "severity": { - "type": "string" - }, - "impact": { - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "type": "string" - } - }, - "required": [ - "id", - "description", - "severity" - ] - } + "reason": { + "type": "string" + } + }, + "required": [ + "status", + "reason" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "const": "planned" }, - "final_status": { + "checks": { + "type": "array", + "items": { "type": "object", "additionalProperties": false, "properties": { - "test_results": { - "type": "array", - "items": { - "type": "object" - } - }, - "ci_status": { - "type": "string", - "enum": [ - "green", - "red" - ] - } + "check_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "target": { + "type": "string", + "enum": [ + "api", + "schema", + "nfr", + "invariant", + "fixture", + "config" + ] + }, + "method": { + "type": "string", + "enum": [ + "runtime-sample", + "log-diff", + "schema-diff", + "trace-replay" + ] + }, + "schedule": { + "type": "string", + "pattern": "^(hourly|daily|weekly|monthly|@(annually|monthly|weekly|daily|hourly)|([0-9*/,-]+ ){4}[0-9*/,-]+)$", + "description": "Named interval (hourly/daily/weekly/monthly) or cron expression" + }, + "severity": { + "$ref": "#/$defs/severityLevel" + }, + "remediation_policy": { + "type": "string" + } + }, + "required": [ + "check_id", + "target", + "method" + ] + } + } + }, + "required": [ + "status" + ] + } + ] + }, + "coverage_status": { + "type": "object", + "additionalProperties": false, + "properties": { + "total": { + "type": "integer", + "minimum": 0 + }, + "verified": { + "type": "integer", + "minimum": 0 + }, + "deferred": { + "type": "integer", + "minimum": 0 + }, + "pending": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "total", + "verified", + "deferred", + "pending" + ] + }, + "scope_validation": { + "type": "object", + "additionalProperties": false, + "properties": { + "in_scope": { + "type": "array", + "items": { + "type": "string" + } + }, + "out_of_scope": { + "type": "array", + "items": { + "type": "string" + } + }, + "acknowledged": { + "type": "boolean" + } + }, + "allOf": [ + { + "if": { + "properties": { + "out_of_scope": { + "minItems": 1 + } + } + }, + "then": { + "required": [ + "acknowledged" + ], + "properties": { + "acknowledged": { + "const": true + } + } + } + } + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "deferred" + } + } + }, + "then": { + "required": [ + "deferred_reason" + ], + "properties": { + "summary": { + "properties": { + "target_file_patterns": { + "maxItems": 0 + } + } + }, + "review_requirements": { + "properties": { + "test_commands": { + "maxItems": 0 + } + } + } + } + }, + "else": { + "properties": { + "summary": { + "properties": { + "target_file_patterns": { + "minItems": 1 + } + } + }, + "review_requirements": { + "properties": { + "test_commands": { + "minItems": 1 + } + } + } + } + } + } + ], + "required": [ + "summary", + "docs_impact", + "spec_alignment", + "review_requirements" + ] + }, + "execution": { + "type": "object", + "description": "Global execution summary.", + "additionalProperties": false, + "properties": { + "files_touched": { + "type": "array", + "items": { + "type": "string" + } + }, + "execution_results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "$ref": "#/$defs/executionStatus" + }, + "outcome_description": { + "type": "string" + }, + "reasoning": { + "type": "string" + }, + "command": { + "type": "string" + }, + "evidence": { + "type": "string", + "minLength": 20 + }, + "evidence_ref": { + "type": "string" + }, + "evidence_binding": { + "type": "object", + "additionalProperties": false, + "properties": { + "timestamp": { + "type": "string", + "format": "date-time" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "exit_code": { + "type": "integer", + "minimum": 0, + "maximum": 255 + }, + "command": { + "type": "string" + } + }, + "required": [ + "timestamp", + "sha256", + "exit_code" + ] + } + }, + "required": [ + "status", + "outcome_description", + "reasoning", + "command", + "evidence" + ], + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "passed" } + } + }, + "then": { + "required": [ + "evidence_ref", + "evidence_binding" + ], + "properties": { + "evidence": { + "pattern": "(PASSED|passed|OK|SUCCESS|\u2713|0 (errors|failures?|failed)|\\d+ passed)" + } + } } + } + ] + } + }, + "critical_evidence": { + "type": "object", + "additionalProperties": false, + "properties": { + "satisfied_checklist_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "passed_test_commands": { + "type": "array", + "items": { + "type": "string" + } } + } }, - "review": { + "config_validation": { + "type": "object", + "additionalProperties": false, + "properties": { + "dashboard_links_valid": { + "type": "boolean" + }, + "alert_rules_valid": { + "type": "boolean" + }, + "drift_schedules_valid": { + "type": "boolean" + }, + "notes": { + "type": "string" + } + } + }, + "emergent_ambiguities": { + "type": "array", + "items": { "type": "object", "additionalProperties": false, "properties": { - "findings": { + "id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "description": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "impact": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "type": "string" + } + }, + "required": [ + "id", + "description", + "severity" + ] + } + }, + "final_status": { + "type": "object", + "additionalProperties": false, + "properties": { + "test_results": { + "type": "array", + "items": { + "type": "object" + } + }, + "ci_status": { + "type": "string", + "enum": [ + "green", + "red" + ] + } + } + } + } + }, + "review": { + "type": "object", + "additionalProperties": false, + "properties": { + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "type": { + "type": "string", + "enum": [ + "bug", + "gap", + "scope_creep", + "style", + "design", + "tests", + "docs" + ] + }, + "severity": { + "type": "string", + "enum": [ + "blocking", + "major", + "minor", + "nit" + ] + }, + "spec_ref": { + "$ref": "#/$defs/specRef" + }, + "description": { + "type": "string" + }, + "related_checklist_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "remediation_task": { + "type": "object", + "additionalProperties": false, + "properties": { + "task_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "summary": { + "type": "string" + }, + "files_to_touch": { "type": "array", "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "type": { - "type": "string", - "enum": [ - "bug", - "gap", - "scope_creep", - "style", - "design", - "tests", - "docs" - ] - }, - "severity": { - "type": "string", - "enum": [ - "blocking", - "major", - "minor", - "nit" - ] - }, - "spec_ref": { - "$ref": "#/$defs/specRef" - }, - "description": { - "type": "string" - }, - "related_checklist_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "remediation_task": { - "type": "object", - "additionalProperties": false, - "properties": { - "task_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "summary": { - "type": "string" - }, - "files_to_touch": { - "type": "array", - "items": { - "type": "string" - } - }, - "checklist_ids": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "task_id", - "summary", - "files_to_touch", - "checklist_ids" - ] - }, - "metadata": { - "type": "object", - "additionalProperties": false, - "properties": { - "source": { - "type": "string" - }, - "impact": { - "type": "string" - } - }, - "required": [ - "source", - "impact" - ] - } - }, - "required": [ - "id", - "type", - "severity", - "spec_ref", - "description", - "metadata" - ], - "allOf": [ - { - "if": { - "properties": { - "severity": { - "enum": [ - "blocking", - "major" - ] - } - } - }, - "then": { - "required": [ - "remediation_task" - ] - } - } - ] + "type": "string" } + }, + "checklist_ids": { + "type": "array", + "items": { + "type": "string" + } + } }, - "ratings": { - "type": "object", - "additionalProperties": false, - "properties": { - "spec_completeness": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "code_quality": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "tests_completeness": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "docs_completeness": { - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - "metadata_usage": { - "type": "integer", - "minimum": 0, - "maximum": 5 - } - }, - "required": [ - "spec_completeness", - "code_quality", - "tests_completeness", - "docs_completeness", - "metadata_usage" - ] + "required": [ + "task_id", + "summary", + "files_to_touch", + "checklist_ids" + ] + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string" + }, + "impact": { + "type": "string" + } + }, + "required": [ + "source", + "impact" + ] + } + }, + "required": [ + "id", + "type", + "severity", + "spec_ref", + "description", + "metadata" + ], + "allOf": [ + { + "if": { + "properties": { + "severity": { + "enum": [ + "blocking", + "major" + ] + } + } }, - "verdict": { + "then": { + "required": [ + "remediation_task" + ] + } + } + ] + } + }, + "ratings": { + "type": "object", + "additionalProperties": false, + "properties": { + "spec_completeness": { + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + "code_quality": { + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + "tests_completeness": { + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + "docs_completeness": { + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + "metadata_usage": { + "type": "integer", + "minimum": 0, + "maximum": 5 + } + }, + "required": [ + "spec_completeness", + "code_quality", + "tests_completeness", + "docs_completeness", + "metadata_usage" + ] + }, + "verdict": { + "type": "string", + "enum": [ + "verified", + "deferred", + "rejected" + ] + }, + "next_actions": { + "type": "string" + }, + "fixture_status": { + "type": "object", + "additionalProperties": false, + "properties": { + "implemented_endpoints": { + "type": "array", + "items": { + "$ref": "https://specdev.local/schema/core/collections/1#traceId" + } + }, + "test_results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "fixture_ref": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "status": { "type": "string", "enum": [ - "verified", - "deferred", - "rejected" + "pass", + "fail", + "skip" ] - }, - "next_actions": { + }, + "notes": { "type": "string" + } }, - "fixture_status": { - "type": "object", - "additionalProperties": false, - "properties": { - "implemented_endpoints": { - "type": "array", - "items": { - "$ref": "https://specdev.local/schema/core/collections/1#traceId" - } - }, - "test_results": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "fixture_ref": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "status": { - "type": "string", - "enum": [ - "pass", - "fail", - "skip" - ] - }, - "notes": { - "type": "string" - } - }, - "required": [ - "fixture_ref", - "status" - ] - } - }, - "ci_status": { - "type": "string", - "enum": [ - "green", - "red" - ] - } - }, - "required": [ - "implemented_endpoints", - "test_results", - "ci_status" + "required": [ + "fixture_ref", + "status" + ] + } + }, + "ci_status": { + "type": "string", + "enum": [ + "green", + "red" + ] + } + }, + "required": [ + "implemented_endpoints", + "test_results", + "ci_status" + ] + }, + "security_status": { + "type": "string", + "enum": [ + "green", + "red" + ] + }, + "delivery_status": { + "type": "object", + "additionalProperties": false, + "properties": { + "deployments": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "env": { + "type": "string", + "enum": [ + "dev", + "staging", + "prod" ] - }, - "security_status": { + }, + "build_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "status": { "type": "string", "enum": [ - "green", - "red" + "pending", + "success", + "failed" ] + } }, - "delivery_status": { - "type": "object", - "additionalProperties": false, + "required": [ + "env", + "build_id" + ] + } + }, + "dashboards_verified": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "dashboard_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "url": { + "type": "string" + }, + "evidence_ref": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "dashboard_id", + "url", + "evidence_ref" + ] + } + }, + "alerts_verified": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "alert_id": { + "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" + }, + "rule": { + "type": "string" + }, + "severity": { + "$ref": "#/$defs/severityLevel" + }, + "evidence_ref": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "alert_id", + "rule", + "severity", + "evidence_ref" + ] + } + } + } + } + }, + "allOf": [ + { + "if": { + "required": [ + "verdict" + ], + "properties": { + "verdict": { + "const": "verified" + } + } + }, + "then": { + "required": [ + "fixture_status" + ], + "properties": { + "fixture_status": { + "properties": { + "ci_status": { + "const": "green" + } + }, + "required": [ + "ci_status" + ] + } + } + } + } + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "plan": { + "required": [ + "delivery" + ], + "properties": { + "delivery": { + "required": [ + "status" + ], + "properties": { + "status": { + "const": "planned" + } + } + } + } + } + } + }, + "then": { + "required": [ + "review" + ], + "properties": { + "review": { + "required": [ + "delivery_status" + ], + "properties": { + "delivery_status": { + "anyOf": [ + { + "required": [ + "deployments" + ], "properties": { - "deployments": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "env": { - "type": "string", - "enum": [ - "dev", - "staging", - "prod" - ] - }, - "build_id": { - "$ref": "https://specdev.local/schema/core/atoms/1#kebabId" - }, - "status": { - "type": "string", - "enum": [ - "pending", - "success", - "failed" - ] - } - }, - "required": [ - "env", - "build_id" - ] - } - } + "deployments": { + "minItems": 1 + } } - } - }, - "allOf": [ - { - "if": { - "required": [ - "verdict" - ], - "properties": { - "verdict": { - "const": "verified" - } - } - }, - "then": { - "required": [ - "fixture_status" - ], - "properties": { - "fixture_status": { - "properties": { - "ci_status": { - "const": "green" - } - }, - "required": [ - "ci_status" - ] - } - } + }, + { + "required": [ + "dashboards_verified" + ], + "properties": { + "dashboards_verified": { + "minItems": 1 + } } - } - ] + }, + { + "required": [ + "alerts_verified" + ], + "properties": { + "alerts_verified": { + "minItems": 1 + } + } + } + ] + } + } + } } - }, - "required": [ - "id", - "owner", - "created_at", - "seed_refs", - "plan" - ] + } + } + ], + "required": [ + "id", + "owner", + "created_at", + "seed_refs", + "plan" + ] } diff --git a/schema/trinity/context_pack.schema.json b/schema/trinity/context_pack.schema.json new file mode 100644 index 00000000..f58a998d --- /dev/null +++ b/schema/trinity/context_pack.schema.json @@ -0,0 +1,314 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/context_pack.schema.json", + "title": "Trinity Context Pack", + "type": "object", + "additionalProperties": false, + "$defs": { + "specRef": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "fr", + "api", + "nfr", + "inv", + "fixture" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "line_range": { + "type": "string", + "pattern": "^L\\d+-L\\d+$" + }, + "commit_hash": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "type", + "id", + "path", + "line_range", + "commit_hash" + ] + }, + "bootstrapRefTrace": { + "type": "object", + "additionalProperties": false, + "properties": { + "spec_type": { + "type": "string", + "enum": [ + "fr", + "api", + "nfr", + "inv", + "fixture" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "selected_from": { + "type": "string", + "minLength": 1 + }, + "selection_mode": { + "type": "string", + "enum": [ + "structured", + "tokenized", + "authority_fallback" + ] + }, + "path": { + "type": "string", + "minLength": 1 + }, + "line_range": { + "type": "string", + "pattern": "^L\\d+-L\\d+$" + } + }, + "required": [ + "spec_type", + "id", + "selected_from", + "selection_mode", + "path", + "line_range" + ] + } + }, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "type": "string", + "minLength": 1, + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + }, + "seed_manifest_path": { + "type": "string", + "minLength": 1 + }, + "seed_files_ordered": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "required_spec_refs": { + "type": "array", + "items": { + "$ref": "#/$defs/specRef" + } + }, + "bootstrap_ref_trace": { + "type": "array", + "items": { + "$ref": "#/$defs/bootstrapRefTrace" + } + }, + "artifact_refs": { + "type": "object", + "additionalProperties": false, + "properties": { + "milestone_context_path": { + "type": "string", + "minLength": 1 + }, + "anchor_path": { + "type": "string", + "minLength": 1 + }, + "workspace_refs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "milestone_context_path" + ] + }, + "allowed_read_paths": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "allowed_write_paths": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "target_file_patterns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "docs_policy": { + "type": "object", + "additionalProperties": false, + "properties": { + "doc_paths": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "readme_required": { + "type": "boolean" + }, + "root_readme_required": { + "type": "boolean" + } + }, + "required": [ + "doc_paths" + ] + }, + "test_contract": { + "type": "object", + "additionalProperties": false, + "properties": { + "test_commands": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "expected_exit_code": { + "type": "integer" + }, + "timeout_seconds": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "command" + ] + } + ] + } + }, + "success_markers": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "test_commands", + "success_markers" + ] + } + }, + "required": [ + "protocol_version", + "phase", + "step_id", + "seed_manifest_path", + "seed_files_ordered", + "required_spec_refs", + "artifact_refs", + "allowed_read_paths", + "allowed_write_paths" + ], + "allOf": [ + { + "if": { + "properties": { + "phase": { + "enum": [ + "16a", + "16b", + "16c" + ] + } + } + }, + "then": { + "required": [ + "target_file_patterns", + "docs_policy" + ], + "properties": { + "required_spec_refs": { + "minItems": 1 + } + } + } + }, + { + "if": { + "properties": { + "phase": { + "enum": [ + "16b", + "16c" + ] + } + } + }, + "then": { + "required": [ + "test_contract" + ] + } + } + ] +} diff --git a/schema/trinity/eval_export_row.schema.json b/schema/trinity/eval_export_row.schema.json new file mode 100644 index 00000000..58be404a --- /dev/null +++ b/schema/trinity/eval_export_row.schema.json @@ -0,0 +1,306 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/eval_export_row.schema.json", + "title": "Trinity Eval Export Row", + "type": "object", + "additionalProperties": false, + "properties": { + "run_id": { + "type": "string", + "minLength": 1 + }, + "event_id": { + "type": "string", + "minLength": 1 + }, + "event_sequence": { + "type": "integer", + "minimum": 1 + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "event_type": { + "type": "string", + "enum": [ + "SPAWN", + "MESSAGE", + "TOOL_CALL", + "TOOL_RESULT", + "VALIDATION", + "TERMINATE", + "ERROR" + ] + }, + "role": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Researcher", + "Auditor", + "Summarizer", + "ToolUser" + ] + }, + "phase_id": { + "type": "string" + }, + "step_id": { + "type": [ + "string", + "null" + ] + }, + "capture_level": { + "type": "string", + "enum": [ + "none", + "summary", + "full" + ] + }, + "prompt_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "prompt_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-f0-9]{64}$" + }, + "response_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "response_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-f0-9]{64}$" + }, + "artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "artifact_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^(sha256:)?[a-f0-9]{64}$" + }, + "diff_ref": { + "type": [ + "string", + "null" + ] + }, + "task_result_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "tool_name": { + "type": [ + "string", + "null" + ], + "enum": [ + "read_file", + "write_file", + "edit_file", + "apply_patch", + "move_file", + "remove_file", + "list_dir", + "glob_match", + "search_text", + "git_head", + "git_show", + "git_diff", + "exec_cmd", + "validate_json", + "checkpoint_branch", + "checkpoint_commit", + null + ] + }, + "tool_command": { + "type": [ + "string", + "null" + ] + }, + "tool_exit_code": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 255 + }, + "validation_schema": { + "type": [ + "string", + "null" + ], + "enum": [ + "pass", + "fail", + "n/a", + null + ] + }, + "validation_deep_validator": { + "type": [ + "string", + "null" + ], + "enum": [ + "pass", + "fail", + "n/a", + null + ] + }, + "validation_governance": { + "type": [ + "string", + "null" + ], + "enum": [ + "pass", + "fail", + "n/a", + null + ] + }, + "phase_outcome": { + "type": [ + "string", + "null" + ], + "enum": [ + "success", + "blocked", + "failed", + "questions", + null + ] + }, + "review_verdict": { + "type": [ + "string", + "null" + ], + "enum": [ + "verified", + "deferred", + "rejected", + null + ] + }, + "checklist_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string", + "minLength": 1 + } + }, + "finding_count": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "max_finding_severity": { + "type": [ + "string", + "null" + ], + "enum": [ + "blocking", + "major", + "minor", + "nit", + null + ] + }, + "remediation_required": { + "type": [ + "boolean", + "null" + ] + }, + "redaction_applied": { + "type": "boolean" + }, + "redaction_total_replacements": { + "type": "integer", + "minimum": 0 + }, + "redaction_classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "token_prompt": { + "type": "integer", + "minimum": 0 + }, + "token_completion": { + "type": "integer", + "minimum": 0 + }, + "token_total": { + "type": "integer", + "minimum": 0 + }, + "event_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "prev_event_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "run_id", + "event_id", + "event_sequence", + "timestamp", + "event_type", + "role", + "capture_level", + "redaction_applied", + "redaction_total_replacements", + "token_prompt", + "token_completion", + "token_total", + "event_sha256" + ] +} diff --git a/schema/trinity/log_capture_policy.schema.json b/schema/trinity/log_capture_policy.schema.json new file mode 100644 index 00000000..f6ab7435 --- /dev/null +++ b/schema/trinity/log_capture_policy.schema.json @@ -0,0 +1,236 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/log_capture_policy.schema.json", + "title": "Trinity Log Capture Policy", + "type": "object", + "additionalProperties": false, + "properties": { + "policy_id": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "default_capture_level": { + "type": "string", + "enum": [ + "none", + "summary", + "full" + ] + }, + "always_full_on_event_types": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "SPAWN", + "MESSAGE", + "TOOL_CALL", + "TOOL_RESULT", + "VALIDATION", + "TERMINATE", + "ERROR" + ] + } + }, + "sample_rate_by_event_type": { + "type": "object", + "additionalProperties": false, + "properties": { + "SPAWN": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "MESSAGE": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "TOOL_CALL": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "TOOL_RESULT": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "VALIDATION": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "TERMINATE": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "ERROR": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "max_full_events_per_run": { + "type": "integer", + "minimum": 0 + }, + "context_window_token_target": { + "type": "integer", + "minimum": 1024 + }, + "max_full_capture_context_fraction": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "full_capture_token_budget_per_run": { + "type": "integer", + "minimum": 0 + }, + "max_full_prompt_tokens_per_event": { + "type": "integer", + "minimum": 0 + }, + "max_full_completion_tokens_per_event": { + "type": "integer", + "minimum": 0 + }, + "oversize_fallback": { + "type": "string", + "enum": [ + "summary", + "none" + ] + }, + "full_capture_allowlist_roles": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Researcher", + "Auditor", + "Summarizer", + "ToolUser" + ] + } + }, + "require_redaction_before_full": { + "type": "boolean" + }, + "sampling_salt": { + "type": "string", + "default": "default" + }, + "operating_profile": { + "type": "object", + "additionalProperties": false, + "properties": { + "profile": { + "type": "string", + "enum": [ + "eval_default", + "eval_extended", + "cost_guarded" + ] + }, + "tier": { + "type": "string", + "enum": [ + "baseline", + "balanced", + "comprehensive" + ] + }, + "budget_tier": { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + }, + "required": [ + "profile", + "tier", + "budget_tier" + ] + }, + "budgets": { + "type": "object", + "additionalProperties": false, + "properties": { + "context_window_token_target": { + "type": "integer", + "minimum": 1024 + }, + "full_capture_token_budget_per_run": { + "type": "integer", + "minimum": 0 + }, + "max_full_prompt_tokens_per_event": { + "type": "integer", + "minimum": 0 + }, + "max_full_completion_tokens_per_event": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "context_window_token_target", + "full_capture_token_budget_per_run", + "max_full_prompt_tokens_per_event", + "max_full_completion_tokens_per_event" + ] + }, + "retention": { + "type": "object", + "additionalProperties": false, + "properties": { + "session_log_days": { + "type": "integer", + "minimum": 1 + }, + "capture_artifact_days": { + "type": "integer", + "minimum": 1 + }, + "eval_export_days": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "session_log_days", + "capture_artifact_days", + "eval_export_days" + ] + } + }, + "required": [ + "policy_id", + "version", + "default_capture_level", + "always_full_on_event_types", + "sample_rate_by_event_type", + "max_full_events_per_run", + "oversize_fallback", + "full_capture_allowlist_roles", + "require_redaction_before_full" + ] +} diff --git a/schema/trinity/scratchpad_state.schema.json b/schema/trinity/scratchpad_state.schema.json new file mode 100644 index 00000000..a257209c --- /dev/null +++ b/schema/trinity/scratchpad_state.schema.json @@ -0,0 +1,138 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/scratchpad_state.schema.json", + "title": "Trinity Scratchpad State", + "type": "object", + "additionalProperties": false, + "properties": { + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "checklist_scope": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "last_validation_gate": { + "type": "object", + "additionalProperties": false, + "properties": { + "schema": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + }, + "deep_validator": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + }, + "governance": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + } + }, + "required": [ + "schema", + "deep_validator", + "governance" + ] + }, + "next_action_ref": { + "type": "string", + "minLength": 1 + }, + "state_summary": { + "type": "string", + "minLength": 1 + }, + "variables": { + "type": "object", + "additionalProperties": true + }, + "retry_count": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Number of retries attempted for the current phase action" + }, + "error_context": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "properties": { + "last_error_type": { + "type": "string", + "description": "Category of last error (schema_fail, deep_validation_fail, tool_error, timeout, infrastructure)" + }, + "last_error_message": { + "type": "string" + }, + "last_error_timestamp": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "last_error_type", + "last_error_message", + "last_error_timestamp" + ], + "default": null, + "description": "Context about the last error for crash recovery diagnostics" + }, + "milestone_step_id": { + "type": "string", + "minLength": 1, + "description": "Active milestone step ID for cross-referencing with roadmap" + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "Timestamp when this scratchpad was first created" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "Timestamp of last scratchpad update" + }, + "parent_task_id": { + "type": [ + "string", + "null" + ], + "description": "Task ID of the parent that spawned this context, null for L1", + "default": null + } + }, + "required": [ + "phase", + "checklist_scope", + "last_validation_gate", + "next_action_ref", + "state_summary", + "milestone_step_id", + "created_at", + "updated_at" + ] +} \ No newline at end of file diff --git a/schema/trinity/session_event.schema.json b/schema/trinity/session_event.schema.json new file mode 100644 index 00000000..74aace88 --- /dev/null +++ b/schema/trinity/session_event.schema.json @@ -0,0 +1,1084 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/session_event.schema.json", + "title": "Trinity Session Event", + "type": "object", + "additionalProperties": false, + "properties": { + "schema_version": { + "type": "string", + "const": "trinity-session-log-v1" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "event_type": { + "type": "string", + "enum": [ + "SPAWN", + "MESSAGE", + "TOOL_CALL", + "TOOL_RESULT", + "VALIDATION", + "TERMINATE", + "ERROR" + ] + }, + "event_id": { + "type": "string", + "minLength": 1 + }, + "event_sequence": { + "type": "integer", + "minimum": 1 + }, + "prev_event_sha256": { + "oneOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "event_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "phase_id": { + "type": "string", + "minLength": 1 + }, + "loop_id": { + "type": "string", + "minLength": 1 + }, + "agent_id": { + "type": "string", + "minLength": 1 + }, + "parent_id": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Researcher", + "Auditor", + "Summarizer", + "ToolUser" + ] + }, + "step_id": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + ] + }, + "tool_call_id": { + "type": [ + "string", + "null" + ] + }, + "result_id": { + "type": [ + "string", + "null" + ] + }, + "artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "artifact_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^(sha256:)?[a-f0-9]{64}$" + }, + "diff_ref": { + "type": [ + "string", + "null" + ] + }, + "model": { + "type": [ + "string", + "null" + ] + }, + "content": { + "type": "object", + "additionalProperties": false, + "properties": { + "summary": { + "type": "string", + "minLength": 1 + }, + "task_input_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "task_result_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "capture_level": { + "type": "string", + "enum": [ + "none", + "summary", + "full" + ] + }, + "capture_decision_reason": { + "type": "string", + "minLength": 1 + }, + "prompt_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "prompt_sha256": { + "oneOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "response_artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "response_sha256": { + "oneOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "tool_call": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "enum": [ + "read_file", + "write_file", + "edit_file", + "apply_patch", + "move_file", + "remove_file", + "list_dir", + "glob_match", + "search_text", + "git_head", + "git_show", + "git_diff", + "exec_cmd", + "validate_json", + "checkpoint_branch", + "checkpoint_commit" + ] + }, + "args": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ + "name", + "args" + ] + }, + "validation": { + "type": "object", + "additionalProperties": false, + "properties": { + "schema": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + }, + "deep_validator": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + }, + "governance": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + }, + "seed_lint": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + }, + "docs_lint": { + "type": "string", + "enum": [ + "pass", + "fail", + "n/a" + ] + } + }, + "required": [ + "schema", + "deep_validator", + "governance", + "seed_lint", + "docs_lint" + ] + }, + "tool_result": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "exit_code": { + "type": "integer", + "minimum": 0, + "maximum": 255 + }, + "duration_ms": { + "type": "integer", + "minimum": 0 + }, + "working_dir": { + "type": "string", + "minLength": 1 + }, + "stdout_excerpt": { + "type": "string" + }, + "stderr_excerpt": { + "type": "string" + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "command", + "exit_code", + "duration_ms", + "working_dir" + ] + } + }, + "required": [ + "summary", + "capture_level", + "capture_decision_reason", + "prompt_artifact_ref", + "prompt_sha256", + "response_artifact_ref", + "response_sha256" + ] + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "toolkit_version": { + "type": "string", + "minLength": 1 + }, + "schema_version": { + "type": "string", + "minLength": 1 + }, + "git_head": { + "type": "string", + "minLength": 1 + }, + "prompt_template_id": { + "type": "string", + "minLength": 1 + }, + "prompt_template_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "redaction_profile": { + "type": "string", + "enum": [ + "eval" + ] + }, + "redaction_applied": { + "type": "boolean" + }, + "capture_policy_ref": { + "type": [ + "string", + "null" + ] + }, + "capture_policy_sha256": { + "oneOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "capture_policy_profile": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "properties": { + "profile": { + "type": "string", + "minLength": 1 + }, + "tier": { + "type": "string", + "minLength": 1 + }, + "budget_tier": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "profile", + "tier", + "budget_tier" + ] + }, + "capture_policy_fallback_applied": { + "type": "boolean" + }, + "capture_policy_fallback_reasons": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "redaction_stats": { + "type": "object", + "additionalProperties": false, + "properties": { + "total_replacements": { + "type": "integer", + "minimum": 0 + }, + "by_class": { + "type": "object", + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + }, + "classes_detected": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "detectors_used": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "min_confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "total_replacements", + "by_class", + "classes_detected", + "detectors_used", + "min_confidence", + "max_confidence" + ] + }, + "decoding": { + "type": "object", + "additionalProperties": false, + "properties": { + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number" + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "temperature", + "top_p", + "max_tokens" + ] + }, + "token_usage": { + "type": "object", + "additionalProperties": false, + "properties": { + "prompt": { + "type": "integer", + "minimum": 0 + }, + "completion": { + "type": "integer", + "minimum": 0 + }, + "total": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "prompt", + "completion", + "total" + ] + }, + "tool_schema_context": { + "type": "object", + "additionalProperties": false, + "properties": { + "mode": { + "type": "string", + "enum": [ + "catalog_only", + "catalog_plus_on_demand", + "full_inline" + ] + }, + "catalog_ref": { + "type": [ + "string", + "null" + ] + }, + "catalog_sha256": { + "oneOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "expanded_tool_names": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "read_file", + "write_file", + "edit_file", + "apply_patch", + "move_file", + "remove_file", + "list_dir", + "glob_match", + "search_text", + "git_head", + "git_show", + "git_diff", + "exec_cmd", + "validate_json", + "checkpoint_branch", + "checkpoint_commit" + ] + } + }, + "request_schema_uri": { + "type": "string", + "const": "https://specdev.local/schema/trinity/tool_call_request.schema.json" + }, + "request_schema_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "result_schema_uri": { + "type": "string", + "const": "https://specdev.local/schema/trinity/tool_call_result.schema.json" + }, + "result_schema_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "mode", + "catalog_ref", + "catalog_sha256", + "expanded_tool_names", + "request_schema_uri", + "request_schema_sha256", + "result_schema_uri", + "result_schema_sha256" + ], + "allOf": [ + { + "if": { + "properties": { + "mode": { + "const": "catalog_only" + } + }, + "required": [ + "mode" + ] + }, + "then": { + "properties": { + "expanded_tool_names": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "mode": { + "const": "catalog_plus_on_demand" + } + }, + "required": [ + "mode" + ] + }, + "then": { + "properties": { + "expanded_tool_names": { + "minItems": 1 + } + } + } + }, + { + "if": { + "properties": { + "mode": { + "enum": [ + "catalog_only", + "catalog_plus_on_demand" + ] + } + }, + "required": [ + "mode" + ] + }, + "then": { + "properties": { + "catalog_ref": { + "type": "string", + "minLength": 1 + }, + "catalog_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } + } + }, + { + "if": { + "properties": { + "mode": { + "const": "full_inline" + } + }, + "required": [ + "mode" + ] + }, + "then": { + "properties": { + "catalog_ref": { + "type": "null" + }, + "catalog_sha256": { + "type": "null" + } + } + } + } + ] + }, + "anchor_union_metrics": { + "type": "object", + "additionalProperties": false, + "properties": { + "active_contexts": { + "type": "integer", + "minimum": 0 + }, + "merged_seed_refs": { + "type": "integer", + "minimum": 0 + }, + "union_scope_in_count": { + "type": "integer", + "minimum": 0 + }, + "union_scope_out_count": { + "type": "integer", + "minimum": 0 + }, + "union_target_patterns_count": { + "type": "integer", + "minimum": 0 + }, + "union_docs_touched_count": { + "type": "integer", + "minimum": 0 + }, + "union_test_commands_count": { + "type": "integer", + "minimum": 0 + }, + "checklist_items_count": { + "type": "integer", + "minimum": 0 + }, + "checklist_conflicts_count": { + "type": "integer", + "minimum": 0 + }, + "checklist_conflict_ids": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "active_contexts", + "merged_seed_refs", + "union_scope_in_count", + "union_scope_out_count", + "union_target_patterns_count", + "union_docs_touched_count", + "union_test_commands_count", + "checklist_items_count", + "checklist_conflicts_count", + "checklist_conflict_ids" + ] + } + }, + "required": [ + "toolkit_version", + "schema_version", + "git_head", + "prompt_template_id", + "prompt_template_sha256", + "redaction_profile", + "redaction_applied", + "capture_policy_ref", + "capture_policy_sha256", + "redaction_stats", + "decoding", + "token_usage" + ] + } + }, + "required": [ + "schema_version", + "timestamp", + "event_type", + "event_id", + "event_sequence", + "prev_event_sha256", + "event_sha256", + "run_id", + "phase_id", + "loop_id", + "agent_id", + "parent_id", + "role", + "step_id", + "tool_call_id", + "result_id", + "artifact_ref", + "artifact_sha256", + "diff_ref", + "model", + "content", + "metadata" + ], + "allOf": [ + { + "if": { + "properties": { + "event_type": { + "const": "TOOL_CALL" + } + } + }, + "then": { + "required": [ + "tool_call_id" + ], + "properties": { + "tool_call_id": { + "type": "string", + "minLength": 1 + }, + "content": { + "required": [ + "tool_call" + ] + }, + "metadata": { + "required": [ + "tool_schema_context" + ] + } + } + } + }, + { + "if": { + "properties": { + "event_type": { + "const": "TOOL_RESULT" + } + } + }, + "then": { + "required": [ + "tool_call_id", + "result_id" + ], + "properties": { + "tool_call_id": { + "type": "string", + "minLength": 1 + }, + "result_id": { + "type": "string", + "minLength": 1 + }, + "content": { + "required": [ + "tool_result" + ] + }, + "metadata": { + "required": [ + "tool_schema_context" + ] + } + } + } + }, + { + "if": { + "properties": { + "event_type": { + "const": "VALIDATION" + } + } + }, + "then": { + "properties": { + "content": { + "required": [ + "validation" + ] + } + } + } + }, + { + "if": { + "properties": { + "event_type": { + "const": "SPAWN" + } + } + }, + "then": { + "properties": { + "content": { + "required": [ + "task_input_artifact_ref" + ], + "properties": { + "task_input_artifact_ref": { + "type": "string", + "pattern": "(^|.*[\\\\/])spawns[\\\\/][^\\\\/]+[\\\\/]task_input\\.json$" + } + } + } + } + } + }, + { + "if": { + "properties": { + "event_type": { + "const": "TERMINATE" + } + } + }, + "then": { + "properties": { + "content": { + "required": [ + "task_result_artifact_ref" + ], + "properties": { + "task_result_artifact_ref": { + "type": "string", + "pattern": "(^|.*[\\\\/])spawns[\\\\/][^\\\\/]+[\\\\/]task_result\\.json$" + } + } + } + } + } + }, + { + "if": { + "properties": { + "content": { + "properties": { + "capture_level": { + "const": "full" + } + }, + "required": [ + "capture_level" + ] + } + } + }, + "then": { + "properties": { + "content": { + "properties": { + "prompt_artifact_ref": { + "type": "string", + "minLength": 1 + }, + "prompt_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "response_artifact_ref": { + "type": "string", + "minLength": 1 + }, + "response_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "prompt_artifact_ref", + "prompt_sha256", + "response_artifact_ref", + "response_sha256" + ] + } + } + } + }, + { + "if": { + "properties": { + "content": { + "properties": { + "capture_level": { + "const": "none" + } + }, + "required": [ + "capture_level" + ] + } + } + }, + "then": { + "properties": { + "content": { + "properties": { + "prompt_artifact_ref": { + "type": "null" + }, + "prompt_sha256": { + "type": "null" + }, + "response_artifact_ref": { + "type": "null" + }, + "response_sha256": { + "type": "null" + } + } + } + } + } + }, + { + "if": { + "properties": { + "metadata": { + "properties": { + "redaction_applied": { + "const": true + } + }, + "required": [ + "redaction_applied" + ] + } + } + }, + "then": { + "properties": { + "metadata": { + "properties": { + "redaction_stats": { + "properties": { + "total_replacements": { + "minimum": 0 + } + } + } + } + } + } + } + }, + { + "if": { + "properties": { + "artifact_sha256": { + "type": "string" + } + }, + "required": [ + "artifact_sha256" + ] + }, + "then": { + "required": [ + "artifact_ref" + ], + "properties": { + "artifact_ref": { + "type": "string", + "minLength": 1 + } + } + } + }, + { + "if": { + "properties": { + "artifact_ref": { + "type": "string" + } + }, + "required": [ + "artifact_ref" + ] + }, + "then": { + "required": [ + "artifact_sha256" + ], + "properties": { + "artifact_sha256": { + "type": "string", + "pattern": "^(sha256:)?[a-f0-9]{64}$" + } + } + } + } + ] +} diff --git a/schema/trinity/session_state.schema.json b/schema/trinity/session_state.schema.json new file mode 100644 index 00000000..c95edf83 --- /dev/null +++ b/schema/trinity/session_state.schema.json @@ -0,0 +1,132 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/session_state.schema.json", + "title": "Trinity Session State", + "type": "object", + "additionalProperties": false, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "parent_id": { + "type": "string", + "minLength": 1 + }, + "active_phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + }, + "status": { + "type": "string", + "enum": [ + "idle", + "waiting_child", + "awaiting_input", + "resuming", + "blocked", + "done" + ] + }, + "pending_child_id": { + "type": [ + "string", + "null" + ] + }, + "pending_spawn_ref": { + "type": [ + "string", + "null" + ] + }, + "pending_questions": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string", + "minLength": 1 + } + }, + "session_log_ref": { + "type": [ + "string", + "null" + ] + }, + "spawn_log_ref": { + "type": [ + "string", + "null" + ] + }, + "scratchpad_ref": { + "type": [ + "string", + "null" + ] + }, + "last_event_id": { + "type": [ + "string", + "null" + ] + }, + "retry_counters": { + "type": "object", + "additionalProperties": false, + "properties": { + "planner": { + "type": "integer", + "minimum": 0 + }, + "builder": { + "type": "integer", + "minimum": 0 + }, + "verifier": { + "type": "integer", + "minimum": 0 + }, + "milestone": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "planner", + "builder", + "verifier", + "milestone" + ] + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "protocol_version", + "run_id", + "parent_id", + "active_phase", + "step_id", + "status", + "retry_counters", + "updated_at" + ] +} diff --git a/schema/trinity/spawn_log.schema.json b/schema/trinity/spawn_log.schema.json new file mode 100644 index 00000000..2e703d48 --- /dev/null +++ b/schema/trinity/spawn_log.schema.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/spawn_log.schema.json", + "title": "Trinity Spawn Log", + "type": "object", + "additionalProperties": false, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "spawn_id": { + "type": "string", + "minLength": 1 + }, + "parent_id": { + "type": "string", + "minLength": 1 + }, + "child_id": { + "type": "string", + "minLength": 1 + }, + "purpose": { + "type": "string", + "minLength": 1 + }, + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + }, + "attempt": { + "type": "integer", + "minimum": 1 + }, + "checklist_scope": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "status": { + "type": "string", + "enum": [ + "spawned", + "completed", + "blocked", + "failed", + "aborted" + ] + }, + "task_input_ref": { + "type": [ + "string", + "null" + ] + }, + "task_result_ref": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "spawn_id", + "parent_id", + "child_id", + "purpose", + "phase", + "step_id", + "attempt", + "status", + "created_at", + "updated_at" + ] + } + } + }, + "required": [ + "protocol_version", + "run_id", + "entries" + ] +} diff --git a/schema/trinity/task_input.schema.json b/schema/trinity/task_input.schema.json new file mode 100644 index 00000000..4cadff46 --- /dev/null +++ b/schema/trinity/task_input.schema.json @@ -0,0 +1,154 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/task_input.schema.json", + "title": "Trinity Task Input", + "type": "object", + "additionalProperties": false, + "$defs": { + "specRefPointer": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "fr", + "api", + "nfr", + "inv", + "fixture" + ] + }, + "id": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "line_range": { + "type": "string", + "pattern": "^L\\d+-L\\d+$" + }, + "commit_hash": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "type", + "id" + ] + } + }, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "child_id": { + "type": "string", + "minLength": 1 + }, + "parent_id": { + "type": "string", + "minLength": 1 + }, + "role": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Auditor", + "Researcher", + "Summarizer", + "ToolUser" + ] + }, + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "type": "string", + "minLength": 1, + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + }, + "task_description": { + "type": "string", + "minLength": 1 + }, + "expected_output_schema": { + "type": "string", + "minLength": 1 + }, + "context_pack_ref": { + "type": "string", + "minLength": 1 + }, + "target_files": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "spec_refs": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/specRefPointer" + } + }, + "role_metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "prompt_source": { + "type": "string", + "minLength": 1 + }, + "persona_goal": { + "type": "string", + "minLength": 1 + }, + "stop_conditions": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "prompt_source", + "persona_goal", + "stop_conditions" + ] + } + }, + "required": [ + "protocol_version", + "child_id", + "parent_id", + "role", + "phase", + "step_id", + "task_description", + "expected_output_schema", + "context_pack_ref", + "target_files", + "spec_refs", + "role_metadata" + ] +} diff --git a/schema/trinity/task_result.schema.json b/schema/trinity/task_result.schema.json new file mode 100644 index 00000000..3f974355 --- /dev/null +++ b/schema/trinity/task_result.schema.json @@ -0,0 +1,197 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/task_result.schema.json", + "title": "Trinity Task Result", + "type": "object", + "additionalProperties": false, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "child_id": { + "type": "string", + "minLength": 1 + }, + "role": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Auditor", + "Researcher", + "Summarizer", + "ToolUser" + ] + }, + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "type": "string", + "minLength": 1, + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + }, + "status": { + "type": "string", + "enum": [ + "success", + "blocked", + "failed", + "questions" + ] + }, + "summary": { + "type": "string", + "minLength": 1 + }, + "artifacts": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": [ + "bug", + "gap", + "scope_creep", + "tests", + "docs", + "design", + "policy" + ] + }, + "severity": { + "type": "string", + "enum": [ + "blocking", + "major", + "minor", + "nit" + ] + }, + "description": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "minLength": 1 + }, + "impact": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "id", + "type", + "severity", + "description", + "source", + "impact" + ] + } + }, + "questions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "protocol_version", + "child_id", + "role", + "phase", + "step_id", + "status", + "summary", + "artifacts" + ], + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "success" + } + } + }, + "then": { + "properties": { + "artifacts": { + "minItems": 1 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "questions" + } + } + }, + "then": { + "required": [ + "questions" + ], + "properties": { + "questions": { + "minItems": 1 + }, + "artifacts": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "blocked", + "failed" + ] + } + } + }, + "then": { + "required": [ + "findings" + ], + "properties": { + "findings": { + "minItems": 1 + } + } + } + } + ] +} diff --git a/schema/trinity/tool_call_request.schema.json b/schema/trinity/tool_call_request.schema.json new file mode 100644 index 00000000..24710ae1 --- /dev/null +++ b/schema/trinity/tool_call_request.schema.json @@ -0,0 +1,727 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/tool_call_request.schema.json", + "title": "Trinity Tool Call Request", + "type": "object", + "additionalProperties": false, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "call_id": { + "type": "string", + "minLength": 1 + }, + "agent_id": { + "type": "string", + "minLength": 1 + }, + "parent_id": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Researcher", + "Auditor", + "Summarizer", + "ToolUser" + ] + }, + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + ] + }, + "tool_name": { + "type": "string", + "enum": [ + "read_file", + "write_file", + "edit_file", + "apply_patch", + "move_file", + "remove_file", + "list_dir", + "glob_match", + "search_text", + "git_head", + "git_show", + "git_diff", + "exec_cmd", + "validate_json", + "checkpoint_branch", + "checkpoint_commit" + ] + }, + "args": { + "type": "object" + }, + "working_dir": { + "type": "string", + "minLength": 1 + }, + "timeout_seconds": { + "type": "integer", + "minimum": 1 + }, + "created_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "protocol_version", + "run_id", + "call_id", + "agent_id", + "role", + "phase", + "step_id", + "tool_name", + "args", + "created_at" + ], + "allOf": [ + { + "if": { + "properties": { + "phase": { + "enum": [ + "16a", + "16b", + "16c" + ] + } + } + }, + "then": { + "properties": { + "step_id": { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "read_file" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "start_line": { + "type": "integer", + "minimum": 1 + }, + "end_line": { + "type": "integer", + "minimum": 1 + }, + "max_chars": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "path" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "write_file" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "content": { + "type": "string" + }, + "encoding": { + "type": "string", + "enum": [ + "utf-8", + "ascii" + ] + }, + "mode": { + "type": "string", + "enum": [ + "overwrite", + "append", + "create_new" + ] + }, + "create_parents": { + "type": "boolean" + } + }, + "required": [ + "path", + "content" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "edit_file" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "edits": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "search": { + "type": "string" + }, + "replace": { + "type": "string" + }, + "occurrence": { + "type": "integer", + "minimum": 1 + }, + "regex": { + "type": "boolean" + } + }, + "required": [ + "search", + "replace" + ] + } + } + }, + "required": [ + "path", + "edits" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "apply_patch" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "patch": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "patch" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "list_dir" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "recursive": { + "type": "boolean" + }, + "include_hidden": { + "type": "boolean" + } + }, + "required": [ + "path" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "move_file" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "src_path": { + "type": "string", + "minLength": 1 + }, + "dst_path": { + "type": "string", + "minLength": 1 + }, + "overwrite": { + "type": "boolean" + }, + "create_parents": { + "type": "boolean" + } + }, + "required": [ + "src_path", + "dst_path" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "remove_file" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "missing_ok": { + "type": "boolean" + } + }, + "required": [ + "path" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "glob_match" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "patterns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "path", + "patterns" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "search_text" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "pattern": { + "type": "string", + "minLength": 1 + }, + "paths": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "case_sensitive": { + "type": "boolean" + }, + "use_regex": { + "type": "boolean" + }, + "max_matches": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "pattern", + "paths" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "git_head" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "maxProperties": 0 + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "git_show" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "rev": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "rev" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "git_diff" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "base_rev": { + "type": "string", + "minLength": 1 + }, + "head_rev": { + "type": "string", + "minLength": 1 + }, + "paths": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "context_lines": { + "type": "integer", + "minimum": 0 + } + }, + "anyOf": [ + { + "required": [ + "base_rev" + ] + }, + { + "required": [ + "head_rev" + ] + }, + { + "required": [ + "paths" + ] + } + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "exec_cmd" + } + } + }, + "then": { + "required": [ + "working_dir" + ], + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "mode": { + "type": "string", + "enum": [ + "standard", + "summarized" + ] + }, + "timeout_seconds": { + "type": "integer", + "minimum": 1 + }, + "working_dir": { + "type": "string", + "minLength": 1 + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "command", + "mode" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "validate_json" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "schema_ref": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "path" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "checkpoint_branch" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "branch_name": { + "type": "string", + "pattern": "^trinity\\/[a-z0-9._\\/-]+$" + }, + "base_ref": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "branch_name" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "checkpoint_commit" + } + } + }, + "then": { + "properties": { + "args": { + "type": "object", + "additionalProperties": false, + "properties": { + "message": { + "type": "string", + "minLength": 1 + }, + "files": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "message" + ] + } + } + } + } + ] +} diff --git a/schema/trinity/tool_call_result.schema.json b/schema/trinity/tool_call_result.schema.json new file mode 100644 index 00000000..d651cca2 --- /dev/null +++ b/schema/trinity/tool_call_result.schema.json @@ -0,0 +1,944 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/tool_call_result.schema.json", + "title": "Trinity Tool Call Result", + "type": "object", + "additionalProperties": false, + "properties": { + "protocol_version": { + "const": "trinity-runtime-v1" + }, + "run_id": { + "type": "string", + "minLength": 1 + }, + "call_id": { + "type": "string", + "minLength": 1 + }, + "result_id": { + "type": "string", + "minLength": 1 + }, + "agent_id": { + "type": "string", + "minLength": 1 + }, + "role": { + "type": "string", + "enum": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Researcher", + "Auditor", + "Summarizer", + "ToolUser" + ] + }, + "phase": { + "type": "string", + "enum": [ + "16a", + "16b", + "16c", + "utility" + ] + }, + "step_id": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + ] + }, + "tool_name": { + "type": "string", + "enum": [ + "read_file", + "write_file", + "edit_file", + "apply_patch", + "move_file", + "remove_file", + "list_dir", + "glob_match", + "search_text", + "git_head", + "git_show", + "git_diff", + "exec_cmd", + "validate_json", + "checkpoint_branch", + "checkpoint_commit" + ] + }, + "status": { + "type": "string", + "enum": [ + "success", + "error", + "blocked", + "timeout" + ] + }, + "summary": { + "type": "string", + "minLength": 1 + }, + "result": { + "type": "object" + }, + "duration_ms": { + "type": "integer", + "minimum": 0 + }, + "exit_code": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 255 + }, + "working_dir": { + "type": [ + "string", + "null" + ] + }, + "stdout_excerpt": { + "type": [ + "string", + "null" + ] + }, + "stderr_excerpt": { + "type": [ + "string", + "null" + ] + }, + "truncated": { + "type": "boolean" + }, + "artifact_ref": { + "type": [ + "string", + "null" + ] + }, + "artifact_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^(sha256:)?[a-f0-9]{64}$" + }, + "error": { + "type": "object", + "additionalProperties": false, + "properties": { + "code": { + "type": "string", + "minLength": 1 + }, + "message": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "code", + "message" + ] + }, + "finished_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "protocol_version", + "run_id", + "call_id", + "result_id", + "agent_id", + "role", + "phase", + "step_id", + "tool_name", + "status", + "summary", + "result", + "duration_ms", + "truncated", + "finished_at" + ], + "allOf": [ + { + "if": { + "properties": { + "phase": { + "enum": [ + "16a", + "16b", + "16c" + ] + } + } + }, + "then": { + "properties": { + "step_id": { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "error", + "blocked", + "timeout" + ] + } + } + }, + "then": { + "required": [ + "error" + ] + } + }, + { + "if": { + "properties": { + "artifact_ref": { + "type": "string" + } + }, + "required": [ + "artifact_ref" + ] + }, + "then": { + "required": [ + "artifact_sha256" + ], + "properties": { + "artifact_sha256": { + "type": "string", + "pattern": "^(sha256:)?[a-f0-9]{64}$" + } + } + } + }, + { + "if": { + "properties": { + "artifact_sha256": { + "type": "string" + } + }, + "required": [ + "artifact_sha256" + ] + }, + "then": { + "required": [ + "artifact_ref" + ], + "properties": { + "artifact_ref": { + "type": "string", + "minLength": 1 + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "read_file" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer", + "minimum": 1 + }, + "line_end": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "bytes_read": { + "type": "integer", + "minimum": 0 + }, + "content": { + "type": "string" + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "path", + "line_start", + "line_end", + "bytes_read", + "content", + "truncated" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "write_file" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "bytes_written": { + "type": "integer", + "minimum": 0 + }, + "content_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^(sha256:)?[a-f0-9]{64}$" + } + }, + "required": [ + "path", + "bytes_written", + "content_sha256" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "edit_file" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "edits_applied": { + "type": "integer", + "minimum": 0 + }, + "content_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^(sha256:)?[a-f0-9]{64}$" + } + }, + "required": [ + "path", + "edits_applied", + "content_sha256" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "apply_patch" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "files_changed": { + "type": "integer", + "minimum": 0 + }, + "hunks_applied": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "files_changed", + "hunks_applied" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "list_dir" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "entries": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "path", + "entries" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "move_file" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "src_path": { + "type": "string", + "minLength": 1 + }, + "dst_path": { + "type": "string", + "minLength": 1 + }, + "content_sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^(sha256:)?[a-f0-9]{64}$" + } + }, + "required": [ + "src_path", + "dst_path", + "content_sha256" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "remove_file" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "removed": { + "type": "boolean" + }, + "previously_missing": { + "type": "boolean" + } + }, + "required": [ + "path", + "removed", + "previously_missing" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "glob_match" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "patterns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "matches": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "path", + "patterns", + "matches" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "search_text" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "pattern": { + "type": "string", + "minLength": 1 + }, + "paths_scanned": { + "type": "integer", + "minimum": 0 + }, + "matches": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "line": { + "type": "integer", + "minimum": 1 + }, + "text_excerpt": { + "type": "string" + } + }, + "required": [ + "path", + "line", + "text_excerpt" + ] + } + } + }, + "required": [ + "pattern", + "paths_scanned", + "matches" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "git_head" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "head": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "head" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "git_show" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "rev": { + "type": "string", + "minLength": 1 + }, + "content_excerpt": { + "type": "string" + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "rev", + "content_excerpt", + "truncated" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "git_diff" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "base_rev": { + "type": [ + "string", + "null" + ] + }, + "head_rev": { + "type": [ + "string", + "null" + ] + }, + "diff_excerpt": { + "type": "string" + }, + "truncated": { + "type": "boolean" + } + }, + "required": [ + "base_rev", + "head_rev", + "diff_excerpt", + "truncated" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "exec_cmd" + } + } + }, + "then": { + "required": [ + "exit_code", + "working_dir" + ], + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "mode": { + "type": "string", + "enum": [ + "standard", + "summarized" + ] + } + }, + "required": [ + "command", + "mode" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "validate_json" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "valid": { + "type": "boolean" + }, + "errors": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "path", + "valid", + "errors" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "checkpoint_branch" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "branch_name": { + "type": "string", + "pattern": "^trinity\\/[a-z0-9._\\/-]+$" + }, + "head": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "branch_name", + "head" + ] + } + } + } + }, + { + "if": { + "properties": { + "tool_name": { + "const": "checkpoint_commit" + } + } + }, + "then": { + "properties": { + "result": { + "type": "object", + "additionalProperties": false, + "properties": { + "commit_sha": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "message": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "commit_sha", + "message" + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "success" + }, + "tool_name": { + "enum": [ + "write_file", + "edit_file", + "move_file", + "remove_file" + ] + } + }, + "required": [ + "status", + "tool_name" + ] + }, + "then": { + "required": [ + "artifact_ref", + "artifact_sha256" + ], + "properties": { + "artifact_ref": { + "type": "string", + "minLength": 1 + }, + "artifact_sha256": { + "type": "string", + "pattern": "^(sha256:)?[a-f0-9]{64}$" + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "success" + }, + "tool_name": { + "const": "exec_cmd" + } + }, + "required": [ + "status", + "tool_name" + ] + }, + "then": { + "properties": { + "exit_code": { + "type": "integer", + "minimum": 0, + "maximum": 255 + }, + "working_dir": { + "type": "string", + "minLength": 1 + } + } + } + } + ] +} diff --git a/schema/trinity/utility_call.schema.json b/schema/trinity/utility_call.schema.json new file mode 100644 index 00000000..441c75cb --- /dev/null +++ b/schema/trinity/utility_call.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/utility_call.schema.json", + "title": "Trinity Utility Call Payload", + "type": "object", + "additionalProperties": false, + "properties": { + "role": { + "type": "string", + "enum": [ + "Researcher", + "ToolUser", + "Summarizer", + "Auditor" + ] + }, + "objective": { + "type": "string", + "minLength": 1 + }, + "input": { + "type": "object", + "additionalProperties": true, + "properties": { + "required_outputs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + }, + "required": [ + "required_outputs" + ] + } + }, + "required": [ + "role", + "objective", + "input" + ] +} diff --git a/schema/trinity/utility_result.schema.json b/schema/trinity/utility_result.schema.json new file mode 100644 index 00000000..e371665a --- /dev/null +++ b/schema/trinity/utility_result.schema.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://specdev.local/schema/trinity/utility_result.schema.json", + "title": "Trinity Utility Result Payload", + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "type": "string", + "enum": [ + "ready", + "questions", + "blocked" + ] + }, + "summary": { + "type": "string", + "minLength": 1 + }, + "open_questions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "errors": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + } + }, + "required": [ + "status", + "summary" + ], + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "questions" + } + }, + "required": [ + "status" + ] + }, + "then": { + "required": [ + "open_questions" + ], + "properties": { + "open_questions": { + "type": "array", + "minItems": 1 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "blocked" + } + }, + "required": [ + "status" + ] + }, + "then": { + "anyOf": [ + { + "required": [ + "errors" + ], + "properties": { + "errors": { + "type": "array", + "minItems": 1 + } + } + }, + { + "required": [ + "findings" + ], + "properties": { + "findings": { + "type": "array", + "minItems": 1 + } + } + } + ] + } + } + ] +} diff --git a/spec/common/seed_manifest.json b/spec/common/seed_manifest.json deleted file mode 100644 index 9ad78a7d..00000000 --- a/spec/common/seed_manifest.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "$schema": "https://specdev.local/schema/seed_manifest.schema.json", - "seed_manifest_id": "seed-manifest-core", - "version": "0.1.0", - "created_at": "2026-02-07T00:00:00Z", - "last_updated": "2026-02-07T00:00:00Z", - "global_seed_order": [ - "seed-overview", - "seed-tech-stack" - ], - "nested_order": [ - { - "level_id": "foundation", - "description": "Baseline project scope and technical constraints.", - "seed_ids": [ - "seed-overview", - "seed-tech-stack" - ] - } - ], - "seeds": [ - { - "seed_id": "seed-overview", - "path": "docs/seed/seed_overview.md", - "description": "Project scope, personas, and business context.", - "required": true, - "source_type": "doc" - }, - { - "seed_id": "seed-tech-stack", - "path": "docs/seed/seed_tech_stack.md", - "description": "Architecture decisions, constraints, and technology baseline.", - "required": true, - "source_type": "doc" - } - ], - "step_requirements": { - "00": [ - "seed-overview", - "seed-tech-stack" - ], - "01": [ - "seed-overview" - ], - "02": [ - "seed-tech-stack" - ], - "02a": [ - "seed-tech-stack" - ], - "03": [ - "seed-overview" - ], - "04": [ - "seed-overview" - ], - "05": [ - "seed-tech-stack" - ], - "06": [ - "seed-tech-stack" - ], - "07": [ - "seed-tech-stack" - ], - "08": [ - "seed-tech-stack" - ], - "09": [ - "seed-tech-stack" - ], - "10": [ - "seed-overview" - ], - "11": [ - "seed-tech-stack" - ], - "12": [ - "seed-tech-stack" - ], - "13": [ - "seed-overview" - ], - "13a": [ - "seed-overview" - ], - "14": [ - "seed-overview" - ], - "15": [ - "seed-tech-stack" - ], - "16a": [ - "seed-overview", - "seed-tech-stack" - ], - "16b": [ - "seed-overview", - "seed-tech-stack" - ], - "16c": [ - "seed-overview", - "seed-tech-stack" - ] - }, - "docs_policy": { - "readme_required": true, - "root_readme_required": true, - "readme_depth_default": 0, - "readme_depth_by_scope": {}, - "scope": [ - "devspec_toolkit/" - ], - "exclusions": [ - "devspec_toolkit/node_modules/", - "devspec_toolkit/.git/", - "devspec_toolkit/.venv/", - "devspec_toolkit/__pycache__/", - "devspec_toolkit/dist/", - "devspec_toolkit/build/", - "devspec_toolkit/coverage/", - "devspec_toolkit/tests/fixtures/", - "devspec_toolkit/tools/specdev_tools.egg-info/" - ], - "doc_paths": [ - "docs/**", - "README.md", - "CHANGELOG.md" - ] - } -} diff --git a/tests/fixtures/step_16/invalid_bad_enum.json b/tests/fixtures/step_16/invalid_bad_enum.json index ce305c79..0cca30ba 100644 --- a/tests/fixtures/step_16/invalid_bad_enum.json +++ b/tests/fixtures/step_16/invalid_bad_enum.json @@ -52,6 +52,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/invalid_delivery_planned_missing_verification.json b/tests/fixtures/step_16/invalid_delivery_planned_missing_verification.json new file mode 100644 index 00000000..fa14483c --- /dev/null +++ b/tests/fixtures/step_16/invalid_delivery_planned_missing_verification.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-delivery-planned-invalid", + "owner": "api", + "created_at": "2024-01-20T10:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Implementation with missing delivery verification.", + "scope_in": [ + "monitoring" + ], + "scope_out": [ + "frontend-ui" + ], + "target_file_patterns": [ + "src/monitoring.py" + ] + }, + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Observability", + "summary": "Provide monitoring coverage for critical NFRs." + } + ], + "checklist": [ + { + "id": "REQ_MONITORING_001", + "spec_ref": { + "type": "nfr", + "id": "nfr-availability-uptime", + "line_range": "L5-L20", + "commit_hash": "1234567890abcdef1234567890abcdef12345678" + }, + "description": "Ensure dashboard and alert coverage exists.", + "type": "logging", + "layer": "integration", + "linked_test_expectation": "monitoring checks pass", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "fixture_ref": "fixture-monitoring-check" + } + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Monitoring coverage must be documented.", + "docs_touched": [ + "README.md" + ] + }, + "review_requirements": { + "guidelines": "Verify delivery status and evidence links.", + "test_commands": [ + "pytest -q" + ] + }, + "delivery": { + "status": "planned", + "dashboards": [ + { + "dashboard_id": "dashboard-availability", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "url": "https://monitoring.example.com/dashboards/availability" + } + ], + "alerts": [ + { + "alert_id": "alert-availability-drop", + "nfr_ref": "nfr-availability-uptime", + "rule": "availability < 99.9%", + "severity": "critical" + } + ] + } + }, + "review": { + "delivery_status": {} + } +} diff --git a/tests/fixtures/step_16/invalid_delivery_planned_unverified_items.json b/tests/fixtures/step_16/invalid_delivery_planned_unverified_items.json new file mode 100644 index 00000000..dd9e4a71 --- /dev/null +++ b/tests/fixtures/step_16/invalid_delivery_planned_unverified_items.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-delivery-planned-unverified-items", + "owner": "api", + "created_at": "2024-01-20T10:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Implementation with unverified planned delivery items.", + "scope_in": [ + "monitoring" + ], + "scope_out": [ + "frontend-ui" + ], + "target_file_patterns": [ + "src/monitoring.py" + ] + }, + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Observability", + "summary": "Provide monitoring coverage for critical NFRs." + } + ], + "checklist": [ + { + "id": "REQ_MONITORING_001", + "spec_ref": { + "type": "nfr", + "id": "nfr-availability-uptime", + "line_range": "L5-L20", + "commit_hash": "1234567890abcdef1234567890abcdef12345678" + }, + "description": "Ensure dashboard and alert coverage exists.", + "type": "logging", + "layer": "integration", + "linked_test_expectation": "monitoring checks pass", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "fixture_ref": "fixture-monitoring-check" + } + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Monitoring coverage must be documented.", + "docs_touched": [ + "README.md" + ] + }, + "review_requirements": { + "guidelines": "Verify delivery status and evidence links.", + "test_commands": [ + "pytest -q" + ] + }, + "delivery": { + "status": "planned", + "dashboards": [ + { + "dashboard_id": "dashboard-availability", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "url": "https://monitoring.example.com/dashboards/availability" + } + ], + "alerts": [ + { + "alert_id": "alert-availability-drop", + "nfr_ref": "nfr-availability-uptime", + "rule": "availability < 99.9%", + "severity": "critical" + } + ] + } + }, + "review": { + "delivery_status": { + "deployments": [ + { + "env": "dev", + "build_id": "build-123", + "status": "success" + } + ] + } + } +} diff --git a/tests/fixtures/step_16/invalid_invalid_layer.json b/tests/fixtures/step_16/invalid_invalid_layer.json index 5a3ae88e..613da0a9 100644 --- a/tests/fixtures/step_16/invalid_invalid_layer.json +++ b/tests/fixtures/step_16/invalid_invalid_layer.json @@ -69,6 +69,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/invalid_invalid_type.json b/tests/fixtures/step_16/invalid_invalid_type.json index 26e16011..b901f53f 100644 --- a/tests/fixtures/step_16/invalid_invalid_type.json +++ b/tests/fixtures/step_16/invalid_invalid_type.json @@ -69,6 +69,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/invalid_missing_evidence.json b/tests/fixtures/step_16/invalid_missing_evidence.json index 79eae56c..063544ff 100644 --- a/tests/fixtures/step_16/invalid_missing_evidence.json +++ b/tests/fixtures/step_16/invalid_missing_evidence.json @@ -56,6 +56,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/invalid_missing_fixture_ref.json b/tests/fixtures/step_16/invalid_missing_fixture_ref.json index 464bdda0..3595494b 100644 --- a/tests/fixtures/step_16/invalid_missing_fixture_ref.json +++ b/tests/fixtures/step_16/invalid_missing_fixture_ref.json @@ -68,6 +68,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/invalid_missing_nfr_refs.json b/tests/fixtures/step_16/invalid_missing_nfr_refs.json index b35414ee..c79d5cd0 100644 --- a/tests/fixtures/step_16/invalid_missing_nfr_refs.json +++ b/tests/fixtures/step_16/invalid_missing_nfr_refs.json @@ -67,6 +67,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/invalid_missing_plan_docs_impact.json b/tests/fixtures/step_16/invalid_missing_plan_docs_impact.json new file mode 100644 index 00000000..27de0bf4 --- /dev/null +++ b/tests/fixtures/step_16/invalid_missing_plan_docs_impact.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-missing-plan-docs-impact", + "owner": "api", + "created_at": "2024-01-01T00:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Minimal implementation.", + "scope_in": [ + "core" + ], + "scope_out": [ + "extras" + ], + "target_file_patterns": [ + "src/auth.py" + ] + }, + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Core Logic", + "summary": "Implement core behavior" + } + ], + "checklist": [ + { + "id": "REQ_CORE_001", + "spec_ref": { + "type": "fr", + "id": "fr-core-login", + "line_range": "L10-L20", + "commit_hash": "a1b2c3d4e5f61234567890123456789012345678" + }, + "description": "Implement login function", + "type": "behavior", + "layer": "service", + "linked_test_expectation": "login success", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "fixture_ref": "fixture-login-success" + } + ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] + } + } +} diff --git a/tests/fixtures/step_16/invalid_missing_plan_review_requirements.json b/tests/fixtures/step_16/invalid_missing_plan_review_requirements.json new file mode 100644 index 00000000..c6248013 --- /dev/null +++ b/tests/fixtures/step_16/invalid_missing_plan_review_requirements.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-missing-plan-review-reqs", + "owner": "api", + "created_at": "2024-01-01T00:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Minimal implementation.", + "scope_in": [ + "core" + ], + "scope_out": [ + "extras" + ], + "target_file_patterns": [ + "src/auth.py" + ] + }, + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Core Logic", + "summary": "Implement core behavior" + } + ], + "checklist": [ + { + "id": "REQ_CORE_001", + "spec_ref": { + "type": "fr", + "id": "fr-core-login", + "line_range": "L10-L20", + "commit_hash": "a1b2c3d4e5f61234567890123456789012345678" + }, + "description": "Implement login function", + "type": "behavior", + "layer": "service", + "linked_test_expectation": "login success", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "fixture_ref": "fixture-login-success" + } + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Code changes require documentation updates for traceability.", + "docs_touched": [ + "README.md" + ] + } + } +} diff --git a/tests/fixtures/step_16/invalid_missing_plan_spec_alignment.json b/tests/fixtures/step_16/invalid_missing_plan_spec_alignment.json new file mode 100644 index 00000000..0b6ac709 --- /dev/null +++ b/tests/fixtures/step_16/invalid_missing_plan_spec_alignment.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-missing-plan-spec-alignment", + "owner": "api", + "created_at": "2024-01-01T00:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Minimal implementation.", + "scope_in": [ + "core" + ], + "scope_out": [ + "extras" + ], + "target_file_patterns": [ + "src/auth.py" + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Code changes require documentation updates for traceability.", + "docs_touched": [ + "README.md" + ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] + } + } +} diff --git a/tests/fixtures/step_16/invalid_missing_plan_summary.json b/tests/fixtures/step_16/invalid_missing_plan_summary.json new file mode 100644 index 00000000..6b7ab263 --- /dev/null +++ b/tests/fixtures/step_16/invalid_missing_plan_summary.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-missing-plan-summary", + "owner": "api", + "created_at": "2024-01-01T00:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Core Logic", + "summary": "Implement core behavior" + } + ], + "checklist": [ + { + "id": "REQ_CORE_001", + "spec_ref": { + "type": "fr", + "id": "fr-core-login", + "line_range": "L10-L20", + "commit_hash": "a1b2c3d4e5f61234567890123456789012345678" + }, + "description": "Implement login function", + "type": "behavior", + "layer": "service", + "linked_test_expectation": "login success", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "fixture_ref": "fixture-login-success" + } + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Code changes require documentation updates for traceability.", + "docs_touched": [ + "README.md" + ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] + } + } +} diff --git a/tests/fixtures/step_16/valid_delivery_planned_with_verification.json b/tests/fixtures/step_16/valid_delivery_planned_with_verification.json new file mode 100644 index 00000000..8faa53ca --- /dev/null +++ b/tests/fixtures/step_16/valid_delivery_planned_with_verification.json @@ -0,0 +1,106 @@ +{ + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-impl-delivery-planned-valid", + "owner": "api", + "created_at": "2024-01-20T10:00:00Z", + "seed_refs": [ + { + "seed_id": "seed-overview" + } + ], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Implementation with delivery verification.", + "scope_in": [ + "monitoring" + ], + "scope_out": [ + "frontend-ui" + ], + "target_file_patterns": [ + "src/monitoring.py", + "README.md" + ] + }, + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Observability", + "summary": "Provide monitoring coverage for critical NFRs." + } + ], + "checklist": [ + { + "id": "REQ_MONITORING_001", + "spec_ref": { + "type": "nfr", + "id": "nfr-availability-uptime", + "line_range": "L5-L20", + "commit_hash": "1234567890abcdef1234567890abcdef12345678" + }, + "description": "Ensure dashboard and alert coverage exists.", + "type": "logging", + "layer": "integration", + "linked_test_expectation": "pytest -q", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "fixture_ref": "fixture-monitoring-check" + } + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Monitoring coverage must be documented.", + "docs_touched": [ + "README.md" + ] + }, + "review_requirements": { + "guidelines": "Verify delivery status and evidence links.", + "test_commands": [ + "pytest -q" + ] + }, + "delivery": { + "status": "planned", + "dashboards": [ + { + "dashboard_id": "dashboard-availability", + "nfr_refs": [ + "nfr-availability-uptime" + ], + "url": "https://monitoring.example.com/dashboards/availability" + } + ], + "alerts": [ + { + "alert_id": "alert-availability-drop", + "nfr_ref": "nfr-availability-uptime", + "rule": "availability < 99.9%", + "severity": "critical" + } + ] + } + }, + "review": { + "delivery_status": { + "dashboards_verified": [ + { + "dashboard_id": "dashboard-availability", + "url": "https://monitoring.example.com/dashboards/availability", + "evidence_ref": "sha256:dashboardsnippet123" + } + ], + "alerts_verified": [ + { + "alert_id": "alert-availability-drop", + "rule": "availability < 99.9%", + "severity": "critical", + "evidence_ref": "sha256:alertsnippet123" + } + ] + } + } +} diff --git a/tests/fixtures/step_16/valid_empty_execution_review.json b/tests/fixtures/step_16/valid_empty_execution_review.json index 8a97b4a7..22b9c2c5 100644 --- a/tests/fixtures/step_16/valid_empty_execution_review.json +++ b/tests/fixtures/step_16/valid_empty_execution_review.json @@ -23,7 +23,8 @@ ], "target_file_patterns": [ "src/main.py", - "src/auth.py" + "src/auth.py", + "README.md" ] }, "spec_alignment": { @@ -45,7 +46,7 @@ "description": "Implement login function", "type": "behavior", "layer": "service", - "linked_test_expectation": "login success", + "linked_test_expectation": "pytest -q", "nfr_refs": ["nfr-availability-uptime"], "fixture_ref": "fixture-login-success", "implementation": { @@ -70,6 +71,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } }, "execution": {}, diff --git a/tests/fixtures/step_16/valid_full.json b/tests/fixtures/step_16/valid_full.json index d03884e8..d2066817 100644 --- a/tests/fixtures/step_16/valid_full.json +++ b/tests/fixtures/step_16/valid_full.json @@ -31,7 +31,8 @@ ], "target_file_patterns": [ "src/api/profile.py", - "migrations/001_users.sql" + "migrations/001_users.sql", + "README.md" ] }, "spec_alignment": { @@ -41,8 +42,8 @@ "summary": "Persist user data securely", "spec_refs": [ { - "type": "doc", - "id": "doc-security-policy", + "type": "nfr", + "id": "nfr-availability-uptime", "line_range": "L50-L60", "commit_hash": "1111111111111111111111111111111111111111" } @@ -53,15 +54,15 @@ { "id": "REQ_DB_SCHEMA", "spec_ref": { - "type": "code", - "id": "schema-user-table", + "type": "fixture", + "id": "fixture-database-migration", "line_range": "L1-L100", "commit_hash": "2222222222222222222222222222222222222222" }, "description": "Create user table migration", "type": "metadata", "layer": "db", - "linked_test_expectation": "migration applies successfully", + "linked_test_expectation": "pytest -q", "nfr_refs": ["nfr-availability-uptime"], "fixture_ref": "fixture-database-migration", "checklist_status": "deferred" @@ -77,7 +78,7 @@ "description": "Implement GET /profile endpoint", "type": "behavior", "layer": "api", - "linked_test_expectation": "returns 200 OK", + "linked_test_expectation": "pytest -q", "nfr_refs": ["nfr-availability-uptime", "nfr-latency-page-load"], "fixture_ref": "fixture-api-profile-get", "implementation": { @@ -114,6 +115,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run checklist-linked tests and verify CI parity.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/fixtures/step_16/valid_minimal.json b/tests/fixtures/step_16/valid_minimal.json index 30035e64..44101734 100644 --- a/tests/fixtures/step_16/valid_minimal.json +++ b/tests/fixtures/step_16/valid_minimal.json @@ -23,7 +23,8 @@ ], "target_file_patterns": [ "src/main.py", - "src/auth.py" + "src/auth.py", + "README.md" ] }, "spec_alignment": { @@ -45,7 +46,7 @@ "description": "Implement login function", "type": "behavior", "layer": "service", - "linked_test_expectation": "login success", + "linked_test_expectation": "pytest -q", "nfr_refs": ["nfr-availability-uptime"], "fixture_ref": "fixture-login-success", "implementation": { @@ -70,6 +71,12 @@ "docs_touched": [ "README.md" ] + }, + "review_requirements": { + "guidelines": "Run focused unit checks for checklist scope.", + "test_commands": [ + "pytest -q" + ] } } } diff --git a/tests/integration/test_step_16.py b/tests/integration/test_step_16.py index e246b643..01f62606 100644 --- a/tests/integration/test_step_16.py +++ b/tests/integration/test_step_16.py @@ -1,7 +1,9 @@ import unittest import os import json +import hashlib import sys +import tempfile from pathlib import Path # Ensure local tools package is importable when tests run from repo roots @@ -68,6 +70,48 @@ def test_invalid_invalid_layer(self): path = os.path.join(self.fixtures_dir, "invalid_invalid_layer.json") errors = validate_file(self.repo_root, path) self.assertTrue(len(errors) > 0, "Invalid fixture (invalid layer) should fail validation") + + def test_invalid_missing_plan_summary(self): + # Expect failure because plan.summary is now required at schema level + path = os.path.join(self.fixtures_dir, "invalid_missing_plan_summary.json") + errors = validate_file(self.repo_root, path) + self.assertTrue(len(errors) > 0, "Invalid fixture (missing plan.summary) should fail validation") + + def test_invalid_missing_plan_docs_impact(self): + # Expect failure because plan.docs_impact is now required at schema level + path = os.path.join(self.fixtures_dir, "invalid_missing_plan_docs_impact.json") + errors = validate_file(self.repo_root, path) + self.assertTrue(len(errors) > 0, "Invalid fixture (missing plan.docs_impact) should fail validation") + + def test_invalid_missing_plan_spec_alignment(self): + # Expect failure because plan.spec_alignment is now required at schema level + path = os.path.join(self.fixtures_dir, "invalid_missing_plan_spec_alignment.json") + errors = validate_file(self.repo_root, path) + self.assertTrue(len(errors) > 0, "Invalid fixture (missing plan.spec_alignment) should fail validation") + + def test_invalid_missing_plan_review_requirements(self): + # Expect failure because plan.review_requirements is now required at schema level + path = os.path.join(self.fixtures_dir, "invalid_missing_plan_review_requirements.json") + errors = validate_file(self.repo_root, path) + self.assertTrue(len(errors) > 0, "Invalid fixture (missing plan.review_requirements) should fail validation") + + def test_valid_delivery_planned_with_verification(self): + # Expect pass: plan.delivery.status == planned includes structured delivery verification evidence + path = os.path.join(self.fixtures_dir, "valid_delivery_planned_with_verification.json") + errors = validate_file(self.repo_root, path) + self.assertEqual(errors, [], f"Valid fixture (delivery planned with verification) should pass. Errors: {errors}") + + def test_invalid_delivery_planned_missing_verification(self): + # Expect failure: planned delivery requires at least one verification entry in review.delivery_status + path = os.path.join(self.fixtures_dir, "invalid_delivery_planned_missing_verification.json") + errors = validate_file(self.repo_root, path) + self.assertTrue(len(errors) > 0, "Invalid fixture (delivery planned missing verification) should fail validation") + + def test_invalid_delivery_planned_unverified_items(self): + # Expect failure: planned dashboard/alert items must have matching verification entries + path = os.path.join(self.fixtures_dir, "invalid_delivery_planned_unverified_items.json") + errors = validate_file(self.repo_root, path) + self.assertTrue(len(errors) > 0, "Invalid fixture (delivery planned with unverified dashboard/alert items) should fail validation") def test_valid_with_new_fields(self): # Test that valid fixtures with new fields pass validation @@ -80,6 +124,166 @@ def test_valid_full_with_new_fields(self): path = os.path.join(self.fixtures_dir, "valid_full.json") errors = validate_file(self.repo_root, path) self.assertEqual(errors, [], f"Valid full fixture with new fields should pass. Errors: {errors}") + + def test_invalid_execution_files_touched_out_of_scope(self): + path = os.path.join(self.fixtures_dir, "valid_minimal.json") + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + payload["execution"] = { + "files_touched": ["scripts/out_of_scope.sh"], + "execution_results": [ + { + "status": "failed", + "outcome_description": "Command failed", + "reasoning": "Expected failure for scope test", + "command": "echo scope", + "evidence": "scope validation failed in deterministic test case" + } + ] + } + with tempfile.TemporaryDirectory() as tmp_dir: + step_dir = os.path.join(tmp_dir, "step_16") + os.makedirs(step_dir, exist_ok=True) + tmp_path = os.path.join(step_dir, "invalid_execution_scope.json") + with open(tmp_path, "w", encoding="utf-8") as tmp: + json.dump(payload, tmp, indent=2) + errors = validate_file(self.repo_root, tmp_path) + self.assertTrue( + any("touched by execution but not covered by target_file_patterns" in e for e in errors), + f"Expected execution scope error. Errors: {errors}", + ) + + def test_invalid_planned_non_doc_scope_requires_docs_impact_required(self): + path = os.path.join(self.fixtures_dir, "valid_minimal.json") + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + payload["plan"]["docs_impact"]["status"] = "not_required" + payload["plan"]["docs_impact"].pop("docs_touched", None) + with tempfile.TemporaryDirectory() as tmp_dir: + step_dir = os.path.join(tmp_dir, "step_16") + os.makedirs(step_dir, exist_ok=True) + tmp_path = os.path.join(step_dir, "invalid_docs_impact_required.json") + with open(tmp_path, "w", encoding="utf-8") as tmp: + json.dump(payload, tmp, indent=2) + errors = validate_file(self.repo_root, tmp_path) + self.assertTrue( + any("docs_impact.status must be 'required'" in e for e in errors), + f"Expected docs_impact requirement error. Errors: {errors}", + ) + + def test_invalid_docs_touched_must_be_in_target_file_patterns(self): + path = os.path.join(self.fixtures_dir, "valid_minimal.json") + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + payload["plan"]["docs_impact"]["docs_touched"] = ["docs/architecture.md"] + with tempfile.TemporaryDirectory() as tmp_dir: + step_dir = os.path.join(tmp_dir, "step_16") + os.makedirs(step_dir, exist_ok=True) + tmp_path = os.path.join(step_dir, "invalid_docs_touched_scope.json") + with open(tmp_path, "w", encoding="utf-8") as tmp: + json.dump(payload, tmp, indent=2) + errors = validate_file(self.repo_root, tmp_path) + self.assertTrue( + any("docs_touched includes paths outside plan.summary.target_file_patterns" in e for e in errors), + f"Expected docs_touched scope error. Errors: {errors}", + ) + + def test_invalid_execution_passed_result_with_bad_evidence_hash(self): + path = os.path.join(self.fixtures_dir, "valid_minimal.json") + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + evidence = "tests/auth/test_login.py::test_login PASSED [100%] deterministic evidence block" + good_sha = hashlib.sha256(evidence.encode("utf-8")).hexdigest() + payload["execution"] = { + "files_touched": ["src/auth.py", "README.md"], + "execution_results": [ + { + "status": "passed", + "outcome_description": "Ran linked tests", + "reasoning": "Linked test command passed", + "command": "pytest -q", + "evidence": evidence, + "evidence_ref": f"sha256:{good_sha}", + "evidence_binding": { + "timestamp": "2026-02-13T00:00:00Z", + "sha256": "0" * 64, + "exit_code": 0, + "command": "pytest -q" + } + } + ], + "critical_evidence": { + "satisfied_checklist_ids": ["REQ_CORE_001"], + "passed_test_commands": ["pytest -q"] + } + } + with tempfile.TemporaryDirectory() as tmp_dir: + step_dir = os.path.join(tmp_dir, "step_16") + os.makedirs(step_dir, exist_ok=True) + tmp_path = os.path.join(step_dir, "invalid_bad_evidence_hash.json") + with open(tmp_path, "w", encoding="utf-8") as tmp: + json.dump(payload, tmp, indent=2) + errors = validate_file(self.repo_root, tmp_path) + self.assertTrue( + any("invalid evidence_binding.sha256" in e for e in errors), + f"Expected evidence hash mismatch error. Errors: {errors}", + ) + + def test_invalid_execution_missing_review_test_command_coverage(self): + path = os.path.join(self.fixtures_dir, "valid_minimal.json") + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + payload["execution"] = { + "files_touched": ["src/auth.py", "README.md"], + "execution_results": [ + { + "status": "failed", + "outcome_description": "Ran wrong command", + "reasoning": "Used wrong command for coverage check", + "command": "echo noop", + "evidence": "noop command output that is long enough for validation" + } + ] + } + with tempfile.TemporaryDirectory() as tmp_dir: + step_dir = os.path.join(tmp_dir, "step_16") + os.makedirs(step_dir, exist_ok=True) + tmp_path = os.path.join(step_dir, "invalid_missing_review_command_coverage.json") + with open(tmp_path, "w", encoding="utf-8") as tmp: + json.dump(payload, tmp, indent=2) + errors = validate_file(self.repo_root, tmp_path) + self.assertTrue( + any("missing required plan.review_requirements.test_commands" in e for e in errors), + f"Expected review command coverage error. Errors: {errors}", + ) + + def test_invalid_execution_sensitive_evidence_content(self): + path = os.path.join(self.fixtures_dir, "valid_minimal.json") + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + payload["execution"] = { + "files_touched": ["src/auth.py", "README.md"], + "execution_results": [ + { + "status": "failed", + "outcome_description": "Captured command output", + "reasoning": "Secret safety check", + "command": "echo token", + "evidence": "token=ghp_123456789012345678901234567890123456", + } + ], + } + with tempfile.TemporaryDirectory() as tmp_dir: + step_dir = os.path.join(tmp_dir, "step_16") + os.makedirs(step_dir, exist_ok=True) + tmp_path = os.path.join(step_dir, "invalid_sensitive_evidence.json") + with open(tmp_path, "w", encoding="utf-8") as tmp: + json.dump(payload, tmp, indent=2) + errors = validate_file(self.repo_root, tmp_path) + self.assertTrue( + any("sensitive content classes" in e for e in errors), + f"Expected sensitive evidence validation error. Errors: {errors}", + ) if __name__ == '__main__': unittest.main() diff --git a/tests/integration/test_step_16_spec_ref_grounding.py b/tests/integration/test_step_16_spec_ref_grounding.py new file mode 100644 index 00000000..dea86405 --- /dev/null +++ b/tests/integration/test_step_16_spec_ref_grounding.py @@ -0,0 +1,207 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from specdev_tools.validate import validate_file + + +class TestStep16SpecRefGrounding(unittest.TestCase): + def setUp(self): + toolkit_root = Path(__file__).resolve().parents[2] + self.repo_root = str(toolkit_root) + self.git_available = subprocess.run( + ["git", "--version"], + capture_output=True, + text=True, + check=False, + ).returncode == 0 + + def _write_json(self, path: str, payload: dict) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + def _init_project(self, tmp: str) -> tuple[str, str]: + if not self.git_available: + self.skipTest("git is required for spec_ref grounding tests") + + seed_manifest = { + "$schema": "https://specdev.local/schema/seed_manifest.schema.json", + "seed_manifest_id": "seed-manifest-core", + "version": "0.1.0", + "created_at": "2026-02-12T00:00:00Z", + "last_updated": "2026-02-12T00:00:00Z", + "global_seed_order": ["seed-overview"], + "nested_order": [], + "seeds": [ + { + "seed_id": "seed-overview", + "path": "docs/seed/seed_overview.md", + "description": "overview", + "required": True, + "source_type": "doc" + } + ], + "step_requirements": { + "16a": ["seed-overview"], + "16b": ["seed-overview"], + "16c": ["seed-overview"] + }, + "docs_policy": { + "readme_required": True, + "root_readme_required": True, + "readme_depth_default": 0, + "readme_depth_by_scope": {}, + "scope": ["."], + "exclusions": [], + "doc_paths": ["docs/**", "README.md"] + } + } + fr_list = { + "$schema": "https://specdev.local/schema/04_fr_list.schema.json", + "functional_requirements": [ + { + "id": "fr-core-login", + "title": "login", + "description": "Implement login." + } + ] + } + impl_context_path = os.path.join(tmp, "spec", "impl_context", "16_step.json") + + self._write_json(os.path.join(tmp, "spec", "common", "seed_manifest.json"), seed_manifest) + self._write_json(os.path.join(tmp, "spec", "04_fr_list.json"), fr_list) + os.makedirs(os.path.join(tmp, "docs"), exist_ok=True) + with open(os.path.join(tmp, "README.md"), "w", encoding="utf-8") as f: + f.write("# temp\n") + + subprocess.run(["git", "init"], cwd=tmp, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=tmp, check=True) + subprocess.run(["git", "config", "user.name", "Test User"], cwd=tmp, check=True) + subprocess.run(["git", "add", "."], cwd=tmp, check=True) + subprocess.run(["git", "commit", "-m", "baseline"], cwd=tmp, check=True, capture_output=True) + commit_hash = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=tmp, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + return impl_context_path, commit_hash + + def _build_impl_context(self, commit_hash: str, spec_ref_id: str, line_range: str) -> dict: + return { + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "step-test-grounding", + "owner": "api", + "created_at": "2026-02-12T00:00:00Z", + "seed_refs": [{"seed_id": "seed-overview"}], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Grounding test", + "scope_in": ["core-auth"], + "scope_out": ["oauth"], + "target_file_patterns": ["src/auth.py", "README.md"] + }, + "spec_alignment": { + "requirements_summary": [ + { + "theme": "Auth", + "summary": "Implement login requirement" + } + ], + "checklist": [ + { + "id": "REQ_AUTH_001", + "spec_ref": { + "type": "fr", + "id": spec_ref_id, + "line_range": line_range, + "commit_hash": commit_hash + }, + "description": "Implement login behavior", + "type": "behavior", + "layer": "service", + "linked_test_expectation": "pytest -q", + "nfr_refs": ["nfr-availability-uptime"], + "fixture_ref": "fixture-login-success" + } + ] + }, + "docs_impact": { + "status": "required", + "rationale": "Non-doc scope requires docs updates for traceability.", + "docs_touched": ["README.md"] + }, + "review_requirements": { + "test_commands": ["pytest -q"] + } + } + } + + def test_spec_ref_grounding_valid(self): + with tempfile.TemporaryDirectory() as tmp: + impl_context_path, commit_hash = self._init_project(tmp) + payload = self._build_impl_context(commit_hash, "fr-core-login", "L1-L5") + self._write_json(impl_context_path, payload) + errors = validate_file(self.repo_root, impl_context_path) + self.assertEqual(errors, [], f"Valid grounded spec_ref should pass. Errors: {errors}") + + def test_spec_ref_grounding_invalid_id(self): + with tempfile.TemporaryDirectory() as tmp: + impl_context_path, commit_hash = self._init_project(tmp) + payload = self._build_impl_context(commit_hash, "fr-does-not-exist", "L1-L5") + self._write_json(impl_context_path, payload) + errors = validate_file(self.repo_root, impl_context_path) + self.assertTrue( + any("id not found for type" in e for e in errors), + f"Expected authority-id grounding error. Errors: {errors}", + ) + + def test_spec_ref_grounding_invalid_commit(self): + with tempfile.TemporaryDirectory() as tmp: + impl_context_path, _ = self._init_project(tmp) + bad_commit = "f" * 40 + payload = self._build_impl_context(bad_commit, "fr-core-login", "L1-L5") + self._write_json(impl_context_path, payload) + errors = validate_file(self.repo_root, impl_context_path) + self.assertTrue( + any("commit_hash" in e and "not found in git" in e for e in errors), + f"Expected commit grounding error. Errors: {errors}", + ) + + def test_spec_ref_grounding_invalid_line_range(self): + with tempfile.TemporaryDirectory() as tmp: + impl_context_path, commit_hash = self._init_project(tmp) + payload = self._build_impl_context(commit_hash, "fr-core-login", "L999-L1000") + self._write_json(impl_context_path, payload) + errors = validate_file(self.repo_root, impl_context_path) + self.assertTrue( + any("line_range" in e and "does not map" in e for e in errors), + f"Expected line-range grounding error. Errors: {errors}", + ) + + def test_spec_ref_grounding_applies_to_real_milestone_filename(self): + with tempfile.TemporaryDirectory() as tmp: + _, _ = self._init_project(tmp) + impl_context_path = os.path.join(tmp, "spec", "impl_context", "m1-core-foundation.json") + bad_commit = "f" * 40 + payload = self._build_impl_context(bad_commit, "fr-core-login", "L1-L5") + self._write_json(impl_context_path, payload) + errors = validate_file(self.repo_root, impl_context_path) + self.assertTrue( + any("commit_hash" in e and "not found in git" in e for e in errors), + f"Expected commit grounding error for milestone filename. Errors: {errors}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_trinity_eval_replay.py b/tests/integration/test_trinity_eval_replay.py new file mode 100644 index 00000000..9365e67a --- /dev/null +++ b/tests/integration/test_trinity_eval_replay.py @@ -0,0 +1,680 @@ +import hashlib +import http.server +import json +import os +import socketserver +import sys +import tempfile +import threading +import unittest +import re +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from specdev_tools.trinity_eval_export import export_eval_rows +from specdev_tools.trinity_replay import replay_session +from specdev_tools.trinity_dashboard import write_dashboard +from specdev_tools.trinity_remediation import build_remediation_plan +from specdev_tools.trinity_runtime_validate import validate_runtime_file +from specdev_tools.trinity_eval_publish import publish_eval_bundle + + +class TestTrinityEvalReplay(unittest.TestCase): + def setUp(self): + toolkit_root = Path(__file__).resolve().parents[2] + self.repo_root = str(toolkit_root) + self.tool_call_request_schema_sha = self._schema_sha_from_rel("schema/trinity/tool_call_request.schema.json") + self.tool_call_result_schema_sha = self._schema_sha_from_rel("schema/trinity/tool_call_result.schema.json") + + def _schema_sha_from_rel(self, rel_path: str) -> str: + abs_path = os.path.join(self.repo_root, rel_path) + with open(abs_path, "r", encoding="utf-8") as f: + payload = json.load(f) + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def _sha256_text(self, value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + def _sha256_file(self, path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + def _event_sha256(self, event: dict) -> str: + payload = dict(event) + payload["event_sha256"] = None + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def _git_head_commit(self) -> str: + import subprocess + + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + self.skipTest("git is required for trinity runtime grounding tests") + return result.stdout.strip() + + def _first_id_line(self, rel_path: str) -> tuple[str, int]: + abs_path = os.path.join(self.repo_root, rel_path) + with open(abs_path, "r", encoding="utf-8") as f: + for idx, line in enumerate(f, start=1): + match = re.search(r'"id"\s*:\s*"([^"]+)"', line) + if match: + return match.group(1), idx + self.fail(f"No id field found in {rel_path}") + + def _base_metadata(self) -> dict: + return { + "toolkit_version": "0.2.3", + "schema_version": "v1", + "git_head": "deadbeef", + "prompt_template_id": "prompt-16b", + "prompt_template_sha256": "a" * 64, + "redaction_profile": "eval", + "redaction_applied": False, + "capture_policy_ref": None, + "capture_policy_sha256": None, + "redaction_stats": { + "total_replacements": 0, + "by_class": {}, + "classes_detected": [], + "detectors_used": ["secret_scanner_v1"], + "min_confidence": 0.0, + "max_confidence": 0.0, + }, + "decoding": {"temperature": 0.2, "top_p": 0.9, "max_tokens": 4096}, + "token_usage": {"prompt": 100, "completion": 50, "total": 150}, + "tool_schema_context": { + "mode": "full_inline", + "catalog_ref": None, + "catalog_sha256": None, + "expanded_tool_names": [], + "request_schema_uri": "https://specdev.local/schema/trinity/tool_call_request.schema.json", + "request_schema_sha256": self.tool_call_request_schema_sha, + "result_schema_uri": "https://specdev.local/schema/trinity/tool_call_result.schema.json", + "result_schema_sha256": self.tool_call_result_schema_sha, + }, + } + + def _build_events(self, tmp: str, mutate_after_log: bool = False) -> tuple[str, list[dict]]: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + captures_dir = os.path.join(tmp, ".trinity", "captures") + os.makedirs(captures_dir, exist_ok=True) + spawns_dir = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1") + os.makedirs(spawns_dir, exist_ok=True) + + artifact_path = os.path.join(tmp, "artifact.json") + with open(artifact_path, "w", encoding="utf-8") as f: + json.dump({"ok": True}, f) + artifact_sha = self._sha256_file(artifact_path) + + commit_hash = self._git_head_commit() + api_id, api_line = self._first_id_line("spec/05_interface_contracts.json") + context_pack_path = os.path.join(tmp, ".trinity", "runtime", "context_pack.json") + with open(context_pack_path, "w", encoding="utf-8") as f: + json.dump( + { + "protocol_version": "trinity-runtime-v1", + "phase": "16a", + "step_id": "m1-core-foundation", + "seed_manifest_path": "spec/common/seed_manifest.json", + "seed_files_ordered": ["docs/seed/seed_overview.md", "docs/seed/seed_tech_stack.md"], + "required_spec_refs": [ + { + "type": "api", + "id": api_id, + "path": "spec/05_interface_contracts.json", + "line_range": f"L{api_line}-L{api_line}", + "commit_hash": commit_hash, + } + ], + "artifact_refs": { + "milestone_context_path": "spec/impl_context/m1-core-foundation.json", + }, + "allowed_read_paths": ["spec/", "src/", "tests/"], + "allowed_write_paths": ["src/", "tests/", "spec/impl_context/", "README.md"], + "target_file_patterns": ["src/*.py"], + "docs_policy": {"doc_paths": ["docs/**", "README.md"]}, + }, + f, + indent=2, + ) + + spawn_task_input_path = os.path.join(spawns_dir, "task_input.json") + with open(spawn_task_input_path, "w", encoding="utf-8") as f: + json.dump( + { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "parent_id": "agent-root", + "role": "Planner", + "phase": "16a", + "step_id": "m1-core-foundation", + "task_description": "Plan implementation for core foundation", + "expected_output_schema": "https://specdev.local/schema/trinity/task_result.schema.json", + "context_pack_ref": context_pack_path, + "target_files": ["src/core.py"], + "spec_refs": [{"type": "api", "id": api_id}], + "role_metadata": { + "prompt_source": "prompt_16a_impl_planner.md", + "persona_goal": "produce actionable plan", + "stop_conditions": ["plan complete"], + }, + }, + f, + indent=2, + ) + + prompt_path = os.path.join(captures_dir, "prompt_evt-2.txt") + response_path = os.path.join(captures_dir, "response_evt-2.txt") + prompt_text = "Prompt content for replay/export" + response_text = "Response content for replay/export" + with open(prompt_path, "w", encoding="utf-8") as f: + f.write(prompt_text) + with open(response_path, "w", encoding="utf-8") as f: + f.write(response_text) + prompt_sha = self._sha256_text(prompt_text) + response_sha = self._sha256_text(response_text) + task_result_path = os.path.join(spawns_dir, "task_result.json") + with open(task_result_path, "w", encoding="utf-8") as f: + json.dump( + { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Planner", + "phase": "16a", + "step_id": "m1-core-foundation", + "status": "blocked", + "summary": "Blocked waiting for clarification", + "artifacts": [], + "findings": [ + { + "id": "amb-missing-requirement", + "type": "gap", + "severity": "blocking", + "description": "Missing deterministic requirement detail", + "source": "replay-test", + "impact": "planning", + } + ], + }, + f, + indent=2, + ) + + event_1 = { + "schema_version": "trinity-session-log-v1", + "timestamp": "2026-02-13T00:00:00Z", + "event_type": "SPAWN", + "event_id": "evt-1", + "event_sequence": 1, + "prev_event_sha256": None, + "event_sha256": "0" * 64, + "run_id": "run-1", + "phase_id": "phase-16a", + "loop_id": "loop-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "step_id": "m1-core-foundation", + "tool_call_id": None, + "result_id": None, + "artifact_ref": artifact_path, + "artifact_sha256": artifact_sha, + "diff_ref": None, + "model": "gpt-5", + "content": { + "summary": "spawn planner", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_input_artifact_ref": spawn_task_input_path, + }, + "metadata": self._base_metadata(), + } + event_1["event_sha256"] = self._event_sha256(event_1) + + event_2 = { + "schema_version": "trinity-session-log-v1", + "timestamp": "2026-02-13T00:00:01Z", + "event_type": "MESSAGE", + "event_id": "evt-2", + "event_sequence": 2, + "prev_event_sha256": event_1["event_sha256"], + "event_sha256": "0" * 64, + "run_id": "run-1", + "phase_id": "phase-16b", + "loop_id": "loop-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "step_id": "m1-core-foundation", + "tool_call_id": None, + "result_id": None, + "artifact_ref": artifact_path, + "artifact_sha256": artifact_sha, + "diff_ref": None, + "model": "gpt-5", + "content": { + "summary": "captured full prompt/response sample", + "capture_level": "full", + "capture_decision_reason": "policy:sampled:MESSAGE", + "prompt_artifact_ref": prompt_path, + "prompt_sha256": prompt_sha, + "response_artifact_ref": response_path, + "response_sha256": response_sha, + }, + "metadata": self._base_metadata(), + } + event_2["event_sha256"] = self._event_sha256(event_2) + + event_3 = { + "schema_version": "trinity-session-log-v1", + "timestamp": "2026-02-13T00:00:02Z", + "event_type": "VALIDATION", + "event_id": "evt-3", + "event_sequence": 3, + "prev_event_sha256": event_2["event_sha256"], + "event_sha256": "0" * 64, + "run_id": "run-1", + "phase_id": "phase-16a", + "loop_id": "loop-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "step_id": "m1-core-foundation", + "tool_call_id": None, + "result_id": None, + "artifact_ref": task_result_path, + "artifact_sha256": self._sha256_file(task_result_path), + "diff_ref": None, + "model": "gpt-5", + "content": { + "summary": "validated child task_input", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_input_artifact_ref": spawn_task_input_path, + "validation": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + "seed_lint": "n/a", + "docs_lint": "n/a", + }, + }, + "metadata": self._base_metadata(), + } + event_3["event_sha256"] = self._event_sha256(event_3) + + event_4 = { + "schema_version": "trinity-session-log-v1", + "timestamp": "2026-02-13T00:00:03Z", + "event_type": "VALIDATION", + "event_id": "evt-4", + "event_sequence": 4, + "prev_event_sha256": event_3["event_sha256"], + "event_sha256": "0" * 64, + "run_id": "run-1", + "phase_id": "phase-16a", + "loop_id": "loop-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "step_id": "m1-core-foundation", + "tool_call_id": None, + "result_id": None, + "artifact_ref": task_result_path, + "artifact_sha256": self._sha256_file(task_result_path), + "diff_ref": None, + "model": "gpt-5", + "content": { + "summary": "validated child task_result", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_result_artifact_ref": task_result_path, + "validation": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + "seed_lint": "n/a", + "docs_lint": "n/a", + }, + }, + "metadata": self._base_metadata(), + } + event_4["event_sha256"] = self._event_sha256(event_4) + + event_5 = { + "schema_version": "trinity-session-log-v1", + "timestamp": "2026-02-13T00:00:04Z", + "event_type": "TERMINATE", + "event_id": "evt-5", + "event_sequence": 5, + "prev_event_sha256": event_4["event_sha256"], + "event_sha256": "0" * 64, + "run_id": "run-1", + "phase_id": "phase-16a", + "loop_id": "loop-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "step_id": "m1-core-foundation", + "tool_call_id": None, + "result_id": None, + "artifact_ref": task_result_path, + "artifact_sha256": self._sha256_file(task_result_path), + "diff_ref": None, + "model": "gpt-5", + "content": { + "summary": "planner child completed", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_result_artifact_ref": task_result_path, + }, + "metadata": self._base_metadata(), + } + event_5["event_sha256"] = self._event_sha256(event_5) + + session_log = os.path.join(sessions_dir, "session.jsonl") + with open(session_log, "w", encoding="utf-8") as f: + f.write(json.dumps(event_1) + "\n") + f.write(json.dumps(event_2) + "\n") + f.write(json.dumps(event_3) + "\n") + f.write(json.dumps(event_4) + "\n") + f.write(json.dumps(event_5) + "\n") + + if mutate_after_log: + with open(prompt_path, "w", encoding="utf-8") as f: + f.write("tampered") + + return session_log, [event_1, event_2, event_3, event_4, event_5] + + def test_export_eval_rows_from_session_log(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + runtime_errors = validate_runtime_file(self.repo_root, session_log, "session_event") + self.assertEqual(runtime_errors, [], f"Session log should validate. Errors: {runtime_errors}") + + out_path = os.path.join(tmp, "eval_rows.jsonl") + rows, errors = export_eval_rows(self.repo_root, session_log, out_path=out_path) + self.assertEqual(errors, [], f"Eval export should succeed. Errors: {errors}") + self.assertEqual(len(rows), 5, "Expected one eval row per event") + + with open(out_path, "r", encoding="utf-8") as f: + exported_lines = [line for line in f.read().splitlines() if line.strip()] + self.assertEqual(len(exported_lines), 5, "Output JSONL should contain five rows") + self.assertEqual(rows[0]["event_id"], "evt-1") + self.assertEqual(rows[1]["capture_level"], "full") + self.assertEqual(rows[-1]["phase_outcome"], "blocked") + self.assertEqual(rows[-1]["max_finding_severity"], "blocking") + self.assertTrue(rows[-1]["remediation_required"]) + + def test_replay_session_strict_ok(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + report = replay_session(self.repo_root, session_log, strict=True) + self.assertEqual(report["status"], "ok", f"Strict replay should pass. Report: {report}") + self.assertEqual(report["artifact_verification"]["mismatch"], 0) + self.assertEqual(report["artifact_verification"]["missing"], 0) + + def test_replay_session_detects_artifact_tamper(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp, mutate_after_log=True) + report_warn = replay_session(self.repo_root, session_log, strict=False) + self.assertEqual(report_warn["status"], "warnings", f"Replay should warn on tamper. Report: {report_warn}") + self.assertGreater(report_warn["artifact_verification"]["mismatch"], 0) + + report_strict = replay_session(self.repo_root, session_log, strict=True) + self.assertEqual(report_strict["status"], "failed", f"Strict replay should fail on tamper. Report: {report_strict}") + + def test_dashboard_aggregation(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + rows_out = os.path.join(tmp, "eval_rows.jsonl") + rows, errors = export_eval_rows(self.repo_root, session_log, out_path=rows_out) + self.assertEqual(errors, [], f"Export should succeed. Errors: {errors}") + self.assertEqual(len(rows), 5) + + replay_out = os.path.join(tmp, "replay.json") + report = replay_session(self.repo_root, session_log, strict=False) + with open(replay_out, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + + out_json = os.path.join(tmp, "dashboard.json") + out_md = os.path.join(tmp, "dashboard.md") + summary, markdown = write_dashboard( + eval_rows_glob=os.path.join(tmp, "eval_rows*.jsonl"), + replay_reports_glob=os.path.join(tmp, "replay*.json"), + out_json=out_json, + out_md=out_md, + ) + self.assertEqual(summary["totals"]["eval_rows"], 5) + self.assertIn("Trinity Eval Dashboard", markdown) + self.assertTrue(os.path.exists(out_json)) + self.assertTrue(os.path.exists(out_md)) + + def test_remediation_plan_generation_with_resume_outputs(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + replay_report = replay_session(self.repo_root, session_log, strict=False) + replay_report_path = os.path.join(tmp, "replay_report.json") + with open(replay_report_path, "w", encoding="utf-8") as f: + json.dump(replay_report, f, indent=2) + + out_state = os.path.join(tmp, "session_state_resume.json") + out_task_input = os.path.join(tmp, "task_input_resume.json") + plan, errors = build_remediation_plan( + repo_root=self.repo_root, + replay_report_path=replay_report_path, + session_log_path=session_log, + emit_session_state_path=out_state, + emit_task_input_path=out_task_input, + ) + self.assertEqual(errors, [], f"Remediation should produce valid resume artifacts. Errors: {errors}") + self.assertIn("actions", plan) + self.assertTrue(os.path.exists(out_state)) + self.assertTrue(os.path.exists(out_task_input)) + self.assertEqual(plan["status"], "ready") + + def test_publish_eval_bundle_local_file(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + rows_out = os.path.join(tmp, "eval_rows.jsonl") + replay_out = os.path.join(tmp, "replay.json") + dashboard_out = os.path.join(tmp, "dashboard.json") + + rows, errors = export_eval_rows(self.repo_root, session_log, out_path=rows_out) + self.assertEqual(errors, [], f"Eval export should succeed. Errors: {errors}") + self.assertEqual(len(rows), 5) + + report = replay_session(self.repo_root, session_log, strict=False) + with open(replay_out, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + summary, _ = write_dashboard( + eval_rows_glob=rows_out, + replay_reports_glob=replay_out, + out_json=dashboard_out, + out_md=None, + ) + self.assertIn("totals", summary) + + bundle_out = os.path.join(tmp, "export_bundle.json") + bundle, publish_result, publish_errors = publish_eval_bundle( + rows_glob=rows_out, + replay_glob=replay_out, + dashboard_json=dashboard_out, + out_path=bundle_out, + source="tests", + ) + self.assertEqual(publish_errors, [], f"Local bundle generation should succeed. Errors: {publish_errors}") + self.assertEqual(publish_result["status"], "skipped") + self.assertEqual(bundle["row_count_exported"], 5) + self.assertTrue(os.path.exists(bundle_out)) + + def test_publish_eval_bundle_http_endpoint(self): + class _Handler(http.server.BaseHTTPRequestHandler): + received = {"count": 0, "body": None, "auth": None} + + def do_POST(self): # type: ignore[override] + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + _Handler.received["count"] += 1 + _Handler.received["body"] = body.decode("utf-8") + _Handler.received["auth"] = self.headers.get("Authorization") + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, format, *args): # noqa: A003 + return + + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + rows_out = os.path.join(tmp, "eval_rows.jsonl") + replay_out = os.path.join(tmp, "replay.json") + export_eval_rows(self.repo_root, session_log, out_path=rows_out) + report = replay_session(self.repo_root, session_log, strict=False) + with open(replay_out, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + + class ReuseTCPServer(socketserver.TCPServer): + allow_reuse_address = True + + with ReuseTCPServer(("127.0.0.1", 0), _Handler) as server: + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + bundle, publish_result, publish_errors = publish_eval_bundle( + rows_glob=rows_out, + replay_glob=replay_out, + source="tests", + endpoint=f"http://127.0.0.1:{port}/ingest", + auth_token="secret-token", + ) + finally: + server.shutdown() + thread.join(timeout=5) + + self.assertEqual(publish_errors, [], f"HTTP publish should succeed. Errors: {publish_errors}") + self.assertEqual(publish_result["status"], "published") + self.assertEqual(_Handler.received["count"], 1) + self.assertIn('"schema_version": "trinity-eval-export-v1"', _Handler.received["body"]) + self.assertEqual(_Handler.received["auth"], "Bearer secret-token") + + def test_remediation_missing_resume_source_soft_policy(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + spawn_task_input = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1", "task_input.json") + os.remove(spawn_task_input) + + replay_report = replay_session(self.repo_root, session_log, strict=False) + replay_report_path = os.path.join(tmp, "replay_report.json") + with open(replay_report_path, "w", encoding="utf-8") as f: + json.dump(replay_report, f, indent=2) + + out_state = os.path.join(tmp, "session_state_resume.json") + out_task_input = os.path.join(tmp, "task_input_resume.json") + plan, errors = build_remediation_plan( + repo_root=self.repo_root, + replay_report_path=replay_report_path, + session_log_path=session_log, + emit_session_state_path=out_state, + emit_task_input_path=out_task_input, + missing_resume_source_policy="soft", + ) + self.assertEqual(errors, [], f"Soft policy should not fail remediation. Errors: {errors}") + self.assertEqual(plan["status"], "ready_with_warnings") + self.assertTrue(plan.get("warnings"), "Soft policy should record warnings for missing source artifacts") + self.assertTrue(os.path.exists(out_state), "Session state should still be emitted in soft mode") + self.assertFalse(os.path.exists(out_task_input), "Task input should not be emitted when source is missing") + + def test_remediation_missing_resume_source_hard_policy(self): + with tempfile.TemporaryDirectory() as tmp: + session_log, _ = self._build_events(tmp) + spawn_task_input = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1", "task_input.json") + os.remove(spawn_task_input) + + replay_report = replay_session(self.repo_root, session_log, strict=False) + replay_report_path = os.path.join(tmp, "replay_report.json") + with open(replay_report_path, "w", encoding="utf-8") as f: + json.dump(replay_report, f, indent=2) + + out_state = os.path.join(tmp, "session_state_resume.json") + out_task_input = os.path.join(tmp, "task_input_resume.json") + plan, errors = build_remediation_plan( + repo_root=self.repo_root, + replay_report_path=replay_report_path, + session_log_path=session_log, + emit_session_state_path=out_state, + emit_task_input_path=out_task_input, + missing_resume_source_policy="hard", + ) + self.assertTrue(errors, "Hard policy should fail remediation on missing source artifacts") + self.assertEqual(plan["status"], "needs_attention") + + def test_remediation_session_state_requires_lineage_fields(self): + with tempfile.TemporaryDirectory() as tmp: + replay_report_path = os.path.join(tmp, "replay_report.json") + with open(replay_report_path, "w", encoding="utf-8") as f: + json.dump( + { + "status": "warnings", + "summary": {"run_id": "run-1", "step_ids": []}, + "timeline": [ + { + "event_id": "evt-1", + "event_type": "MESSAGE", + "phase_id": "phase-unknown", + "agent_id": None, + } + ], + "warnings": ["synthetic warning"], + "errors": [], + }, + f, + indent=2, + ) + + out_state = os.path.join(tmp, "session_state_resume.json") + plan, errors = build_remediation_plan( + repo_root=self.repo_root, + replay_report_path=replay_report_path, + emit_session_state_path=out_state, + ) + self.assertTrue(errors, "Lineage gaps should fail remediation state generation.") + self.assertEqual(plan["status"], "needs_attention") + self.assertFalse(os.path.exists(out_state), "Session state must not be emitted with missing lineage.") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_trinity_runtime_orchestration.py b/tests/integration/test_trinity_runtime_orchestration.py new file mode 100644 index 00000000..be1b43ba --- /dev/null +++ b/tests/integration/test_trinity_runtime_orchestration.py @@ -0,0 +1,1664 @@ +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Optional +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from specdev_tools.trinity_runtime import SessionLogger, ToolExecutor, TrinityConfig, TrinityRuntime, run_trinity +from specdev_tools.trinity_runtime_validate import validate_runtime_file +from specdev_tools.validate import validate_file + + +class TestTrinityRuntimeOrchestration(unittest.TestCase): + def setUp(self): + self.toolkit_root = str(Path(__file__).resolve().parents[2]) + + def _write_json(self, path: str, payload: dict) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + f.write("\n") + + def _write_text(self, path: str, content: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(content) + + def _loop_checkpoint(self, label: str) -> dict: + return { + "draft": f"{label} draft evidence captured from grounded inputs.", + "review": f"{label} review evidence confirms contract alignment.", + "refine": f"{label} refine evidence captures final adjustments.", + } + + def _init_git_repo(self, repo_root: str) -> None: + subprocess.run(["git", "init"], cwd=repo_root, check=True, capture_output=True, text=True) + subprocess.run(["git", "config", "user.name", "Trinity Test"], cwd=repo_root, check=True, capture_output=True, text=True) + subprocess.run(["git", "config", "user.email", "trinity-test@example.com"], cwd=repo_root, check=True, capture_output=True, text=True) + subprocess.run(["git", "add", "-A"], cwd=repo_root, check=True, capture_output=True, text=True) + subprocess.run(["git", "commit", "-m", "baseline"], cwd=repo_root, check=True, capture_output=True, text=True) + + def _start_fake_openai_server(self): + calls = [] + outer_self = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): # noqa: A003 + return + + def do_POST(self): # noqa: N802 + raw_len = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(raw_len).decode("utf-8") + payload = json.loads(raw) + messages = payload.get("messages", []) if isinstance(payload.get("messages"), list) else [] + phase = "unknown" + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + if isinstance(candidate, dict) and isinstance(candidate.get("task_input"), dict): + maybe_phase = candidate["task_input"].get("phase") + if isinstance(maybe_phase, str) and maybe_phase: + phase = maybe_phase + break + calls.append({"path": self.path, "phase": phase}) + + response_payload = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1700000000, + "model": "input-model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": json.dumps( + { + "action": "final_result", + "summary": f"{phase} completed by fake llm", + "loop_checkpoint": outer_self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + ), + }, + } + ], + "usage": {"prompt_tokens": 120, "completion_tokens": 48, "total_tokens": 168}, + } + data = json.dumps(response_payload).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, calls + + def _start_scripted_openai_server(self, responder): + calls = [] + state = {"turn": 0} + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): # noqa: A003 + return + + def do_POST(self): # noqa: N802 + raw_len = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(raw_len).decode("utf-8") + payload = json.loads(raw) + messages = payload.get("messages", []) if isinstance(payload.get("messages"), list) else [] + state["turn"] = int(state.get("turn", 0)) + 1 + reply = responder(messages, state) + calls.append({"turn": state["turn"], "reply": reply}) + response_payload = { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "created": 1700000000 + state["turn"], + "model": "input-model", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": json.dumps(reply)}}], + "usage": {"prompt_tokens": 120, "completion_tokens": 48, "total_tokens": 168}, + } + data = json.dumps(response_payload).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, calls + + def _create_fixture_repo( + self, + repo_root: str, + allow_dirty: bool = False, + checkpoint_commits: bool = False, + conformance_mode: Optional[bool] = None, + execution_mode: str = "deterministic", + preverified_artifact: bool = False, + retry_cap_planner: int = 10, + retry_cap_builder: int = 10, + retry_cap_verifier: int = 10, + retry_cap_milestone: int = 10, + ) -> None: + shutil.copytree(os.path.join(self.toolkit_root, "schema"), os.path.join(repo_root, "schema")) + shutil.copytree(os.path.join(self.toolkit_root, "prompts"), os.path.join(repo_root, "prompts")) + os.makedirs(os.path.join(repo_root, "tools"), exist_ok=True) + shutil.copy2( + os.path.join(self.toolkit_root, "tools", "schema_registry.json"), + os.path.join(repo_root, "tools", "schema_registry.json"), + ) + + self._write_text(os.path.join(repo_root, "docs", "seed", "seed_overview.md"), "# overview\n") + self._write_text(os.path.join(repo_root, "docs", "seed", "seed_tech_stack.md"), "# tech stack\n") + self._write_text(os.path.join(repo_root, "README.md"), "Fixture repository for Trinity runtime.\n") + + self._write_json( + os.path.join(repo_root, "spec", "common", "seed_manifest.json"), + { + "$schema": "https://specdev.local/schema/seed_manifest.schema.json", + "seed_manifest_id": "seed-manifest-core", + "version": "0.1.0", + "created_at": "2026-02-07T00:00:00Z", + "last_updated": "2026-02-07T00:00:00Z", + "global_seed_order": ["seed-overview", "seed-tech-stack"], + "seeds": [ + {"seed_id": "seed-overview", "path": "docs/seed/seed_overview.md", "required": True, "source_type": "doc"}, + {"seed_id": "seed-tech-stack", "path": "docs/seed/seed_tech_stack.md", "required": True, "source_type": "doc"}, + ], + "step_requirements": { + "16a": ["seed-overview", "seed-tech-stack"], + "16b": ["seed-overview", "seed-tech-stack"], + "16c": ["seed-overview", "seed-tech-stack"], + }, + "docs_policy": { + "readme_required": True, + "root_readme_required": True, + "scope": ["."], + "exclusions": [".git/"], + "doc_paths": ["docs/**", "README.md", "CHANGELOG.md"], + }, + }, + ) + + interface_contracts_path = os.path.join(repo_root, "spec", "05_interface_contracts.json") + self._write_json( + interface_contracts_path, + { + "$schema": "https://specdev.local/schema/05_interface_contracts.schema.json", + "id": "interface-contracts", + "owner": "api", + "created_at": "2025-01-01T00:00:00Z", + "seed_refs": [{"seed_id": "seed-overview"}, {"seed_id": "seed-tech-stack"}], + "apis": [ + { + "api_id": "api-trinity-bootstrap", + "name": "Trinity Bootstrap", + "version": "v1", + "protocol": "http", + "route": "/trinity", + "method": "GET", + "owner": "api", + } + ], + }, + ) + with open(interface_contracts_path, "r", encoding="utf-8") as f: + contract_lines = f.readlines() + interface_id_line = 1 + for idx, line in enumerate(contract_lines, start=1): + if '"id": "interface-contracts"' in line: + interface_id_line = idx + break + interface_id_range = f"L{interface_id_line}-L{interface_id_line}" + + self._write_json( + os.path.join(repo_root, "spec", "14_roadmap.json"), + { + "$schema": "https://specdev.local/schema/14_roadmap.schema.json", + "id": "roadmap-core", + "owner": "api", + "created_at": "2026-02-10T00:00:00Z", + "seed_refs": [{"seed_id": "seed-overview"}], + "tech_stack": {"languages": [{"name": "python", "version": "3.11"}], "frameworks": [{"name": "stdlib", "version": "1"}]}, + "milestones": [ + { + "milestone_id": "m1-core-foundation", + "name": "Core Foundation", + "target_date": "2026-03-01", + "status": "pending", + "user_story": "As an engineer, I can run Trinity on one milestone.", + "source_milestones": ["m0-source"], + "tasks": [{"task_id": "task-bootstrap", "description": "bootstrap milestone context"}], + "deliverables": [{"type": "api", "id": "interface-contracts"}], + } + ], + "dependencies": [], + }, + ) + + if conformance_mode is None: + conformance_mode = bool(checkpoint_commits) + self._write_text( + os.path.join(repo_root, ".trinity", "trinity.yaml"), + ( + "llm:\n" + " api_base: \"http://localhost:1234/v1\"\n" + " model: \"input-model\"\n" + " timeout: 300\n\n" + "limits:\n" + " soft_token_limit: 60000\n" + " hard_token_limit: 80000\n" + f" max_loops: {retry_cap_milestone}\n\n" + "runtime:\n" + f" allow_dirty: {'true' if allow_dirty else 'false'}\n" + f" checkpoint_commits: {'true' if checkpoint_commits else 'false'}\n" + f" conformance_mode: {'true' if conformance_mode else 'false'}\n" + f" execution_mode: \"{execution_mode}\"\n" + " max_child_turns: 6\n" + " retry_caps:\n" + f" planner: {retry_cap_planner}\n" + f" builder: {retry_cap_builder}\n" + f" verifier: {retry_cap_verifier}\n" + f" milestone: {retry_cap_milestone}\n" + ), + ) + + self._init_git_repo(repo_root) + base_commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + smoke_cmd = "python3 -c \"print('SUCCESS TRINITY_OK evidence marker')\"" + evidence = "SUCCESS TRINITY_OK evidence marker" + evidence_sha = hashlib.sha256(evidence.encode("utf-8")).hexdigest() + implementation_status = "verified" if preverified_artifact else "pending" + run_action_evidence = ( + {"type": "snippet", "content": evidence, "evidence_ref": f"sha256:{evidence_sha}"} + if preverified_artifact + else None + ) + verify_action_evidence = ( + {"type": "snippet", "content": evidence, "evidence_ref": f"sha256:{evidence_sha}"} + if preverified_artifact + else None + ) + execution_block = ( + { + "files_touched": ["README.md"], + "execution_results": [ + { + "status": "passed", + "outcome_description": "Command passed", + "reasoning": "Pre-seeded execution evidence for llm mode fixture.", + "command": smoke_cmd, + "evidence": evidence, + "evidence_ref": f"sha256:{evidence_sha}", + "evidence_binding": { + "timestamp": "2026-02-13T00:00:00Z", + "sha256": evidence_sha, + "exit_code": 0, + "command": smoke_cmd, + }, + } + ], + "critical_evidence": { + "satisfied_checklist_ids": ["CHK_TRINITY_RUNTIME_001"], + "passed_test_commands": [smoke_cmd], + }, + } + if preverified_artifact + else None + ) + review_block = ( + { + "findings": [], + "ratings": { + "spec_completeness": 5, + "code_quality": 5, + "tests_completeness": 5, + "docs_completeness": 5, + "metadata_usage": 5, + }, + "verdict": "verified", + "next_actions": "Milestone verified.", + "fixture_status": { + "implemented_endpoints": [], + "test_results": [{"fixture_ref": "fixture-trinity-runtime-smoke", "status": "pass"}], + "ci_status": "green", + }, + } + if preverified_artifact + else None + ) + self._write_json( + os.path.join(repo_root, "spec", "impl_context", "m1-core-foundation.json"), + { + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "m1-core-foundation", + "owner": "api", + "created_at": "2026-02-10T00:00:00Z", + "seed_refs": [{"seed_id": "seed-overview"}, {"seed_id": "seed-tech-stack"}], + "plan": { + "status": "active", + "summary": { + "functional_summary": "Execute deterministic Trinity vertical-slice implementation checks.", + "scope_in": ["milestone execution lifecycle"], + "scope_out": ["cross-milestone feature work"], + "target_file_patterns": ["docs/**", "README.md", "spec/impl_context/*.json", "spec/16_impl_context.json"], + }, + "docs_impact": { + "status": "required", + "rationale": "Step 16 contract requires docs impact tracking for non-doc target scope.", + "docs_touched": ["README.md"], + }, + "spec_alignment": { + "checklist": [ + { + "id": "CHK_TRINITY_RUNTIME_001", + "spec_ref": { + "type": "api", + "id": "interface-contracts", + "line_range": interface_id_range, + "commit_hash": base_commit, + }, + "description": "Builder executes deterministic smoke command and binds evidence for verification.", + "type": "validation", + "layer": "integration", + "checklist_status": "active", + "linked_test_expectation": smoke_cmd, + "nfr_refs": ["nfr-runtime-determinism"], + "fixture_ref": "fixture-trinity-runtime-smoke", + "implementation": { + "status": implementation_status, + "actions": [ + { + "type": "run_command", + "description": "Execute deterministic smoke command with explicit success marker.", + "command": smoke_cmd, + **({"evidence": run_action_evidence} if run_action_evidence else {}), + }, + { + "type": "manual_verification", + "description": "Bind manual verification to command evidence for checklist closure.", + **({"evidence": verify_action_evidence} if verify_action_evidence else {}), + }, + ], + }, + } + ] + }, + "review_requirements": { + "guidelines": "Require explicit success marker evidence for all passed command outputs.", + "test_commands": [smoke_cmd], + }, + "solution": { + "architecture_sketch": "Single-checklist deterministic runtime validation plan.", + "sequence_of_concerns": ["16a", "16b", "16c"], + "risks": ["false positives if evidence markers are not enforced"], + }, + }, + **({"execution": execution_block} if execution_block else {}), + **({"review": review_block} if review_block else {}), + }, + ) + subprocess.run(["git", "add", "-A"], cwd=repo_root, check=True, capture_output=True, text=True) + subprocess.run(["git", "commit", "-m", "milestone fixture"], cwd=repo_root, check=True, capture_output=True, text=True) + + def test_run_trinity_full_runtime_vertical_slice(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False) + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + + self.assertEqual(result.get("status"), "completed", result) + self.assertEqual(result.get("step_id"), "m1-core-foundation") + self.assertEqual(result.get("verdict"), "verified") + + milestone_path = os.path.join(tmp, result["milestone_artifact"]) + anchor_path = os.path.join(tmp, result["anchor_artifact"]) + session_path = os.path.join(tmp, result["session_log"]) + + self.assertTrue(os.path.exists(milestone_path), "Milestone artifact should be created") + self.assertTrue(os.path.exists(anchor_path), "Anchor artifact should be created") + self.assertTrue(os.path.exists(session_path), "Session log should be created") + + milestone_errors = validate_file(tmp, milestone_path) + self.assertEqual(milestone_errors, [], f"Milestone artifact must pass Step 16 validation: {milestone_errors}") + + session_errors = validate_runtime_file(tmp, session_path, "session_event") + self.assertEqual(session_errors, [], f"Session log must pass runtime validation: {session_errors}") + + with open(os.path.join(tmp, "spec", "14_roadmap.json"), "r", encoding="utf-8") as f: + roadmap = json.load(f) + milestone = roadmap["milestones"][0] + self.assertEqual(milestone.get("status"), "done", "Roadmap milestone should be synced to done on verified closure") + + def test_run_trinity_blocks_on_dirty_tree_when_not_allowed(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False) + self._write_text(os.path.join(tmp, "notes.txt"), "dirty\n") + with self.assertRaises(RuntimeError): + run_trinity(repo_root=tmp, step_id="m1-core-foundation") + + def test_cli_trinity_command_json_output(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False) + cmd = [ + sys.executable, + "-m", + "specdev_tools.cli", + "trinity", + "--step-id", + "m1-core-foundation", + "--repo-root", + tmp, + "--json", + ] + env = os.environ.copy() + current = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + os.path.join(self.toolkit_root, "tools") + if not current + else os.path.join(self.toolkit_root, "tools") + os.pathsep + current + ) + env["SPECDEV_SKIP_VENV_CHECK"] = "1" + proc = subprocess.run(cmd, capture_output=True, text=True, check=False, env=env) + self.assertEqual(proc.returncode, 0, proc.stderr) + payload = json.loads(proc.stdout) + self.assertEqual(payload.get("status"), "completed", payload) + self.assertEqual(payload.get("step_id"), "m1-core-foundation") + + def test_run_trinity_fallback_step_selection(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False) + result = run_trinity(repo_root=tmp, step_id=None) + self.assertEqual(result.get("status"), "completed", result) + self.assertEqual(result.get("step_id"), "m1-core-foundation") + + def test_run_trinity_with_checkpoints_enabled(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + config = TrinityConfig.load(tmp) + runtime = TrinityRuntime(tmp, config, step_id="m1-core-foundation") + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16a", + step_id="m1-core-foundation", + allowed_read_paths=["."], + allowed_write_paths=["."], + enable_checkpoints=False, + ) + runtime._ensure_branch(tools) + request_path = os.path.join(tmp, ".trinity", "runtime", "tools", "tool_call_request.json") + with open(request_path, "r", encoding="utf-8") as f: + request = json.load(f) + self.assertEqual(request.get("tool_name"), "checkpoint_branch") + self.assertEqual(request.get("args", {}).get("branch_name"), "trinity/m1-core-foundation") + + def test_exec_cmd_blocks_secret_dump_patterns(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16c", + step_id="m1-core-foundation", + allowed_read_paths=["."], + allowed_write_paths=["."], + enable_checkpoints=False, + ) + blocked = tools.call( + "exec_cmd", + {"command": "printenv", "mode": "summarized", "timeout_seconds": 30}, + role="Builder", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(blocked.get("status"), "blocked", blocked) + self.assertEqual((blocked.get("error") or {}).get("code"), "blocked", blocked) + + blocked_env = tools.call( + "exec_cmd", + {"command": "env", "mode": "summarized", "timeout_seconds": 30}, + role="Builder", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(blocked_env.get("status"), "blocked", blocked_env) + self.assertEqual((blocked_env.get("error") or {}).get("code"), "blocked", blocked_env) + + def test_exec_cmd_blocks_redirection_in_readonly_phases(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16c", + step_id="m1-core-foundation", + allowed_read_paths=["."], + allowed_write_paths=["."], + enable_checkpoints=False, + ) + blocked = tools.call( + "exec_cmd", + {"command": "echo blocked > /tmp/trinity_scope_probe.txt", "mode": "summarized", "timeout_seconds": 30}, + role="Verifier", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(blocked.get("status"), "blocked", blocked) + self.assertEqual((blocked.get("error") or {}).get("code"), "blocked", blocked) + + def test_path_allowlist_rejects_parent_traversal_segments(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16b", + step_id="m1-core-foundation", + allowed_read_paths=[".trinity", "spec"], + allowed_write_paths=["spec/impl_context"], + enable_checkpoints=False, + ) + self.assertFalse(tools._is_allowed_path("../.trinity/secrets.txt", [".trinity"])) + self.assertFalse(tools._is_allowed_path("../../spec/14_roadmap.json", ["spec"])) + + def test_write_file_append_reports_resulting_file_hash(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16b", + step_id="m1-core-foundation", + allowed_read_paths=["."], + allowed_write_paths=["spec/impl_context"], + enable_checkpoints=False, + ) + seed_path = os.path.join(tmp, "spec", "impl_context", "append_hash.txt") + self._write_text(seed_path, "A") + result = tools.call( + "write_file", + {"path": "spec/impl_context/append_hash.txt", "content": "B", "mode": "append"}, + role="Builder", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(result.get("status"), "success", result) + with open(seed_path, "rb") as f: + expected_sha = hashlib.sha256(f.read()).hexdigest() + self.assertEqual(result.get("artifact_sha256"), f"sha256:{expected_sha}", result) + + def test_apply_patch_reports_artifact_hash_for_patched_file(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16b", + step_id="m1-core-foundation", + allowed_read_paths=["."], + allowed_write_paths=["spec/impl_context"], + enable_checkpoints=False, + ) + target = os.path.join(tmp, "spec", "impl_context", "patch_hash.txt") + self._write_text(target, "old line\n") + patch = ( + "--- a/spec/impl_context/patch_hash.txt\n" + "+++ b/spec/impl_context/patch_hash.txt\n" + "@@ -1 +1 @@\n" + "-old line\n" + "+new line\n" + ) + result = tools.call( + "apply_patch", + {"patch": patch}, + role="Builder", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(result.get("status"), "success", result) + with open(target, "rb") as f: + expected_sha = hashlib.sha256(f.read()).hexdigest() + self.assertEqual(result.get("artifact_ref"), "spec/impl_context/patch_hash.txt", result) + self.assertEqual(result.get("artifact_sha256"), f"sha256:{expected_sha}", result) + + def test_move_file_and_remove_file_report_artifact_hashes(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + tools = ToolExecutor( + tmp, + logger, + "run-test", + agent_id="orchestrator-test", + phase="16b", + step_id="m1-core-foundation", + allowed_read_paths=["."], + allowed_write_paths=["spec/impl_context"], + enable_checkpoints=False, + ) + src = os.path.join(tmp, "spec", "impl_context", "move_src.txt") + dst = os.path.join(tmp, "spec", "impl_context", "move_dst.txt") + self._write_text(src, "moved-by-trinity\n") + moved = tools.call( + "move_file", + {"src_path": "spec/impl_context/move_src.txt", "dst_path": "spec/impl_context/move_dst.txt"}, + role="Builder", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(moved.get("status"), "success", moved) + self.assertFalse(os.path.exists(src), "move_file should remove source path") + self.assertTrue(os.path.exists(dst), "move_file should create destination path") + with open(dst, "rb") as f: + moved_sha = hashlib.sha256(f.read()).hexdigest() + self.assertEqual(moved.get("artifact_ref"), "spec/impl_context/move_dst.txt", moved) + self.assertEqual(moved.get("artifact_sha256"), f"sha256:{moved_sha}", moved) + + removed = tools.call( + "remove_file", + {"path": "spec/impl_context/move_dst.txt"}, + role="Builder", + parent_id="orchestrator-test", + loop_id="l3", + ) + self.assertEqual(removed.get("status"), "success", removed) + self.assertFalse(os.path.exists(dst), "remove_file should delete destination path") + self.assertEqual(removed.get("artifact_ref"), "spec/impl_context/move_dst.txt", removed) + self.assertEqual(removed.get("artifact_sha256"), f"sha256:{moved_sha}", removed) + + def test_capture_artifact_hash_matches_written_bytes(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + rel, sha = logger._write_capture_artifact(event_id="evt-hash", kind="prompt", content="capture-content") + abs_path = os.path.join(tmp, rel) + with open(abs_path, "rb") as f: + expected_sha = hashlib.sha256(f.read()).hexdigest() + self.assertEqual(sha, expected_sha) + + def test_session_logger_rejects_invalid_event_before_persist(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False, checkpoint_commits=False) + logger = SessionLogger(tmp, "run-test", "orchestrator-test", "m1-core-foundation", "input-model") + with self.assertRaises(RuntimeError): + logger.append( + "MESSAGE", + role="InvalidRole", + phase_id="16a", + loop_id="l1", + agent_id="orchestrator-test", + parent_id=None, + summary="invalid role test", + prompt_template_id="prompt_16a", + step_id="m1-core-foundation", + ) + self.assertTrue(os.path.exists(logger.path), "Session log file should exist") + self.assertEqual(os.path.getsize(logger.path), 0, "Invalid event must not be persisted") + + def test_run_trinity_sets_capture_policy_fallback_metadata_for_incomplete_policy(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=True, checkpoint_commits=False, execution_mode="deterministic") + self._write_json( + os.path.join(tmp, ".trinity", "logging", "log_capture_policy.json"), + { + "policy_id": "legacy-policy", + "version": "1", + "default_capture_level": "summary", + "always_full_on_event_types": ["ERROR"], + "sample_rate_by_event_type": { + "SPAWN": 0.0, + "MESSAGE": 0.0, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.0, + "VALIDATION": 0.0, + "TERMINATE": 0.0, + "ERROR": 0.0, + }, + "max_full_events_per_run": 4, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": [], + "require_redaction_before_full": True, + }, + ) + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + + session_path = os.path.join(tmp, result["session_log"]) + with open(session_path, "r", encoding="utf-8") as f: + events = [json.loads(line) for line in f if line.strip()] + fallback_events = [ + e for e in events + if isinstance(e.get("metadata"), dict) and e["metadata"].get("capture_policy_fallback_applied") is True + ] + self.assertTrue(fallback_events, "Expected capture_policy_fallback_applied=true when policy profile is incomplete") + + def test_run_trinity_resume_from_latest_session_state(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo(tmp, allow_dirty=False) + first = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(first.get("status"), "completed", first) + resumed = run_trinity(repo_root=tmp, step_id=None, resume=True) + self.assertEqual(resumed.get("status"), "completed", resumed) + self.assertEqual(resumed.get("step_id"), "m1-core-foundation") + + def test_run_trinity_llm_mode_openai_compatible_endpoint(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + server, thread, calls = self._start_fake_openai_server() + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + self.assertEqual(result.get("execution_mode"), "llm", result) + phases = {entry["phase"] for entry in calls} + self.assertTrue({"16a", "16b", "16c"}.issubset(phases), calls) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_session_log_aggregates_child_events(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + server, thread, _calls = self._start_fake_openai_server() + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + session_log_path = os.path.join(tmp, result["session_log"]) + with open(session_log_path, "r", encoding="utf-8") as f: + events = [json.loads(line) for line in f if line.strip()] + + self.assertTrue(events, "Session log should contain events") + self.assertTrue( + any(e.get("event_type") == "MESSAGE" and e.get("role") in {"Planner", "Builder", "Verifier"} for e in events), + "Canonical session log should include child MESSAGE events", + ) + self.assertTrue( + all(e.get("run_id") == result.get("run_id") for e in events if isinstance(e, dict)), + "All events in canonical session log should share one run_id", + ) + + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + session_files = [name for name in os.listdir(sessions_dir) if name.endswith(".jsonl")] + self.assertEqual(len(session_files), 1, session_files) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_llm_mode_supports_utility_role_invocation(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + state = {"utility_called": False} + + def _phase_and_role(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + task_input = candidate.get("task_input") + if isinstance(task_input, dict): + maybe_phase = task_input.get("phase") + maybe_role = task_input.get("role") + if isinstance(maybe_phase, str) and maybe_phase: + return maybe_phase, maybe_role if isinstance(maybe_role, str) else None + maybe_phase = candidate.get("phase") + maybe_role = candidate.get("role") + if isinstance(maybe_phase, str) and maybe_phase: + return maybe_phase, maybe_role if isinstance(maybe_role, str) else None + return "16a", None + + def responder(messages, _state): + phase, role = _phase_and_role(messages) + if phase == "utility" and role == "Researcher": + return { + "action": "final_result", + "summary": "research ready", + "loop_checkpoint": self._loop_checkpoint("utility-researcher"), + "utility_result": { + "status": "ready", + "summary": "Grounded context collected", + "open_questions": [], + "findings": [], + }, + } + if phase == "16a": + if not state["utility_called"]: + state["utility_called"] = True + return { + "action": "utility_call", + "summary": "run researcher utility", + "utility_call": { + "role": "Researcher", + "objective": "Collect grounded references for planner stage", + "input": {"required_outputs": ["findings", "recommended_spec_refs"]}, + }, + } + return { + "action": "final_result", + "summary": "16a success", + "loop_checkpoint": self._loop_checkpoint("16a"), + "task_result": { + "status": "success", + "summary": "16a success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + return { + "action": "final_result", + "summary": f"{phase} success", + "loop_checkpoint": self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + + server, thread, _calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + self.assertTrue(state["utility_called"], "Planner should invoke utility role before final_result") + + session_path = os.path.join(tmp, result["session_log"]) + with open(session_path, "r", encoding="utf-8") as f: + events = [json.loads(line) for line in f if line.strip()] + researcher_events = [e for e in events if e.get("role") == "Researcher" and e.get("phase_id") == "utility"] + self.assertTrue(researcher_events, "Utility role events should be present in session log") + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_llm_mode_blocks_malformed_utility_result_schema(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + + state = {"utility_called": False} + + def _phase_and_role(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + task_input = candidate.get("task_input") + if isinstance(task_input, dict): + phase = task_input.get("phase") + role = task_input.get("role") + if isinstance(phase, str): + return phase, role if isinstance(role, str) else None + phase = candidate.get("phase") + role = candidate.get("role") + if isinstance(phase, str): + return phase, role if isinstance(role, str) else None + return "16a", None + + def _has_utility_feedback(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + if isinstance(candidate, dict) and isinstance(candidate.get("utility_result"), dict): + return True + return False + + def responder(messages, _state): + phase, role = _phase_and_role(messages) + if phase == "utility" and role == "Researcher": + return { + "action": "final_result", + "summary": "malformed utility result", + "loop_checkpoint": self._loop_checkpoint("utility-researcher"), + "utility_result": { + "summary": "missing status should fail schema" + }, + } + if phase == "16a": + if not state["utility_called"]: + state["utility_called"] = True + return { + "action": "utility_call", + "summary": "run researcher utility", + "utility_call": { + "role": "Researcher", + "objective": "Collect references", + "input": {"required_outputs": ["findings"]}, + }, + } + if _has_utility_feedback(messages): + return { + "action": "final_result", + "summary": "planner blocked by malformed utility result", + "loop_checkpoint": self._loop_checkpoint("16a"), + "task_result": { + "status": "blocked", + "summary": "utility payload invalid", + "artifacts": [], + "findings": [ + { + "id": "planner-utility-result-invalid", + "type": "policy", + "severity": "blocking", + "description": "Utility result schema validation failed.", + "source": "Planner", + "impact": "Planner cannot continue", + } + ], + }, + } + return { + "action": "final_result", + "summary": f"{phase} success", + "loop_checkpoint": self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + + server, thread, _calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertIn(result.get("status"), {"blocked", "completed"}, result) + + session_path = os.path.join(tmp, result["session_log"]) + with open(session_path, "r", encoding="utf-8") as f: + events = [json.loads(line) for line in f if line.strip()] + fail_events = [ + e for e in events + if e.get("event_type") == "VALIDATION" + and e.get("role") == "Researcher" + and "utility_result schema fail" in (e.get("content", {}).get("summary", "")) + ] + self.assertTrue(fail_events, "Expected utility_result schema fail validation event for malformed payload") + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_fresh_run_resets_spawn_attempts(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="deterministic", + preverified_artifact=False, + ) + + first = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(first.get("status"), "completed", first) + + second = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(second.get("status"), "completed", second) + + spawn_log_path = os.path.join(tmp, ".trinity", "runtime", "spawn_log.json") + with open(spawn_log_path, "r", encoding="utf-8") as f: + spawn_log = json.load(f) + + self.assertEqual(spawn_log.get("run_id"), second.get("run_id")) + entries = spawn_log.get("entries", []) + self.assertTrue(entries, "spawn_log should contain entries for the latest run") + attempts = [int(e.get("attempt", 0)) for e in entries if isinstance(e, dict)] + self.assertTrue(all(attempt == 1 for attempt in attempts), attempts) + + def test_run_trinity_bootstraps_missing_milestone_artifact(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="deterministic", + preverified_artifact=False, + ) + milestone_path = os.path.join(tmp, "spec", "impl_context", "m1-core-foundation.json") + os.remove(milestone_path) + + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + self.assertTrue(os.path.exists(milestone_path), "Planner bootstrap should recreate missing milestone artifact") + + with open(milestone_path, "r", encoding="utf-8") as f: + milestone = json.load(f) + checklist = milestone.get("plan", {}).get("spec_alignment", {}).get("checklist", []) + self.assertTrue(isinstance(checklist, list) and checklist, "Bootstrapped milestone should include checklist contract") + spawn_root = os.path.join(tmp, ".trinity", "runtime", "spawns") + context_paths = [] + if os.path.isdir(spawn_root): + for child in os.listdir(spawn_root): + candidate = os.path.join(spawn_root, child, "context_pack.json") + if os.path.exists(candidate): + context_paths.append(candidate) + self.assertTrue(context_paths, "Expected planner spawn context pack artifacts") + planner_context = None + for candidate in sorted(context_paths): + with open(candidate, "r", encoding="utf-8") as f: + loaded = json.load(f) + if loaded.get("phase") == "16a": + planner_context = loaded + break + self.assertIsInstance(planner_context, dict, "Expected a 16a planner context pack artifact") + trace = planner_context.get("bootstrap_ref_trace") + self.assertTrue(isinstance(trace, list) and trace, "Planner bootstrap should emit bootstrap_ref_trace explainability") + + def test_run_trinity_emits_anchor_union_metrics_validation_event(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="deterministic", + preverified_artifact=False, + ) + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + + session_path = os.path.join(tmp, result["session_log"]) + with open(session_path, "r", encoding="utf-8") as f: + events = [json.loads(line) for line in f if line.strip()] + + union_events = [ + e for e in events + if e.get("event_type") == "VALIDATION" + and isinstance(e.get("metadata"), dict) + and isinstance(e["metadata"].get("anchor_union_metrics"), dict) + ] + self.assertTrue(union_events, "Expected VALIDATION event with anchor_union_metrics metadata") + metrics = union_events[-1]["metadata"]["anchor_union_metrics"] + self.assertGreaterEqual(metrics.get("active_contexts", -1), 1, metrics) + self.assertIn("checklist_conflicts_count", metrics) + + def test_run_trinity_retry_caps_are_driven_by_yaml_config(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="deterministic", + preverified_artifact=False, + retry_cap_planner=2, + retry_cap_builder=10, + retry_cap_verifier=10, + retry_cap_milestone=10, + ) + os.remove(os.path.join(tmp, "spec", "05_interface_contracts.json")) + + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "blocked", result) + self.assertEqual(result.get("phase"), "16a", result) + + config = TrinityConfig.load(tmp) + self.assertEqual(config.retry_cap_planner, 2) + + spawn_log_path = os.path.join(tmp, ".trinity", "runtime", "spawn_log.json") + with open(spawn_log_path, "r", encoding="utf-8") as f: + spawn_log = json.load(f) + attempts = [ + int(e.get("attempt", 0)) + for e in spawn_log.get("entries", []) + if isinstance(e, dict) and e.get("phase") == "16a" + ] + self.assertEqual(sorted(set(attempts)), [1, 2], attempts) + + def test_run_trinity_child_timeouts_are_driven_by_yaml_config(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="deterministic", + preverified_artifact=False, + ) + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "a", encoding="utf-8") as f: + f.write(" child_timeout_seconds: 7200\n") + f.write(" child_timeout_by_phase:\n") + f.write(" 16a: 1200\n") + f.write(" 16b: 14400\n") + f.write(" 16c: 3600\n") + f.write(" allow_bootstrap_authority_fallback: true\n") + f.write(" allow_anchor_conflicts: true\n") + + config = TrinityConfig.load(tmp) + runtime = TrinityRuntime(tmp, config, step_id="m1-core-foundation") + self.assertEqual(config.child_timeout_seconds, 7200) + self.assertEqual(config.child_timeout_by_phase.get("16b"), 14400) + self.assertTrue(config.allow_bootstrap_authority_fallback) + self.assertTrue(config.allow_anchor_conflicts) + self.assertEqual(runtime._child_timeout_for_phase("16a"), 1200) + self.assertEqual(runtime._child_timeout_for_phase("16b"), 14400) + self.assertEqual(runtime._child_timeout_for_phase("utility"), 7200) + + def test_run_trinity_terminal_questions_path_runs_session_log_validation(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + + def responder(_messages, _state): + return { + "action": "final_result", + "summary": "Need clarification", + "loop_checkpoint": self._loop_checkpoint("16a"), + "task_result": { + "status": "questions", + "summary": "Clarification required", + "artifacts": [], + "questions": ["Confirm scope."], + }, + } + + def _validate_with_session_error(_repo_root, _path, schema_type): + if schema_type == "session_event": + return ["forced terminal session log validation failure"] + return [] + + server, thread, _calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + with patch("specdev_tools.trinity_runtime.validate_runtime_file", side_effect=_validate_with_session_error): + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "blocked", result) + self.assertEqual(result.get("phase"), "session_log", result) + self.assertTrue(any("forced terminal session log validation failure" in e for e in result.get("errors", [])), result) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_terminal_blocked_path_runs_session_log_validation(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="deterministic", + preverified_artifact=False, + retry_cap_planner=1, + retry_cap_builder=1, + retry_cap_verifier=1, + retry_cap_milestone=1, + ) + os.remove(os.path.join(tmp, "spec", "05_interface_contracts.json")) + + def _validate_with_session_error(_repo_root, _path, schema_type): + if schema_type == "session_event": + return ["forced terminal session log validation failure"] + return [] + + with patch("specdev_tools.trinity_runtime.validate_runtime_file", side_effect=_validate_with_session_error): + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "blocked", result) + self.assertEqual(result.get("phase"), "session_log", result) + self.assertTrue(any("forced terminal session log validation failure" in e for e in result.get("errors", [])), result) + + def test_run_trinity_llm_mode_tool_call_loop(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + + def _phase_from_messages(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + if isinstance(candidate, dict) and isinstance(candidate.get("task_input"), dict): + maybe_phase = candidate["task_input"].get("phase") + if isinstance(maybe_phase, str): + return maybe_phase + return "16a" + + def _has_tool_result(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + if isinstance(candidate, dict) and isinstance(candidate.get("tool_result"), dict): + return True + return False + + def responder(messages, _state): + phase = _phase_from_messages(messages) + if _has_tool_result(messages): + return { + "action": "final_result", + "summary": f"{phase} success after tool loop", + "loop_checkpoint": self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + return { + "action": "tool_call", + "summary": f"{phase} inspect milestone artifact", + "tool_call": { + "tool_name": "read_file", + "args": {"path": "spec/impl_context/m1-core-foundation.json", "start_line": 1, "end_line": 5}, + }, + } + + server, thread, calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "completed", result) + self.assertEqual(result.get("execution_mode"), "llm", result) + self.assertGreaterEqual(len(calls), 6, calls) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_llm_mode_questions_resume_with_answers(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + + def _phase_and_desc(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + task_input = candidate.get("task_input") + if isinstance(task_input, dict): + phase = task_input.get("phase") + desc = task_input.get("task_description", "") + if isinstance(phase, str): + return phase, str(desc) + return "16a", "" + + def responder(messages, _state): + phase, desc = _phase_and_desc(messages) + if phase == "16a" and "User clarifications:" not in desc: + return { + "action": "final_result", + "summary": "Need clarification", + "loop_checkpoint": self._loop_checkpoint("16a"), + "task_result": { + "status": "questions", + "summary": "Clarification required", + "artifacts": [], + "questions": ["Confirm scope for m1-core-foundation implementation loop."], + }, + } + return { + "action": "final_result", + "summary": f"{phase} success", + "loop_checkpoint": self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + + server, thread, _calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + + first = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(first.get("status"), "questions", first) + self.assertEqual(first.get("phase"), "16a", first) + self.assertTrue(first.get("questions"), first) + + resumed = run_trinity( + repo_root=tmp, + step_id=None, + resume=True, + answers=["Stay within the existing milestone plan and target files."], + ) + self.assertEqual(resumed.get("status"), "completed", resumed) + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + session_files = [name for name in os.listdir(sessions_dir) if name.endswith(".jsonl")] + self.assertEqual(len(session_files), 1, session_files) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_llm_mode_blocks_out_of_scope_write(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + seen = {"blocked_tool_result": False} + + def _phase(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + task_input = candidate.get("task_input") + if isinstance(task_input, dict): + maybe_phase = task_input.get("phase") + if isinstance(maybe_phase, str): + return maybe_phase + return "16a" + + def _tool_result_status(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + tool_result = candidate.get("tool_result") + if isinstance(tool_result, dict): + return tool_result.get("status") + return None + + def responder(messages, _state): + phase = _phase(messages) + if phase == "16a": + return { + "action": "final_result", + "summary": "16a success", + "loop_checkpoint": self._loop_checkpoint("16a"), + "task_result": { + "status": "success", + "summary": "16a success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + if phase == "16b": + status = _tool_result_status(messages) + if status: + seen["blocked_tool_result"] = (status == "blocked") + return { + "action": "final_result", + "summary": "16b blocked on out-of-scope write", + "loop_checkpoint": self._loop_checkpoint("16b"), + "task_result": { + "status": "blocked", + "summary": "Attempted out-of-scope write", + "artifacts": [], + "findings": [ + { + "id": "llm-out-of-scope-write", + "type": "scope_creep", + "severity": "blocking", + "description": "Write target outside scope", + "source": "LLM", + "impact": "Scope violation", + } + ], + }, + } + return { + "action": "tool_call", + "summary": "Attempt out-of-scope write", + "tool_call": { + "tool_name": "write_file", + "args": {"path": "src/out_of_scope.py", "content": "print('x')"}, + }, + } + return { + "action": "final_result", + "summary": f"{phase} success", + "loop_checkpoint": self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + + server, thread, _calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "blocked", result) + self.assertEqual(result.get("phase"), "16b", result) + self.assertTrue(seen["blocked_tool_result"], "LLM tool_result should be blocked for out-of-scope write") + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_run_trinity_llm_mode_evidence_binding_failure_retries_planner_first(self): + with tempfile.TemporaryDirectory() as tmp: + self._create_fixture_repo( + tmp, + allow_dirty=True, + checkpoint_commits=False, + execution_mode="llm", + preverified_artifact=True, + ) + phase_counts = {"16a": 0, "16b": 0, "16c": 0} + + def _phase(messages): + for msg in messages: + if not (isinstance(msg, dict) and msg.get("role") == "user" and isinstance(msg.get("content"), str)): + continue + try: + candidate = json.loads(msg["content"]) + except Exception: + continue + task_input = candidate.get("task_input") + if isinstance(task_input, dict): + maybe_phase = task_input.get("phase") + if isinstance(maybe_phase, str): + return maybe_phase + return "16a" + + def responder(messages, _state): + phase = _phase(messages) + phase_counts[phase] = phase_counts.get(phase, 0) + 1 + if phase == "16c": + return { + "action": "final_result", + "summary": "Evidence binding missing", + "loop_checkpoint": self._loop_checkpoint("16c"), + "task_result": { + "status": "blocked", + "summary": "Verifier blocked: missing evidence binding", + "artifacts": [], + "findings": [ + { + "id": "llm-missing-evidence-binding", + "type": "policy", + "severity": "blocking", + "description": "Evidence binding missing for required checklist command.", + "source": "LLM", + "impact": "Cannot verify milestone closure.", + } + ], + }, + } + return { + "action": "final_result", + "summary": f"{phase} success", + "loop_checkpoint": self._loop_checkpoint(phase), + "task_result": { + "status": "success", + "summary": f"{phase} success", + "artifacts": ["spec/impl_context/m1-core-foundation.json"], + }, + } + + server, thread, _calls = self._start_scripted_openai_server(responder) + try: + port = server.server_address[1] + config_path = os.path.join(tmp, ".trinity", "trinity.yaml") + with open(config_path, "r", encoding="utf-8") as f: + config_text = f.read() + config_text = config_text.replace("http://localhost:1234/v1", f"http://127.0.0.1:{port}/v1") + self._write_text(config_path, config_text) + result = run_trinity(repo_root=tmp, step_id="m1-core-foundation") + self.assertEqual(result.get("status"), "blocked", result) + self.assertEqual(result.get("phase"), "16c", result) + self.assertTrue(any("verifier retry cap exceeded" in e for e in result.get("errors", [])), result) + self.assertGreaterEqual(phase_counts["16c"], 1, phase_counts) + self.assertGreaterEqual(phase_counts["16a"], 2, phase_counts) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_trinity_runtime_validation.py b/tests/integration/test_trinity_runtime_validation.py new file mode 100644 index 00000000..b5fdedbc --- /dev/null +++ b/tests/integration/test_trinity_runtime_validation.py @@ -0,0 +1,1810 @@ +import json +import os +import re +import subprocess +import hashlib +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from specdev_tools.trinity_runtime_validate import validate_runtime_file +from specdev_tools.validate import validate_file + + +class TestTrinityRuntimeValidation(unittest.TestCase): + def setUp(self): + toolkit_root = Path(__file__).resolve().parents[2] + self.repo_root = str(toolkit_root) + self.tool_call_request_schema_sha = self._schema_sha_from_rel("schema/trinity/tool_call_request.schema.json") + self.tool_call_result_schema_sha = self._schema_sha_from_rel("schema/trinity/tool_call_result.schema.json") + + def _schema_sha_from_rel(self, rel_path: str) -> str: + abs_path = os.path.join(self.repo_root, rel_path) + with open(abs_path, "r", encoding="utf-8") as f: + payload = json.load(f) + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def _git_head_commit(self) -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + self.skipTest("git is required for trinity runtime grounding tests") + return result.stdout.strip() + + def _first_id_line(self, rel_path: str) -> tuple[str, int]: + abs_path = os.path.join(self.repo_root, rel_path) + with open(abs_path, "r", encoding="utf-8") as f: + for idx, line in enumerate(f, start=1): + match = re.search(r'"id"\s*:\s*"([^"]+)"', line) + if match: + return match.group(1), idx + self.fail(f"No id field found in {rel_path}") + + def _write_json(self, path: str, payload: dict) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + def _event_sha256(self, event: dict) -> str: + payload = dict(event) + payload["event_sha256"] = None + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def _valid_task_input_payload(self) -> dict: + api_id, _ = self._first_id_line("spec/05_interface_contracts.json") + return { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "parent_id": "parent-1", + "role": "Planner", + "phase": "16a", + "step_id": "m1-core-foundation", + "task_description": "Plan milestone m1-core-foundation.", + "expected_output_schema": "https://specdev.local/schema/16_impl_context.schema.json", + "context_pack_ref": ".trinity/runtime/spawns/child-1/context_pack.json", + "target_files": ["spec/impl_context/m1-core-foundation.json"], + "spec_refs": [{"type": "api", "id": api_id}], + "role_metadata": { + "prompt_source": "devspec_toolkit/prompts/prompt_16a_impl_planner.md", + "persona_goal": "Create checklist-driven implementation plan", + "stop_conditions": ["missing required seed", "schema validation fail"], + }, + } + + def _valid_context_pack_16b(self) -> dict: + commit_hash = self._git_head_commit() + api_id, api_line = self._first_id_line("spec/05_interface_contracts.json") + return { + "protocol_version": "trinity-runtime-v1", + "phase": "16b", + "step_id": "m1-core-foundation", + "seed_manifest_path": "spec/common/seed_manifest.json", + "seed_files_ordered": ["docs/seed/seed_overview.md", "docs/seed/seed_tech_stack.md"], + "required_spec_refs": [ + { + "type": "api", + "id": api_id, + "path": "spec/05_interface_contracts.json", + "line_range": f"L{api_line}-L{api_line}", + "commit_hash": commit_hash, + } + ], + "artifact_refs": { + "milestone_context_path": "spec/impl_context/m1-core-foundation.json", + "anchor_path": "spec/16_impl_context.json", + }, + "allowed_read_paths": ["spec/", "src/", "tests/"], + "allowed_write_paths": ["src/", "tests/", "spec/impl_context/", "README.md"], + "target_file_patterns": ["src/auth.py", "tests/auth/test_login.py", "README.md"], + "docs_policy": { + "doc_paths": ["docs/**", "README.md"], + "readme_required": True, + "root_readme_required": True, + }, + "test_contract": { + "test_commands": ["pytest tests/auth/test_login.py::test_jwt -q"], + "success_markers": ["PASSED", "0 failed"], + }, + } + + def _valid_context_pack_16a_for_task_input(self) -> dict: + payload = self._valid_context_pack_16b() + payload["phase"] = "16a" + payload["target_file_patterns"] = ["spec/impl_context/m1-core-foundation.json"] + payload["allowed_write_paths"] = ["spec/impl_context/", "README.md"] + payload.pop("test_contract", None) + return payload + + def _valid_session_event(self) -> dict: + event = { + "schema_version": "trinity-session-log-v1", + "timestamp": "2026-02-13T00:00:00Z", + "event_type": "SPAWN", + "event_id": "evt-1", + "event_sequence": 1, + "prev_event_sha256": None, + "event_sha256": "0" * 64, + "run_id": "run-1", + "phase_id": "phase-16a", + "loop_id": "loop-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "step_id": "m1-core-foundation", + "tool_call_id": None, + "result_id": None, + "artifact_ref": ".trinity/runtime/spawns/child-1/task_input.json", + "artifact_sha256": "a" * 64, + "diff_ref": None, + "model": "gpt-5", + "content": { + "summary": "spawn planner", + "task_input_artifact_ref": ".trinity/runtime/spawns/child-1/task_input.json", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + }, + "metadata": { + "toolkit_version": "0.2.3", + "schema_version": "v1", + "git_head": "deadbeef", + "prompt_template_id": "prompt-16a", + "prompt_template_sha256": "b" * 64, + "redaction_profile": "eval", + "redaction_applied": False, + "capture_policy_ref": None, + "capture_policy_sha256": None, + "redaction_stats": { + "total_replacements": 0, + "by_class": {}, + "classes_detected": [], + "detectors_used": ["secret_scanner_v1"], + "min_confidence": 0.0, + "max_confidence": 0.0, + }, + "decoding": {"temperature": 0.2, "top_p": 0.9, "max_tokens": 4096}, + "token_usage": {"prompt": 100, "completion": 50, "total": 150}, + "tool_schema_context": { + "mode": "full_inline", + "catalog_ref": None, + "catalog_sha256": None, + "expanded_tool_names": [], + "request_schema_uri": "https://specdev.local/schema/trinity/tool_call_request.schema.json", + "request_schema_sha256": self.tool_call_request_schema_sha, + "result_schema_uri": "https://specdev.local/schema/trinity/tool_call_result.schema.json", + "result_schema_sha256": self.tool_call_result_schema_sha, + }, + }, + } + event["event_sha256"] = self._event_sha256(event) + return event + + def test_validate_file_fallback_for_task_input(self): + payload = self._valid_task_input_payload() + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "task_input.json") + context_pack_path = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1", "context_pack.json") + self._write_json(path, payload) + self._write_json(context_pack_path, self._valid_context_pack_16a_for_task_input()) + errors = validate_file(self.repo_root, path) + self.assertEqual(errors, [], f"Runtime task_input should validate via fallback. Errors: {errors}") + + def test_validate_file_fallback_invalid_task_input(self): + payload = self._valid_task_input_payload() + payload.pop("role_metadata") + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "task_input.json") + context_pack_path = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1", "context_pack.json") + self._write_json(path, payload) + self._write_json(context_pack_path, self._valid_context_pack_16a_for_task_input()) + errors = validate_file(self.repo_root, path) + self.assertTrue(errors, "Invalid runtime task_input should fail validation") + + def test_validate_runtime_context_pack_valid_16b(self): + payload = self._valid_context_pack_16b() + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Context pack should validate. Errors: {errors}") + + def test_validate_runtime_context_pack_missing_phase_required_seed_fails(self): + payload = self._valid_context_pack_16b() + payload["seed_files_ordered"] = ["docs/seed/seed_overview.md"] + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("step_requirements['16b']" in e for e in errors), + f"Context pack should fail when phase-required seeds are missing. Errors: {errors}", + ) + + def test_validate_runtime_context_pack_missing_required_spec_refs_fails(self): + payload = self._valid_context_pack_16b() + payload["required_spec_refs"] = [] + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("required_spec_refs" in e for e in errors), + f"Context pack should fail when required_spec_refs is empty for phase 16x. Errors: {errors}", + ) + + def test_validate_runtime_context_pack_bootstrap_ref_trace_maps_to_required_refs(self): + payload = self._valid_context_pack_16a_for_task_input() + ref = payload["required_spec_refs"][0] + payload["bootstrap_ref_trace"] = [ + { + "spec_type": ref["type"], + "id": ref["id"], + "selected_from": "roadmap.milestones[m1-core-foundation].deliverables[0]", + "selection_mode": "structured", + "path": ref["path"], + "line_range": ref["line_range"], + } + ] + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Context pack bootstrap_ref_trace should validate. Errors: {errors}") + + def test_validate_runtime_context_pack_bootstrap_ref_trace_rejects_unmapped_refs(self): + payload = self._valid_context_pack_16a_for_task_input() + ref = payload["required_spec_refs"][0] + payload["bootstrap_ref_trace"] = [ + { + "spec_type": ref["type"], + "id": "api-missing-from-required-refs", + "selected_from": "roadmap.milestones[m1-core-foundation].name", + "selection_mode": "tokenized", + "path": ref["path"], + "line_range": ref["line_range"], + } + ] + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("bootstrap_ref_trace" in e and "required_spec_refs" in e for e in errors), + f"Context pack should reject bootstrap_ref_trace entries not mapped to required_spec_refs. Errors: {errors}", + ) + + def test_validate_runtime_context_pack_invalid_missing_test_contract_for_16b(self): + payload = self._valid_context_pack_16b() + payload.pop("test_contract") + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue(errors, "16b context pack without test_contract should fail validation") + + def test_validate_runtime_task_result_questions(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Planner", + "phase": "16a", + "step_id": "m1-core-foundation", + "status": "questions", + "summary": "Need clarification before plan emission", + "artifacts": [], + "questions": ["Which roadmap milestone should be active?"], + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "task_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Questions task_result should validate. Errors: {errors}") + + def test_validate_runtime_task_result_questions_missing_questions(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Planner", + "phase": "16a", + "step_id": "m1-core-foundation", + "status": "questions", + "summary": "Need clarification before plan emission", + "artifacts": [], + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "task_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue(errors, "Questions task_result without questions array should fail validation") + + def test_validate_runtime_task_result_blocked_requires_finding_provenance(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "status": "blocked", + "summary": "Blocked by missing context", + "artifacts": [], + "findings": [ + { + "id": "missing-seed", + "type": "policy", + "severity": "blocking", + "description": "Required seed missing", + } + ], + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "task_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("source" in e or "impact" in e for e in errors), + f"Blocked task_result findings should require source and impact. Errors: {errors}", + ) + + def test_validate_runtime_task_result_success_16b_requires_execution_section(self): + with tempfile.TemporaryDirectory() as tmp: + artifact_path = os.path.join(tmp, "m1-core-foundation.json") + self._write_json( + artifact_path, + { + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "m1-core-foundation", + "owner": "api", + "created_at": "2026-02-13T00:00:00Z", + "seed_refs": [{"seed_id": "seed-overview"}], + "plan": { + "status": "active", + "summary": { + "functional_summary": "x", + "scope_in": ["x"], + "scope_out": [], + "target_file_patterns": ["src/x.py", "README.md"], + }, + "docs_impact": { + "status": "required", + "rationale": "x rationale for docs impact.", + "docs_touched": ["README.md"], + }, + "spec_alignment": { + "checklist": [ + { + "id": "CHK_X_01", + "spec_ref": { + "type": "fr", + "id": "fr-core-login", + "line_range": "L1-L1", + "commit_hash": "a1b2c3d4e5f61234567890123456789012345678", + }, + "description": "x", + "linked_test_expectation": "pytest -q", + "type": "behavior", + "layer": "service", + "nfr_refs": ["nfr-availability-uptime"], + "fixture_ref": "fixture-login-success", + "implementation": { + "status": "pending", + "actions": [ + { + "type": "manual_verification", + "description": "x", + } + ], + }, + } + ] + }, + "review_requirements": {"test_commands": ["pytest -q"]}, + }, + }, + ) + task_result_path = os.path.join(tmp, "task_result.json") + self._write_json( + task_result_path, + { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "status": "success", + "summary": "builder completed", + "artifacts": [artifact_path], + }, + ) + errors = validate_runtime_file(self.repo_root, task_result_path) + self.assertTrue( + any("phase 16b success artifact" in e for e in errors), + f"16b success task_result should require execution section. Errors: {errors}", + ) + + def test_validate_runtime_task_result_success_16b_referenced_step16_must_be_schema_valid(self): + with tempfile.TemporaryDirectory() as tmp: + artifact_path = os.path.join(tmp, "m1-core-foundation.json") + # Contains the Step 16 schema URI but is intentionally schema-invalid: + # missing required top-level plan/owner/created_at/seed_refs. + self._write_json( + artifact_path, + { + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "m1-core-foundation", + "execution": { + "execution_results": [ + { + "status": "passed", + "outcome_description": "ran tests", + "reasoning": "ok", + "command": "pytest -q", + "evidence": "tests PASSED", + "evidence_ref": "sha256:" + ("a" * 64), + "evidence_binding": { + "sha256": "a" * 64, + "command": "pytest -q", + "exit_code": 0, + }, + } + ] + }, + }, + ) + task_result_path = os.path.join(tmp, "task_result.json") + self._write_json( + task_result_path, + { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "status": "success", + "summary": "builder completed", + "artifacts": [artifact_path], + }, + ) + errors = validate_runtime_file(self.repo_root, task_result_path) + self.assertTrue( + any("failed step16 validation" in e for e in errors), + f"16b success task_result should fail when referenced Step 16 artifact is schema-invalid. Errors: {errors}", + ) + + def test_validate_runtime_task_result_success_16c_verified_rejects_blocking_findings(self): + with tempfile.TemporaryDirectory() as tmp: + artifact_path = os.path.join(tmp, "m1-core-foundation.json") + evidence = "PASSED marker content" + evidence_sha = hashlib.sha256(evidence.encode("utf-8")).hexdigest() + api_id, api_line = self._first_id_line("spec/05_interface_contracts.json") + commit_hash = self._git_head_commit() + line_range = f"L{api_line}-L{api_line}" + self._write_json( + artifact_path, + { + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": "m1-core-foundation", + "owner": "api", + "created_at": "2026-02-13T00:00:00Z", + "seed_refs": [{"seed_id": "seed-overview"}], + "plan": { + "status": "active", + "summary": { + "functional_summary": "x", + "scope_in": ["x"], + "scope_out": [], + "target_file_patterns": ["src/x.py", "README.md"], + }, + "docs_impact": { + "status": "required", + "rationale": "x rationale for docs impact.", + "docs_touched": ["README.md"], + }, + "spec_alignment": { + "checklist": [ + { + "id": "CHK_X_01", + "spec_ref": { + "type": "api", + "id": api_id, + "line_range": line_range, + "commit_hash": commit_hash, + }, + "description": "x", + "linked_test_expectation": "pytest -q", + "type": "behavior", + "layer": "service", + "nfr_refs": ["nfr-availability-uptime"], + "fixture_ref": "fixture-login-success", + "implementation": { + "status": "verified", + "actions": [ + { + "type": "manual_verification", + "description": "x", + "evidence": { + "type": "snippet", + "content": evidence, + "evidence_ref": f"sha256:{evidence_sha}", + }, + } + ], + }, + } + ] + }, + "review_requirements": {"test_commands": ["pytest -q"]}, + }, + "execution": { + "files_touched": ["README.md"], + "execution_results": [ + { + "status": "passed", + "outcome_description": "ok", + "reasoning": "ok", + "command": "pytest -q", + "evidence": evidence, + "evidence_ref": f"sha256:{evidence_sha}", + "evidence_binding": { + "sha256": evidence_sha, + "command": "pytest -q", + "exit_code": 0, + "timestamp": "2026-02-13T00:00:00Z", + }, + } + ], + "critical_evidence": { + "satisfied_checklist_ids": ["CHK_X_01"], + "passed_test_commands": ["pytest -q"], + }, + }, + "review": { + "findings": [ + { + "id": "f-1", + "type": "tests", + "severity": "blocking", + "description": "blocking issue present", + "spec_ref": { + "type": "api", + "id": api_id, + "line_range": line_range, + "commit_hash": commit_hash, + }, + "metadata": {"source": "Verifier", "impact": "functional-failure"}, + "remediation_task": { + "task_id": "rem-1", + "summary": "fix it", + "checklist_ids": ["CHK_X_01"], + "files_to_touch": ["README.md"], + }, + } + ], + "ratings": { + "spec_completeness": 5, + "code_quality": 5, + "tests_completeness": 5, + "docs_completeness": 5, + "metadata_usage": 5, + }, + "verdict": "verified", + "next_actions": "Milestone verified.", + "fixture_status": { + "implemented_endpoints": [], + "test_results": [], + "ci_status": "green", + }, + }, + }, + ) + task_result_path = os.path.join(tmp, "task_result.json") + self._write_json( + task_result_path, + { + "protocol_version": "trinity-runtime-v1", + "child_id": "child-1", + "role": "Verifier", + "phase": "16c", + "step_id": "m1-core-foundation", + "status": "success", + "summary": "verifier completed", + "artifacts": [artifact_path], + }, + ) + errors = validate_runtime_file(self.repo_root, task_result_path) + self.assertTrue( + any("verdict=verified but includes blocking/major findings" in e for e in errors), + f"16c success task_result should fail when verified verdict conflicts with findings severity. Errors: {errors}", + ) + + def test_validate_runtime_context_pack_invalid_protocol_version(self): + payload = self._valid_context_pack_16b() + payload["protocol_version"] = "v1" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue(errors, "Context pack with non-constant protocol_version should fail validation") + + def test_validate_runtime_scratchpad_state(self): + payload = { + "phase": "16b", + "checklist_scope": ["CHK_AUTH_01"], + "last_validation_gate": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + }, + "next_action_ref": "checklist:CHK_AUTH_01:run_tests", + "state_summary": "Continue implementation and run linked tests", + "milestone_step_id": "m1-core-foundation", + "created_at": "2026-02-13T00:00:00+00:00", + "updated_at": "2026-02-13T00:00:00+00:00", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "scratchpad_abc123.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Scratchpad state should validate. Errors: {errors}") + + def test_validate_runtime_session_event_log(self): + event = self._valid_session_event() + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + validation_input = self._valid_session_event() + validation_input["event_type"] = "VALIDATION" + validation_input["event_id"] = "evt-2" + validation_input["event_sequence"] = 2 + validation_input["prev_event_sha256"] = event["event_sha256"] + validation_input["content"] = { + "summary": "validated spawn task input", + "task_input_artifact_ref": ".trinity/runtime/spawns/child-1/task_input.json", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "validation": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + "seed_lint": "n/a", + "docs_lint": "n/a", + }, + } + validation_input["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_input.json" + validation_input["artifact_sha256"] = "b" * 64 + validation_input["event_sha256"] = self._event_sha256(validation_input) + + validation_result = self._valid_session_event() + validation_result["event_type"] = "VALIDATION" + validation_result["event_id"] = "evt-3" + validation_result["event_sequence"] = 3 + validation_result["prev_event_sha256"] = validation_input["event_sha256"] + validation_result["content"] = { + "summary": "validated child result", + "task_result_artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "validation": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + "seed_lint": "n/a", + "docs_lint": "n/a", + }, + } + validation_result["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_result.json" + validation_result["artifact_sha256"] = "d" * 64 + validation_result["event_sha256"] = self._event_sha256(validation_result) + + terminate = self._valid_session_event() + terminate["event_type"] = "TERMINATE" + terminate["event_id"] = "evt-4" + terminate["event_sequence"] = 4 + terminate["prev_event_sha256"] = validation_result["event_sha256"] + terminate["content"] = { + "summary": "child completed", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_result_artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + } + terminate["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_result.json" + terminate["artifact_sha256"] = "c" * 64 + terminate["event_sha256"] = self._event_sha256(terminate) + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + f.write(json.dumps(validation_input) + "\n") + f.write(json.dumps(validation_result) + "\n") + f.write(json.dumps(terminate) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Session event log should validate. Errors: {errors}") + + def test_validate_runtime_session_event_log_fails_when_spawn_not_terminated(self): + event = self._valid_session_event() + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("SPAWN event(s) but only" in e for e in errors), + f"Session log should fail when SPAWN does not have matching TERMINATE. Errors: {errors}", + ) + + def test_validate_runtime_session_event_spawn_ref_must_be_canonical(self): + event = self._valid_session_event() + event["content"]["task_input_artifact_ref"] = "task_input.json" + event["event_sha256"] = self._event_sha256(event) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("task_input_artifact_ref" in e for e in errors), + f"Session log should fail when SPAWN task_input_artifact_ref is non-canonical. Errors: {errors}", + ) + + def test_validate_runtime_session_event_terminate_ref_must_be_canonical(self): + spawn = self._valid_session_event() + terminate = self._valid_session_event() + terminate["event_type"] = "TERMINATE" + terminate["event_id"] = "evt-2" + terminate["event_sequence"] = 2 + terminate["prev_event_sha256"] = spawn["event_sha256"] + terminate["content"] = { + "summary": "child completed", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_result_artifact_ref": "task_result.json", + } + terminate["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_result.json" + terminate["artifact_sha256"] = "c" * 64 + terminate["event_sha256"] = self._event_sha256(terminate) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(spawn) + "\n") + f.write(json.dumps(terminate) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("task_result_artifact_ref" in e for e in errors), + f"Session log should fail when TERMINATE task_result_artifact_ref is non-canonical. Errors: {errors}", + ) + + def test_validate_runtime_session_event_artifact_ref_requires_artifact_sha256(self): + event = self._valid_session_event() + event["event_type"] = "MESSAGE" + event["content"].pop("task_input_artifact_ref", None) + event["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_input.json" + event["artifact_sha256"] = None + event["event_sha256"] = self._event_sha256(event) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("artifact_sha256" in e for e in errors), + f"Session event should fail when artifact_ref is set without artifact_sha256. Errors: {errors}", + ) + + def test_validate_runtime_session_event_tool_result_requires_result_id(self): + event = self._valid_session_event() + event["event_type"] = "TOOL_RESULT" + event["tool_call_id"] = "tool-1" + event["result_id"] = None + event["content"].pop("task_input_artifact_ref", None) + event["content"]["tool_result"] = { + "command": "pytest -q", + "exit_code": 0, + "duration_ms": 1200, + "working_dir": "/tmp/workspace", + } + event["event_sha256"] = self._event_sha256(event) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue(errors, "TOOL_RESULT event without result_id should fail validation") + + def test_validate_runtime_session_event_tool_call_missing_tool_schema_context_fails(self): + event = self._valid_session_event() + event["event_type"] = "TOOL_CALL" + event["tool_call_id"] = "tool-1" + event["content"]["tool_call"] = {"name": "exec_cmd", "args": {"command": "pytest -q", "mode": "summarized"}} + event["metadata"].pop("tool_schema_context", None) + event["event_sha256"] = self._event_sha256(event) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("tool_schema_context" in e for e in errors), + f"TOOL_CALL should fail when tool_schema_context is missing. Errors: {errors}", + ) + + def test_validate_runtime_session_event_tool_call_on_demand_context_requires_expanded_tool(self): + event = self._valid_session_event() + event["event_type"] = "TOOL_CALL" + event["tool_call_id"] = "tool-1" + event["content"]["tool_call"] = {"name": "exec_cmd", "args": {"command": "pytest -q", "mode": "summarized"}} + event["metadata"]["tool_schema_context"] = { + "mode": "catalog_plus_on_demand", + "catalog_ref": ".trinity/runtime/tools/catalog.json", + "catalog_sha256": "9" * 64, + "expanded_tool_names": ["read_file"], + "request_schema_uri": "https://specdev.local/schema/trinity/tool_call_request.schema.json", + "request_schema_sha256": self.tool_call_request_schema_sha, + "result_schema_uri": "https://specdev.local/schema/trinity/tool_call_result.schema.json", + "result_schema_sha256": self.tool_call_result_schema_sha, + } + event["event_sha256"] = self._event_sha256(event) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("expanded_tool_names" in e for e in errors), + f"TOOL_CALL on-demand context should fail when expanded tool list omits active tool. Errors: {errors}", + ) + + def test_validate_runtime_session_event_tool_result_requires_prior_tool_call(self): + tool_result = self._valid_session_event() + tool_result["event_type"] = "TOOL_RESULT" + tool_result["tool_call_id"] = "tool-missing" + tool_result["result_id"] = "result-1" + tool_result["content"].pop("task_input_artifact_ref", None) + tool_result["content"]["tool_result"] = { + "command": "pytest -q", + "exit_code": 0, + "duration_ms": 1200, + "working_dir": "/tmp/workspace", + } + tool_result["event_sha256"] = self._event_sha256(tool_result) + + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(tool_result) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("unknown tool_call_id" in e for e in errors), + f"Session log should fail when TOOL_RESULT has no prior TOOL_CALL. Errors: {errors}", + ) + + def test_validate_runtime_session_event_detects_sensitive_content(self): + event = self._valid_session_event() + event["event_type"] = "TOOL_RESULT" + event["tool_call_id"] = "tool-1" + event["result_id"] = "result-1" + event["content"].pop("task_input_artifact_ref", None) + event["content"]["tool_result"] = { + "command": "pytest -q", + "exit_code": 0, + "duration_ms": 1200, + "working_dir": "/tmp/workspace", + "stdout_excerpt": "token=ghp_123456789012345678901234567890123456", + "stderr_excerpt": "", + "truncated": False, + } + event["event_sha256"] = self._event_sha256(event) + + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + # include matching TOOL_CALL first so failure is from secret detection + tool_call = self._valid_session_event() + tool_call["event_type"] = "TOOL_CALL" + tool_call["tool_call_id"] = "tool-1" + tool_call["result_id"] = None + tool_call["content"]["tool_call"] = {"name": "exec_cmd", "args": {"command": "pytest -q"}} + tool_call["event_sha256"] = self._event_sha256(tool_call) + f.write(json.dumps(tool_call) + "\n") + + event["event_sequence"] = 2 + event["prev_event_sha256"] = tool_call["event_sha256"] + event["event_sha256"] = self._event_sha256(event) + f.write(json.dumps(event) + "\n") + + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("sensitive content detected" in e for e in errors), + f"Session log should fail when sensitive content is persisted. Errors: {errors}", + ) + + def test_validate_runtime_session_event_log_invalid_hash_chain(self): + event_1 = self._valid_session_event() + event_2 = self._valid_session_event() + event_2["event_id"] = "evt-2" + event_2["event_sequence"] = 2 + event_2["prev_event_sha256"] = "f" * 64 + event_2["event_sha256"] = self._event_sha256(event_2) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event_1) + "\n") + f.write(json.dumps(event_2) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("prev_event_sha256 does not match previous event hash" in e for e in errors), + f"Session log should fail when hash chain is invalid. Errors: {errors}", + ) + + def test_validate_runtime_session_event_log_invalid_redaction_stats(self): + event = self._valid_session_event() + event["metadata"]["redaction_applied"] = True + event["metadata"]["redaction_stats"]["total_replacements"] = 0 + event["metadata"]["redaction_stats"]["by_class"] = {"openai_key": 1} + event["metadata"]["redaction_stats"]["classes_detected"] = ["openai_key"] + event["event_sha256"] = self._event_sha256(event) + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("by_class total exceeds" in e for e in errors), + f"Session log should fail for inconsistent redaction stats. Errors: {errors}", + ) + + def test_validate_runtime_session_event_log_policy_enforced_capture_level(self): + event = self._valid_session_event() + policy = { + "policy_id": "policy-default", + "version": "1", + "default_capture_level": "none", + "always_full_on_event_types": ["SPAWN"], + "sample_rate_by_event_type": { + "SPAWN": 0.0, + "MESSAGE": 0.0, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.0, + "VALIDATION": 0.0, + "TERMINATE": 0.0, + "ERROR": 0.0 + }, + "max_full_events_per_run": 10, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": ["Orchestrator"], + "require_redaction_before_full": False, + "sampling_salt": "salt-1" + } + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + policy_path = os.path.join(tmp, "log_capture_policy.json") + self._write_json(policy_path, policy) + + policy_sha = hashlib.sha256( + json.dumps(policy, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + event["metadata"]["capture_policy_ref"] = policy_path + event["metadata"]["capture_policy_sha256"] = policy_sha + event["content"]["capture_level"] = "none" + event["content"]["capture_decision_reason"] = "policy:default:none" + event["event_sha256"] = self._event_sha256(event) + + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("policy-expected level 'full'" in e for e in errors), + f"Session log should fail when capture policy mandates full capture. Errors: {errors}", + ) + + def test_validate_runtime_session_event_log_policy_sampled_full(self): + event = self._valid_session_event() + policy = { + "policy_id": "policy-sampled", + "version": "1", + "default_capture_level": "none", + "always_full_on_event_types": [], + "sample_rate_by_event_type": { + "SPAWN": 1.0, + "MESSAGE": 0.0, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.0, + "VALIDATION": 0.0, + "TERMINATE": 0.0, + "ERROR": 0.0 + }, + "max_full_events_per_run": 1, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": ["Orchestrator"], + "require_redaction_before_full": False, + "sampling_salt": "salt-2" + } + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + policy_path = os.path.join(tmp, "log_capture_policy.json") + self._write_json(policy_path, policy) + + policy_sha = hashlib.sha256( + json.dumps(policy, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + event["metadata"]["capture_policy_ref"] = policy_path + event["metadata"]["capture_policy_sha256"] = policy_sha + event["content"]["capture_level"] = "full" + event["content"]["capture_decision_reason"] = "policy:sampled:SPAWN" + event["content"]["prompt_artifact_ref"] = ".trinity/captures/prompt_evt-1.txt" + event["content"]["prompt_sha256"] = "1" * 64 + event["content"]["response_artifact_ref"] = ".trinity/captures/response_evt-1.txt" + event["content"]["response_sha256"] = "2" * 64 + event["event_sha256"] = self._event_sha256(event) + + validation_input = self._valid_session_event() + validation_input["event_type"] = "VALIDATION" + validation_input["event_id"] = "evt-2" + validation_input["event_sequence"] = 2 + validation_input["prev_event_sha256"] = event["event_sha256"] + validation_input["content"] = { + "summary": "validated task input", + "task_input_artifact_ref": ".trinity/runtime/spawns/child-1/task_input.json", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "validation": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + "seed_lint": "n/a", + "docs_lint": "n/a", + }, + } + validation_input["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_input.json" + validation_input["artifact_sha256"] = "3" * 64 + validation_input["metadata"]["capture_policy_ref"] = policy_path + validation_input["metadata"]["capture_policy_sha256"] = policy_sha + validation_input["event_sha256"] = self._event_sha256(validation_input) + + validation_result = self._valid_session_event() + validation_result["event_type"] = "VALIDATION" + validation_result["event_id"] = "evt-3" + validation_result["event_sequence"] = 3 + validation_result["prev_event_sha256"] = validation_input["event_sha256"] + validation_result["content"] = { + "summary": "validated task result", + "task_result_artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "validation": { + "schema": "pass", + "deep_validator": "pass", + "governance": "n/a", + "seed_lint": "n/a", + "docs_lint": "n/a", + }, + } + validation_result["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_result.json" + validation_result["artifact_sha256"] = "4" * 64 + validation_result["metadata"]["capture_policy_ref"] = policy_path + validation_result["metadata"]["capture_policy_sha256"] = policy_sha + validation_result["event_sha256"] = self._event_sha256(validation_result) + + terminate = self._valid_session_event() + terminate["event_type"] = "TERMINATE" + terminate["event_id"] = "evt-4" + terminate["event_sequence"] = 4 + terminate["prev_event_sha256"] = validation_result["event_sha256"] + terminate["content"] = { + "summary": "child completed", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_result_artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + } + terminate["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_result.json" + terminate["artifact_sha256"] = "d" * 64 + terminate["metadata"]["capture_policy_ref"] = policy_path + terminate["metadata"]["capture_policy_sha256"] = policy_sha + terminate["event_sha256"] = self._event_sha256(terminate) + + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + f.write(json.dumps(validation_input) + "\n") + f.write(json.dumps(validation_result) + "\n") + f.write(json.dumps(terminate) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Session log should pass when sampled policy is honored. Errors: {errors}") + + def test_validate_runtime_session_event_log_policy_token_budget_fallback(self): + event = self._valid_session_event() + policy = { + "policy_id": "policy-budget", + "version": "1", + "default_capture_level": "none", + "always_full_on_event_types": ["SPAWN"], + "sample_rate_by_event_type": { + "SPAWN": 0.0, + "MESSAGE": 0.0, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.0, + "VALIDATION": 0.0, + "TERMINATE": 0.0, + "ERROR": 0.0 + }, + "max_full_events_per_run": 10, + "context_window_token_target": 80000, + "max_full_capture_context_fraction": 0.001, + "full_capture_token_budget_per_run": 50, + "max_full_prompt_tokens_per_event": 1000, + "max_full_completion_tokens_per_event": 1000, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": ["Orchestrator"], + "require_redaction_before_full": False, + "sampling_salt": "salt-budget" + } + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + policy_path = os.path.join(tmp, "log_capture_policy.json") + self._write_json(policy_path, policy) + + policy_sha = hashlib.sha256( + json.dumps(policy, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + event["metadata"]["capture_policy_ref"] = policy_path + event["metadata"]["capture_policy_sha256"] = policy_sha + event["metadata"]["token_usage"] = {"prompt": 90, "completion": 80, "total": 170} + event["content"]["capture_level"] = "full" + event["content"]["capture_decision_reason"] = "policy:always_full:SPAWN" + event["content"]["prompt_artifact_ref"] = ".trinity/captures/prompt_evt-1.txt" + event["content"]["prompt_sha256"] = "1" * 64 + event["content"]["response_artifact_ref"] = ".trinity/captures/response_evt-1.txt" + event["content"]["response_sha256"] = "2" * 64 + event["event_sha256"] = self._event_sha256(event) + + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("policy-expected level 'summary'" in e for e in errors), + f"Session log should fail when full-capture token budget is exceeded. Errors: {errors}", + ) + + def test_validate_runtime_session_event_log_policy_per_event_token_guard(self): + event = self._valid_session_event() + policy = { + "policy_id": "policy-per-event", + "version": "1", + "default_capture_level": "none", + "always_full_on_event_types": ["SPAWN"], + "sample_rate_by_event_type": { + "SPAWN": 0.0, + "MESSAGE": 0.0, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.0, + "VALIDATION": 0.0, + "TERMINATE": 0.0, + "ERROR": 0.0 + }, + "max_full_events_per_run": 10, + "context_window_token_target": 80000, + "max_full_capture_context_fraction": 0.5, + "full_capture_token_budget_per_run": 10000, + "max_full_prompt_tokens_per_event": 20, + "max_full_completion_tokens_per_event": 1000, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": ["Orchestrator"], + "require_redaction_before_full": False, + "sampling_salt": "salt-per-event" + } + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + policy_path = os.path.join(tmp, "log_capture_policy.json") + self._write_json(policy_path, policy) + + policy_sha = hashlib.sha256( + json.dumps(policy, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + event["metadata"]["capture_policy_ref"] = policy_path + event["metadata"]["capture_policy_sha256"] = policy_sha + event["metadata"]["token_usage"] = {"prompt": 100, "completion": 10, "total": 110} + event["content"]["capture_level"] = "full" + event["content"]["capture_decision_reason"] = "policy:always_full:SPAWN" + event["content"]["prompt_artifact_ref"] = ".trinity/captures/prompt_evt-1.txt" + event["content"]["prompt_sha256"] = "1" * 64 + event["content"]["response_artifact_ref"] = ".trinity/captures/response_evt-1.txt" + event["content"]["response_sha256"] = "2" * 64 + event["event_sha256"] = self._event_sha256(event) + + path = os.path.join(sessions_dir, "session.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("policy-expected level 'summary'" in e for e in errors), + f"Session log should fail when per-event prompt token guard is exceeded. Errors: {errors}", + ) + + def test_validate_runtime_context_pack_invalid_outside_allowed_write_paths(self): + payload = self._valid_context_pack_16b() + payload["target_file_patterns"] = ["infra/scripts/deploy.sh"] + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue(errors, "Context pack should fail when target_file_patterns are outside allowed_write_paths") + + def test_validate_runtime_context_pack_invalid_ungrounded_spec_ref_commit(self): + payload = self._valid_context_pack_16b() + payload["required_spec_refs"][0]["commit_hash"] = "f" * 40 + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "context_pack.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("commit_hash" in e and "not found in git" in e for e in errors), + f"Context pack should fail grounding when commit hash is invalid. Errors: {errors}", + ) + + def test_validate_runtime_log_capture_policy_schema(self): + payload = { + "policy_id": "policy-basic", + "version": "1", + "default_capture_level": "summary", + "always_full_on_event_types": ["ERROR"], + "sample_rate_by_event_type": { + "SPAWN": 0.0, + "MESSAGE": 0.0, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.0, + "VALIDATION": 0.0, + "TERMINATE": 0.0, + "ERROR": 0.0 + }, + "max_full_events_per_run": 10, + "context_window_token_target": 80000, + "max_full_capture_context_fraction": 0.25, + "full_capture_token_budget_per_run": 20000, + "max_full_prompt_tokens_per_event": 2048, + "max_full_completion_tokens_per_event": 2048, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": [], + "require_redaction_before_full": True, + "sampling_salt": "seed", + "operating_profile": { + "profile": "eval_default", + "tier": "balanced", + "budget_tier": "medium", + }, + "budgets": { + "context_window_token_target": 80000, + "full_capture_token_budget_per_run": 20000, + "max_full_prompt_tokens_per_event": 2048, + "max_full_completion_tokens_per_event": 2048, + }, + "retention": { + "session_log_days": 30, + "capture_artifact_days": 14, + "eval_export_days": 90, + }, + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "log_capture_policy.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Log capture policy should validate. Errors: {errors}") + + def test_validate_runtime_utility_call_schema(self): + payload = { + "role": "Researcher", + "objective": "Collect grounded references", + "input": {"required_outputs": ["findings"]}, + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "utility_call.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"utility_call payload should validate. Errors: {errors}") + + def test_validate_runtime_utility_result_schema(self): + payload = { + "status": "questions", + "summary": "Need one clarification", + "open_questions": ["Which fixture should be prioritized?"], + "findings": [], + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "utility_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"utility_result payload should validate. Errors: {errors}") + + def test_validate_runtime_eval_export_row_schema(self): + payload = { + "run_id": "run-1", + "event_id": "evt-1", + "event_sequence": 1, + "timestamp": "2026-02-13T00:00:00Z", + "event_type": "SPAWN", + "role": "Orchestrator", + "phase_id": "phase-16a", + "step_id": "m1-core-foundation", + "capture_level": "summary", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "artifact_ref": ".trinity/runtime/spawns/child-1/task_input.json", + "artifact_sha256": "a" * 64, + "diff_ref": None, + "redaction_applied": False, + "redaction_total_replacements": 0, + "redaction_classes": [], + "token_prompt": 100, + "token_completion": 50, + "token_total": 150, + "event_sha256": "b" * 64, + "prev_event_sha256": None + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "eval_export_row.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Eval export row should validate. Errors: {errors}") + + def test_validate_runtime_tool_call_request_schema(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "exec_cmd", + "args": {"command": "pytest -q", "mode": "summarized"}, + "working_dir": "/tmp/workspace", + "timeout_seconds": 30, + "created_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_request.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Tool call request should validate. Errors: {errors}") + + def test_validate_runtime_tool_call_result_schema(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-1", + "result_id": "result-1", + "agent_id": "agent-root", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "exec_cmd", + "status": "success", + "summary": "Command executed", + "result": {"command": "pytest -q", "mode": "summarized"}, + "duration_ms": 1200, + "exit_code": 0, + "working_dir": "/tmp/workspace", + "stdout_excerpt": "tests PASSED", + "stderr_excerpt": "", + "truncated": False, + "artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + "artifact_sha256": "a" * 64, + "finished_at": "2026-02-13T00:00:02Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Tool call result should validate. Errors: {errors}") + + def test_validate_runtime_tool_call_result_read_file_schema(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-2", + "result_id": "result-2", + "agent_id": "agent-root", + "role": "Planner", + "phase": "16a", + "step_id": "m1-core-foundation", + "tool_name": "read_file", + "status": "success", + "summary": "File read", + "result": { + "path": "spec/impl_context/m1-core-foundation.json", + "line_start": 1, + "line_end": 5, + "bytes_read": 128, + "content": "{\\n \\\"id\\\": \\\"m1-core-foundation\\\"\\n}\\n", + "truncated": False, + }, + "duration_ms": 25, + "truncated": False, + "finished_at": "2026-02-13T00:00:02Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"read_file tool call result should validate. Errors: {errors}") + + def test_validate_runtime_tool_call_result_apply_patch_schema(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-3", + "result_id": "result-3", + "agent_id": "agent-root", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "apply_patch", + "status": "success", + "summary": "Patch applied", + "result": {"files_changed": 1, "hunks_applied": 2}, + "duration_ms": 40, + "artifact_ref": "src/example.py", + "artifact_sha256": "a" * 64, + "truncated": False, + "finished_at": "2026-02-13T00:00:02Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"apply_patch tool call result should validate. Errors: {errors}") + + def test_validate_runtime_tool_call_result_artifact_hash_pairing(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-1", + "result_id": "result-1", + "agent_id": "agent-root", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "exec_cmd", + "status": "success", + "summary": "Command executed", + "result": {"command": "pytest -q", "mode": "summarized"}, + "duration_ms": 1200, + "exit_code": 0, + "working_dir": "/tmp/workspace", + "stdout_excerpt": "tests PASSED", + "stderr_excerpt": "", + "truncated": False, + "artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + "finished_at": "2026-02-13T00:00:02Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("artifact_sha256" in e for e in errors), + f"Tool call result should fail when artifact_ref lacks artifact_sha256. Errors: {errors}", + ) + + def test_validate_runtime_tool_call_request_invalid_exec_cmd_args(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "exec_cmd", + "args": {"command": "pytest -q"}, + "working_dir": "/tmp/workspace", + "created_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_request.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("mode" in e for e in errors), + f"Tool call request should fail when exec_cmd mode is missing. Errors: {errors}", + ) + + def test_validate_runtime_tool_call_request_checkpoint_branch_branch_name(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-branch-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Orchestrator", + "phase": "16a", + "step_id": "m1-core-foundation", + "tool_name": "checkpoint_branch", + "args": {"branch_name": "trinity/m1-core-foundation"}, + "working_dir": "/tmp/workspace", + "created_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_request.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"checkpoint_branch request should validate with branch_name. Errors: {errors}") + + def test_validate_runtime_tool_call_request_search_text_use_regex(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-search-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "search_text", + "args": {"pattern": "foo", "paths": ["README.md"], "use_regex": True}, + "working_dir": "/tmp/workspace", + "created_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_request.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"search_text request should validate with use_regex. Errors: {errors}") + + def test_validate_runtime_tool_call_request_git_diff_base_head(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-diff-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "git_diff", + "args": {"base_rev": "HEAD~1", "head_rev": "HEAD"}, + "working_dir": "/tmp/workspace", + "created_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_request.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"git_diff request should validate with base/head rev keys. Errors: {errors}") + + def test_validate_runtime_tool_call_request_move_file(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-move-1", + "agent_id": "agent-root", + "parent_id": None, + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "move_file", + "args": {"src_path": "src/a.py", "dst_path": "src/b.py", "overwrite": True}, + "working_dir": "/tmp/workspace", + "created_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_request.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"move_file request should validate. Errors: {errors}") + + def test_validate_runtime_tool_call_result_remove_file(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "call_id": "call-remove-1", + "result_id": "result-remove-1", + "agent_id": "agent-root", + "role": "Builder", + "phase": "16b", + "step_id": "m1-core-foundation", + "tool_name": "remove_file", + "status": "success", + "summary": "File removed", + "result": { + "path": "src/obsolete.py", + "removed": True, + "previously_missing": False, + }, + "duration_ms": 12, + "artifact_ref": "src/obsolete.py", + "artifact_sha256": "a" * 64, + "truncated": False, + "finished_at": "2026-02-13T00:00:02Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tool_call_result.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"remove_file result should validate. Errors: {errors}") + + def test_validate_runtime_session_event_log_missing_validation_closure_fails(self): + event = self._valid_session_event() + with tempfile.TemporaryDirectory() as tmp: + sessions_dir = os.path.join(tmp, ".trinity", "sessions") + os.makedirs(sessions_dir, exist_ok=True) + path = os.path.join(sessions_dir, "session.jsonl") + terminate = self._valid_session_event() + terminate["event_type"] = "TERMINATE" + terminate["event_id"] = "evt-2" + terminate["event_sequence"] = 2 + terminate["prev_event_sha256"] = event["event_sha256"] + terminate["content"] = { + "summary": "child completed", + "capture_level": "none", + "capture_decision_reason": "policy:default:none", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + "task_result_artifact_ref": ".trinity/runtime/spawns/child-1/task_result.json", + } + terminate["artifact_ref"] = ".trinity/runtime/spawns/child-1/task_result.json" + terminate["artifact_sha256"] = "d" * 64 + terminate["event_sha256"] = self._event_sha256(terminate) + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(event) + "\n") + f.write(json.dumps(terminate) + "\n") + errors = validate_runtime_file(self.repo_root, path) + self.assertTrue( + any("missing pass VALIDATION" in e for e in errors), + f"Session log should fail when spawn/terminate transaction lacks validation closure. Errors: {errors}", + ) + + def test_validate_runtime_task_input_invalid_target_files_not_in_context_pack_patterns(self): + task_input = self._valid_task_input_payload() + context_pack = self._valid_context_pack_16a_for_task_input() + context_pack["target_file_patterns"] = ["src/**"] + context_pack["allowed_write_paths"] = ["src/"] + with tempfile.TemporaryDirectory() as tmp: + task_input_path = os.path.join(tmp, "task_input.json") + context_pack_path = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1", "context_pack.json") + self._write_json(task_input_path, task_input) + self._write_json(context_pack_path, context_pack) + errors = validate_runtime_file(self.repo_root, task_input_path) + self.assertTrue( + any("target_files entry" in e for e in errors), + f"Task input should fail when target_files are not covered by context pack scope. Errors: {errors}", + ) + + def test_validate_runtime_task_input_step_id_mismatch_context_pack(self): + task_input = self._valid_task_input_payload() + context_pack = self._valid_context_pack_16a_for_task_input() + context_pack["step_id"] = "m2-another-step" + with tempfile.TemporaryDirectory() as tmp: + task_input_path = os.path.join(tmp, "task_input.json") + context_pack_path = os.path.join(tmp, ".trinity", "runtime", "spawns", "child-1", "context_pack.json") + self._write_json(task_input_path, task_input) + self._write_json(context_pack_path, context_pack) + errors = validate_runtime_file(self.repo_root, task_input_path) + self.assertTrue( + any("does not match context_pack step_id" in e for e in errors), + f"Task input should fail when step_id mismatches context pack. Errors: {errors}", + ) + + def test_validate_runtime_session_state(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "parent_id": "agent-root", + "active_phase": "16b", + "step_id": "m1-core-foundation", + "status": "waiting_child", + "pending_child_id": "child-9", + "pending_spawn_ref": ".trinity/runtime/spawns/child-9/task_input.json", + "spawn_log_ref": ".trinity/runtime/spawn_log.json", + "scratchpad_ref": ".trinity/runtime/scratchpads/scratchpad_child-9.json", + "last_event_id": "evt-10", + "retry_counters": {"planner": 0, "builder": 1, "verifier": 0, "milestone": 1}, + "updated_at": "2026-02-13T00:00:00Z", + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "session_state_parent-1.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Session state should validate. Errors: {errors}") + + def test_validate_runtime_spawn_log(self): + payload = { + "protocol_version": "trinity-runtime-v1", + "run_id": "run-1", + "entries": [ + { + "spawn_id": "spawn-1", + "parent_id": "agent-root", + "child_id": "child-1", + "purpose": "planner phase", + "phase": "16a", + "step_id": "m1-core-foundation", + "attempt": 1, + "status": "completed", + "task_input_ref": ".trinity/runtime/spawns/child-1/task_input.json", + "task_result_ref": ".trinity/runtime/spawns/child-1/task_result.json", + "created_at": "2026-02-13T00:00:00Z", + "updated_at": "2026-02-13T00:00:01Z", + } + ], + } + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "spawn_log.json") + self._write_json(path, payload) + errors = validate_runtime_file(self.repo_root, path) + self.assertEqual(errors, [], f"Spawn log should validate. Errors: {errors}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/pyproject.toml b/tools/pyproject.toml index ad8bb08e..59c106d0 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "specdev_tools" -version = "0.2.2" +version = "0.2.3" description = "Internal tooling for DevSpec Toolkit" authors = [{name = "Vichitra Collective"}] license = {text = "MIT"} diff --git a/tools/schema_registry.json b/tools/schema_registry.json index bbdff861..2ef813d6 100644 --- a/tools/schema_registry.json +++ b/tools/schema_registry.json @@ -21,5 +21,18 @@ "https://specdev.local/schema/15_scaffold.schema.json": "schema/15_scaffold.schema.json", "https://specdev.local/schema/16_impl_context.schema.json": "schema/16_impl_context.schema.json", "https://specdev.local/schema/02a_delivery_baseline.schema.json": "schema/02a_delivery_baseline.schema.json", - "https://specdev.local/schema/seed_manifest.schema.json": "schema/seed_manifest.schema.json" + "https://specdev.local/schema/seed_manifest.schema.json": "schema/seed_manifest.schema.json", + "https://specdev.local/schema/trinity/task_input.schema.json": "schema/trinity/task_input.schema.json", + "https://specdev.local/schema/trinity/context_pack.schema.json": "schema/trinity/context_pack.schema.json", + "https://specdev.local/schema/trinity/task_result.schema.json": "schema/trinity/task_result.schema.json", + "https://specdev.local/schema/trinity/tool_call_request.schema.json": "schema/trinity/tool_call_request.schema.json", + "https://specdev.local/schema/trinity/tool_call_result.schema.json": "schema/trinity/tool_call_result.schema.json", + "https://specdev.local/schema/trinity/utility_call.schema.json": "schema/trinity/utility_call.schema.json", + "https://specdev.local/schema/trinity/utility_result.schema.json": "schema/trinity/utility_result.schema.json", + "https://specdev.local/schema/trinity/session_event.schema.json": "schema/trinity/session_event.schema.json", + "https://specdev.local/schema/trinity/log_capture_policy.schema.json": "schema/trinity/log_capture_policy.schema.json", + "https://specdev.local/schema/trinity/eval_export_row.schema.json": "schema/trinity/eval_export_row.schema.json", + "https://specdev.local/schema/trinity/scratchpad_state.schema.json": "schema/trinity/scratchpad_state.schema.json", + "https://specdev.local/schema/trinity/session_state.schema.json": "schema/trinity/session_state.schema.json", + "https://specdev.local/schema/trinity/spawn_log.schema.json": "schema/trinity/spawn_log.schema.json" } diff --git a/tools/specdev_tools/cli.py b/tools/specdev_tools/cli.py index 7463e9af..b2767c01 100644 --- a/tools/specdev_tools/cli.py +++ b/tools/specdev_tools/cli.py @@ -8,6 +8,8 @@ def check_venv(): # Helper to check if we are running in a virtual environment # sys.prefix != sys.base_prefix is the standard check for venv/virtualenv + if os.environ.get("SPECDEV_SKIP_VENV_CHECK") == "1": + return if sys.prefix == sys.base_prefix: print("Error: Running without a virtual environment. Please activate 'devspec_env' or similar.", file=sys.stderr) sys.exit(1) @@ -22,6 +24,120 @@ def main(): v.add_argument("--repo-root", default=".") v.add_argument("--json", action="store_true", help="Output results as JSON") + vr = sub.add_parser("validate-runtime", help="Validate Trinity runtime protocol artifacts") + vr.add_argument("file") + vr.add_argument( + "--type", + dest="artifact_type", + choices=[ + "task_input", + "context_pack", + "task_result", + "tool_call_request", + "tool_call_result", + "session_event", + "log_capture_policy", + "eval_export_row", + "scratchpad_state", + "session_state", + "spawn_log", + ], + help="Optional explicit runtime artifact type", + ) + vr.add_argument("--repo-root", default=".") + vr.add_argument("--json", action="store_true", help="Output results as JSON") + + tr = sub.add_parser("trinity", help="Run Trinity runtime orchestration for one milestone") + tr.add_argument("--step-id", help="Explicit roadmap milestone id to execute") + tr.add_argument( + "--resume", + action="store_true", + help="Resume from a matching session_state under .trinity/runtime (requires disambiguation if multiple)", + ) + tr.add_argument("--resume-run-id", help="Explicit run_id to resume (disambiguates multiple session states)") + tr.add_argument( + "--answer", + action="append", + default=[], + help="Answer to pending Trinity clarification questions (repeatable)", + ) + tr.add_argument("--mode", choices=["llm", "deterministic"], help="Override runtime.execution_mode for this run") + tr.add_argument("--repo-root", default=".") + tr.add_argument("--json", action="store_true", help="Output run summary as JSON") + + tc = sub.add_parser("trinity-child", help=argparse.SUPPRESS) + tc.add_argument("--repo-root", required=True) + tc.add_argument("--step-id", required=True) + tc.add_argument("--phase", required=True, choices=["16a", "16b", "16c", "utility"]) + tc.add_argument("--role", required=True) + tc.add_argument("--child-id", required=True) + tc.add_argument("--milestone-path", required=True) + tc.add_argument("--task-input", required=True) + tc.add_argument("--context-pack", required=True) + tc.add_argument("--task-result", required=True) + tc.add_argument("--session-log", required=True) + tc.add_argument("--run-id", required=True) + tc.add_argument("--parent-id", required=True) + tc.add_argument("--mode", choices=["llm", "deterministic"]) + + ee = sub.add_parser("trinity-export-eval", help="Export eval rows from a Trinity session log") + ee.add_argument("session_log") + ee.add_argument("--out", help="Optional output JSONL path for exported rows") + ee.add_argument("--repo-root", default=".") + ee.add_argument("--json", action="store_true", help="Output rows/results as JSON") + + rp = sub.add_parser("trinity-replay", help="Replay and verify a Trinity session log") + rp.add_argument("session_log") + rp.add_argument("--repo-root", default=".") + replay_mode = rp.add_mutually_exclusive_group() + replay_mode.add_argument("--strict", dest="strict", action="store_true", help="Treat replay warnings as failure") + replay_mode.add_argument( + "--allow-warnings", + dest="strict", + action="store_false", + help="Allow replay warnings without failing the replay report", + ) + rp.set_defaults(strict=True) + rp.add_argument("--out", help="Optional output path for replay report JSON") + rp.add_argument("--json", action="store_true", help="Output replay report as JSON") + + db = sub.add_parser("trinity-dashboard", help="Build aggregated Trinity eval/replay dashboard") + db.add_argument("--rows-glob", required=True, help="Glob for eval row JSONL files") + db.add_argument("--replay-glob", required=True, help="Glob for replay report JSON files") + db.add_argument("--out-json", help="Optional output JSON path") + db.add_argument("--out-md", help="Optional output markdown path") + db.add_argument("--json", action="store_true", help="Output dashboard summary as JSON") + + pb = sub.add_parser("trinity-publish-eval", help="Bundle and optionally publish Trinity eval artifacts") + pb.add_argument("--rows-glob", required=True, help="Glob for eval row JSONL files") + pb.add_argument("--replay-glob", required=True, help="Glob for replay report JSON files") + pb.add_argument("--dashboard-json", help="Optional dashboard JSON path to embed") + pb.add_argument("--out", help="Optional output path for bundled payload JSON") + pb.add_argument("--source", help="Optional source label, e.g. github-actions::") + pb.add_argument("--max-rows", type=int, default=50000, help="Maximum rows to include in the payload") + pb.add_argument("--endpoint", help="Optional explicit HTTP endpoint for publish") + pb.add_argument("--endpoint-env", default="TRINITY_EVAL_EXPORT_ENDPOINT", help="Environment variable to resolve endpoint from") + pb.add_argument("--auth-token", help="Optional explicit bearer token for publish") + pb.add_argument("--auth-token-env", default="TRINITY_EVAL_EXPORT_TOKEN", help="Environment variable to resolve bearer token from") + pb.add_argument("--timeout-seconds", type=int, default=20, help="Publish HTTP timeout in seconds") + pb.add_argument("--require-publish", action="store_true", help="Fail if endpoint is not configured or publish fails") + pb.add_argument("--json", action="store_true", help="Output bundle/publish result as JSON") + + rm = sub.add_parser("trinity-remediate", help="Generate remediation/resume actions from replay findings") + rm.add_argument("replay_report") + rm.add_argument("--session-log", help="Optional source session log for resume task-input generation") + rm.add_argument("--emit-session-state", help="Optional output path for generated session_state JSON") + rm.add_argument("--emit-task-input", help="Optional output path for generated resume task_input JSON") + rm.add_argument( + "--missing-resume-source-policy", + choices=["soft", "hard"], + default="hard", + help="How to treat missing source task_input when generating resume artifacts", + ) + rm.add_argument("--out", help="Optional output path for remediation plan JSON") + rm.add_argument("--repo-root", default=".") + rm.add_argument("--json", action="store_true", help="Output remediation plan as JSON") + va = sub.add_parser("validate-all") va.add_argument("spec_dir") va.add_argument("--repo-root", default=".") @@ -77,7 +193,7 @@ def main(): args = p.parse_args() - if args.repo_root == ".": + if hasattr(args, "repo_root") and args.repo_root == ".": # Auto-detect toolkit root by scanning immediate subdirectories current_dir = Path(".") found = False @@ -107,6 +223,194 @@ def main(): for e in errs: print(e, file=sys.stderr) sys.exit(1) print("OK") + elif args.cmd == "validate-runtime": + from .trinity_runtime_validate import validate_runtime_file + repo_root = os.path.abspath(args.repo_root) + file_path = os.path.abspath(args.file) + errs = validate_runtime_file(repo_root, file_path, args.artifact_type) + if args.json: + output = [] + if errs: + for e in errs: + output.append({"file": file_path, "error": e, "status": "FAIL"}) + else: + output.append({"file": file_path, "status": "PASS"}) + print(json.dumps(output, indent=2)) + else: + if errs: + for e in errs: + print(e, file=sys.stderr) + sys.exit(1) + print("OK") + elif args.cmd == "trinity": + from .trinity_runtime import run_trinity + repo_root = os.path.abspath(args.repo_root) + result = run_trinity( + repo_root=repo_root, + step_id=args.step_id, + resume=bool(args.resume), + mode_override=args.mode, + answers=args.answer, + resume_run_id=args.resume_run_id, + ) + if args.json: + print(json.dumps(result, indent=2)) + else: + status = str(result.get("status", "unknown")).upper() + print(f"{status} - step_id={result.get('step_id')} run_id={result.get('run_id')}") + if result.get("milestone_artifact"): + print(f"Milestone: {result['milestone_artifact']}") + if result.get("anchor_artifact"): + print(f"Anchor: {result['anchor_artifact']}") + if result.get("session_log"): + print(f"Session Log: {result['session_log']}") + if result.get("errors"): + for err in result["errors"]: + print(err, file=sys.stderr) + if result.get("status") not in {"completed", "questions"}: + sys.exit(1) + elif args.cmd == "trinity-child": + from .trinity_runtime import run_trinity_child + repo_root = os.path.abspath(args.repo_root) + result = run_trinity_child( + repo_root=repo_root, + step_id=args.step_id, + phase=args.phase, + role=args.role, + child_id=args.child_id, + milestone_path=args.milestone_path, + task_input_path=args.task_input, + context_pack_path=args.context_pack, + task_result_path=args.task_result, + session_log_path=args.session_log, + run_id=args.run_id, + parent_id=args.parent_id, + mode_override=args.mode, + ) + print(json.dumps(result, indent=2)) + elif args.cmd == "trinity-export-eval": + from .trinity_eval_export import export_eval_rows + repo_root = os.path.abspath(args.repo_root) + session_log = os.path.abspath(args.session_log) + out_path = os.path.abspath(args.out) if args.out else None + rows, errs = export_eval_rows(repo_root, session_log, out_path=out_path) + if args.json: + print(json.dumps({"rows": rows, "errors": errs}, indent=2)) + else: + if errs: + for e in errs: + print(e, file=sys.stderr) + sys.exit(1) + if out_path: + print(f"OK - exported {len(rows)} rows to {out_path}") + else: + print(f"OK - exported {len(rows)} rows") + elif args.cmd == "trinity-replay": + from .trinity_replay import replay_session + repo_root = os.path.abspath(args.repo_root) + session_log = os.path.abspath(args.session_log) + report = replay_session(repo_root, session_log, strict=args.strict) + out_path = os.path.abspath(args.out) if args.out else None + if out_path: + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + if args.json: + print(json.dumps(report, indent=2)) + else: + print( + f"{report.get('status', 'unknown').upper()} - " + f"events={report.get('summary', {}).get('total_events', 0)} " + f"warnings={len(report.get('warnings', []))} " + f"errors={len(report.get('errors', []))}" + ) + if out_path: + print(f"Report: {out_path}") + if report.get("status") == "failed": + sys.exit(1) + elif args.cmd == "trinity-dashboard": + from .trinity_dashboard import write_dashboard + summary, markdown = write_dashboard( + eval_rows_glob=args.rows_glob, + replay_reports_glob=args.replay_glob, + out_json=args.out_json, + out_md=args.out_md, + ) + if args.json: + print(json.dumps(summary, indent=2)) + else: + print(markdown) + elif args.cmd == "trinity-publish-eval": + from .trinity_eval_publish import publish_eval_bundle + + bundle, publish_result, errs = publish_eval_bundle( + rows_glob=args.rows_glob, + replay_glob=args.replay_glob, + dashboard_json=args.dashboard_json, + out_path=args.out, + source=args.source, + max_rows=args.max_rows, + endpoint=args.endpoint, + endpoint_env=args.endpoint_env, + auth_token=args.auth_token, + auth_token_env=args.auth_token_env, + require_publish=args.require_publish, + timeout_seconds=args.timeout_seconds, + ) + if args.json: + print(json.dumps({"bundle": bundle, "publish_result": publish_result, "errors": errs}, indent=2)) + else: + print( + f"{publish_result.get('status', 'unknown').upper()} - " + f"rows={bundle.get('row_count_exported', 0)} " + f"endpoint={publish_result.get('endpoint') or 'none'} " + f"http_status={publish_result.get('http_status')}" + ) + if args.out: + print(f"Bundle: {os.path.abspath(args.out)}") + if errs: + for e in errs: + print(e, file=sys.stderr) + sys.exit(1) + elif args.cmd == "trinity-remediate": + from .trinity_remediation import build_remediation_plan + repo_root = os.path.abspath(args.repo_root) + replay_report = os.path.abspath(args.replay_report) + session_log = os.path.abspath(args.session_log) if args.session_log else None + emit_session_state = os.path.abspath(args.emit_session_state) if args.emit_session_state else None + emit_task_input = os.path.abspath(args.emit_task_input) if args.emit_task_input else None + plan, errs = build_remediation_plan( + repo_root=repo_root, + replay_report_path=replay_report, + session_log_path=session_log, + emit_session_state_path=emit_session_state, + emit_task_input_path=emit_task_input, + missing_resume_source_policy=args.missing_resume_source_policy, + ) + out_path = os.path.abspath(args.out) if args.out else None + if out_path: + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "w", encoding="utf-8") as f: + json.dump(plan, f, indent=2) + if args.json: + print(json.dumps({"plan": plan, "errors": errs}, indent=2)) + else: + print( + f"{plan.get('status', 'unknown').upper()} - " + f"actions={len(plan.get('actions', []))} " + f"warnings={len(plan.get('warnings', []))} " + f"errors={len(errs)}" + ) + if out_path: + print(f"Plan: {out_path}") + if emit_session_state: + print(f"Session state: {emit_session_state}") + if emit_task_input: + print(f"Task input: {emit_task_input}") + if errs: + for e in errs: + print(e, file=sys.stderr) + sys.exit(1) elif args.cmd == "validate-all": from .validate import validate_dir repo_root = os.path.abspath(args.repo_root) @@ -188,7 +492,7 @@ def main(): print("AI Interaction Guide:") print("1. Locate the prompt file in prompts/prompt_XX_stepname.md") print("2. Copy the full content into your AI assistant") - print("3. Paste only the fenced JSON block into spec/NN_name.json") + print("3. Write or update spec/NN_name.json directly (disk-first); do not rely on fenced JSON chat output") print("4. Validate with: python -m specdev_tools.cli validate spec/NN_name.json --repo-root ") print("5. Ensure all IDs use kebab-case format") print("6. No examples should be included in the AI output") diff --git a/tools/specdev_tools/seed_lint.py b/tools/specdev_tools/seed_lint.py index 1ef32d23..b311f01f 100644 --- a/tools/specdev_tools/seed_lint.py +++ b/tools/specdev_tools/seed_lint.py @@ -42,9 +42,13 @@ def _collect_required_seeds(manifest: Dict, step_id: str) -> Set[str]: global_required = set(manifest.get("global_seed_order", [])) step_requirements = manifest.get("step_requirements", {}) if step_id == "16": - required = set() - for key in ("16a", "16b", "16c"): - required.update(step_requirements.get(key, [])) + explicit_step_16 = step_requirements.get("16") + if isinstance(explicit_step_16, list): + required = set(explicit_step_16) + else: + required = set() + for key in ("16a", "16b", "16c"): + required.update(step_requirements.get(key, [])) required.update(global_required) return required required = set(step_requirements.get(step_id, [])) diff --git a/tools/specdev_tools/trinity_dashboard.py b/tools/specdev_tools/trinity_dashboard.py new file mode 100644 index 00000000..28e8f6c2 --- /dev/null +++ b/tools/specdev_tools/trinity_dashboard.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import glob +import json +import os +from typing import Any, Dict, List, Tuple + + +def _load_json(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _load_jsonl(path: str) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as f: + for raw_line in f: + stripped = raw_line.strip() + if not stripped: + continue + rows.append(json.loads(stripped)) + return rows + + +def build_dashboard( + eval_rows_glob: str, + replay_reports_glob: str, +) -> Tuple[Dict[str, Any], str]: + eval_row_files = sorted(glob.glob(eval_rows_glob)) + replay_files = sorted(glob.glob(replay_reports_glob)) + + total_rows = 0 + capture_level_counts: Dict[str, int] = {} + event_type_counts: Dict[str, int] = {} + token_prompt_total = 0 + token_completion_total = 0 + token_total = 0 + redaction_total = 0 + + for path in eval_row_files: + for row in _load_jsonl(path): + total_rows += 1 + capture = str(row.get("capture_level", "unknown")) + capture_level_counts[capture] = capture_level_counts.get(capture, 0) + 1 + event_type = str(row.get("event_type", "UNKNOWN")) + event_type_counts[event_type] = event_type_counts.get(event_type, 0) + 1 + token_prompt_total += int(row.get("token_prompt", 0) or 0) + token_completion_total += int(row.get("token_completion", 0) or 0) + token_total += int(row.get("token_total", 0) or 0) + redaction_total += int(row.get("redaction_total_replacements", 0) or 0) + + replay_status_counts: Dict[str, int] = {} + replay_warnings = 0 + replay_errors = 0 + replay_artifact_checked = 0 + replay_artifact_mismatch = 0 + replay_artifact_missing = 0 + replay_total_events = 0 + + for path in replay_files: + report = _load_json(path) + status = str(report.get("status", "unknown")) + replay_status_counts[status] = replay_status_counts.get(status, 0) + 1 + replay_warnings += len(report.get("warnings", [])) + replay_errors += len(report.get("errors", [])) + summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {} + replay_total_events += int(summary.get("total_events", 0) or 0) + artifact = report.get("artifact_verification", {}) if isinstance(report.get("artifact_verification"), dict) else {} + replay_artifact_checked += int(artifact.get("checked", 0) or 0) + replay_artifact_mismatch += int(artifact.get("mismatch", 0) or 0) + replay_artifact_missing += int(artifact.get("missing", 0) or 0) + + summary: Dict[str, Any] = { + "files": { + "eval_row_files": eval_row_files, + "replay_report_files": replay_files, + }, + "totals": { + "sessions": len(replay_files), + "eval_rows": total_rows, + "replay_events": replay_total_events, + "token_prompt": token_prompt_total, + "token_completion": token_completion_total, + "token_total": token_total, + "redaction_total_replacements": redaction_total, + "replay_warnings": replay_warnings, + "replay_errors": replay_errors, + "artifact_checked": replay_artifact_checked, + "artifact_mismatch": replay_artifact_mismatch, + "artifact_missing": replay_artifact_missing, + }, + "capture_level_counts": capture_level_counts, + "event_type_counts": event_type_counts, + "replay_status_counts": replay_status_counts, + } + + lines: List[str] = [] + lines.append("## Trinity Eval Dashboard") + lines.append("") + lines.append(f"- Sessions: {summary['totals']['sessions']}") + lines.append(f"- Eval rows: {summary['totals']['eval_rows']}") + lines.append(f"- Replay events: {summary['totals']['replay_events']}") + lines.append(f"- Tokens (prompt/completion/total): {token_prompt_total}/{token_completion_total}/{token_total}") + lines.append(f"- Redaction replacements: {redaction_total}") + lines.append(f"- Replay warnings/errors: {replay_warnings}/{replay_errors}") + lines.append(f"- Artifact checks (checked/mismatch/missing): {replay_artifact_checked}/{replay_artifact_mismatch}/{replay_artifact_missing}") + lines.append("") + lines.append("### Replay Status") + for key in sorted(replay_status_counts): + lines.append(f"- {key}: {replay_status_counts[key]}") + lines.append("") + lines.append("### Capture Levels") + for key in sorted(capture_level_counts): + lines.append(f"- {key}: {capture_level_counts[key]}") + lines.append("") + lines.append("### Event Types") + for key in sorted(event_type_counts): + lines.append(f"- {key}: {event_type_counts[key]}") + lines.append("") + + return summary, "\n".join(lines) + + +def write_dashboard( + eval_rows_glob: str, + replay_reports_glob: str, + out_json: str | None = None, + out_md: str | None = None, +) -> Tuple[Dict[str, Any], str]: + summary, markdown = build_dashboard(eval_rows_glob, replay_reports_glob) + if out_json: + out_json_abs = os.path.abspath(out_json) + os.makedirs(os.path.dirname(out_json_abs), exist_ok=True) + with open(out_json_abs, "w", encoding="utf-8") as f: + json.dump(summary, f, indent=2) + if out_md: + out_md_abs = os.path.abspath(out_md) + os.makedirs(os.path.dirname(out_md_abs), exist_ok=True) + with open(out_md_abs, "w", encoding="utf-8") as f: + f.write(markdown + "\n") + return summary, markdown diff --git a/tools/specdev_tools/trinity_eval_export.py b/tools/specdev_tools/trinity_eval_export.py new file mode 100644 index 00000000..1a219132 --- /dev/null +++ b/tools/specdev_tools/trinity_eval_export.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Tuple + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +from .registry import SchemaRegistry +from .trinity_runtime_validate import validate_runtime_file + + +def _registry_for(registry: SchemaRegistry) -> Registry: + store = {uri: Resource.from_contents(schema) for uri, schema in registry.store.items()} + return Registry().with_resources(store.items()) + + +def _load_session_events(session_log_path: str) -> Tuple[List[Dict[str, Any]], List[str]]: + events: List[Dict[str, Any]] = [] + errors: List[str] = [] + with open(session_log_path, "r", encoding="utf-8") as f: + for idx, raw_line in enumerate(f, start=1): + stripped = raw_line.strip() + if not stripped: + continue + try: + events.append(json.loads(stripped)) + except json.JSONDecodeError as e: + errors.append(f"{session_log_path}:{idx}: invalid json line ({e})") + return events, errors + + +_FINDING_SEVERITY_RANK = { + "blocking": 4, + "major": 3, + "minor": 2, + "nit": 1, +} + + +def _resolve_ref(session_log_path: str, repo_root: str, ref: str) -> str: + if os.path.isabs(ref): + return ref + local = os.path.abspath(os.path.join(os.path.dirname(session_log_path), ref)) + if os.path.exists(local): + return local + return os.path.abspath(os.path.join(repo_root, ref)) + + +def _safe_load_json(path: str) -> Dict[str, Any] | None: + try: + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + if isinstance(payload, dict): + return payload + except Exception: + return None + return None + + +def _max_finding_severity(findings: Any) -> str | None: + if not isinstance(findings, list): + return None + best: str | None = None + best_rank = 0 + for finding in findings: + if not isinstance(finding, dict): + continue + severity = finding.get("severity") + if not isinstance(severity, str): + continue + rank = _FINDING_SEVERITY_RANK.get(severity, 0) + if rank > best_rank: + best_rank = rank + best = severity + return best + + +def _derive_eval_labels(repo_root: str, session_log_path: str, event: Dict[str, Any]) -> Dict[str, Any]: + content = event.get("content", {}) if isinstance(event.get("content"), dict) else {} + task_result_ref = content.get("task_result_artifact_ref") + labels: Dict[str, Any] = { + "task_result_artifact_ref": task_result_ref if isinstance(task_result_ref, str) else None, + "phase_outcome": None, + "review_verdict": None, + "checklist_ids": None, + "finding_count": None, + "max_finding_severity": None, + "remediation_required": None, + } + if not isinstance(task_result_ref, str) or not task_result_ref: + return labels + + task_result_path = _resolve_ref(session_log_path, repo_root, task_result_ref) + task_result = _safe_load_json(task_result_path) + if not task_result: + return labels + + status = task_result.get("status") + if isinstance(status, str) and status in {"success", "blocked", "failed", "questions"}: + labels["phase_outcome"] = status + + findings = task_result.get("findings", []) + if isinstance(findings, list): + labels["finding_count"] = len(findings) + labels["max_finding_severity"] = _max_finding_severity(findings) + + if labels["phase_outcome"] in {"blocked", "failed"}: + labels["remediation_required"] = True + + artifacts = task_result.get("artifacts", []) + if not isinstance(artifacts, list): + return labels + + checklist_ids: list[str] = [] + for artifact_ref in artifacts: + if not isinstance(artifact_ref, str) or not artifact_ref.endswith(".json"): + continue + artifact_path = _resolve_ref(session_log_path, repo_root, artifact_ref) + artifact = _safe_load_json(artifact_path) + if not artifact: + continue + if artifact.get("$schema") != "https://specdev.local/schema/16_impl_context.schema.json": + continue + + review = artifact.get("review", {}) + if isinstance(review, dict): + verdict = review.get("verdict") + if isinstance(verdict, str) and verdict in {"verified", "deferred", "rejected"}: + labels["review_verdict"] = verdict + + plan = artifact.get("plan", {}) + if isinstance(plan, dict): + checklist = plan.get("spec_alignment", {}).get("checklist", []) + if isinstance(checklist, list): + for item in checklist: + if not isinstance(item, dict): + continue + cid = item.get("id") + if isinstance(cid, str) and cid and cid not in checklist_ids: + checklist_ids.append(cid) + + if checklist_ids: + labels["checklist_ids"] = checklist_ids + + if labels["remediation_required"] is None: + max_sev = labels.get("max_finding_severity") + review_verdict = labels.get("review_verdict") + labels["remediation_required"] = bool( + max_sev in {"blocking", "major"} or review_verdict in {"deferred", "rejected"} + ) + + return labels + + +def _event_to_eval_row(repo_root: str, session_log_path: str, event: Dict[str, Any]) -> Dict[str, Any]: + content = event.get("content", {}) if isinstance(event.get("content"), dict) else {} + metadata = event.get("metadata", {}) if isinstance(event.get("metadata"), dict) else {} + token_usage = metadata.get("token_usage", {}) if isinstance(metadata.get("token_usage"), dict) else {} + redaction_stats = metadata.get("redaction_stats", {}) if isinstance(metadata.get("redaction_stats"), dict) else {} + redaction_classes = redaction_stats.get("classes_detected", []) + tool_call = content.get("tool_call", {}) if isinstance(content.get("tool_call"), dict) else {} + tool_result = content.get("tool_result", {}) if isinstance(content.get("tool_result"), dict) else {} + validation = content.get("validation", {}) if isinstance(content.get("validation"), dict) else {} + labels = _derive_eval_labels(repo_root, session_log_path, event) + + return { + "run_id": event.get("run_id"), + "event_id": event.get("event_id"), + "event_sequence": event.get("event_sequence"), + "timestamp": event.get("timestamp"), + "event_type": event.get("event_type"), + "role": event.get("role"), + "phase_id": event.get("phase_id"), + "step_id": event.get("step_id"), + "capture_level": content.get("capture_level"), + "prompt_artifact_ref": content.get("prompt_artifact_ref"), + "prompt_sha256": content.get("prompt_sha256"), + "response_artifact_ref": content.get("response_artifact_ref"), + "response_sha256": content.get("response_sha256"), + "artifact_ref": event.get("artifact_ref"), + "artifact_sha256": event.get("artifact_sha256"), + "diff_ref": event.get("diff_ref"), + "task_result_artifact_ref": labels.get("task_result_artifact_ref"), + "tool_name": tool_call.get("name"), + "tool_command": tool_result.get("command"), + "tool_exit_code": tool_result.get("exit_code"), + "validation_schema": validation.get("schema"), + "validation_deep_validator": validation.get("deep_validator"), + "validation_governance": validation.get("governance"), + "phase_outcome": labels.get("phase_outcome"), + "review_verdict": labels.get("review_verdict"), + "checklist_ids": labels.get("checklist_ids"), + "finding_count": labels.get("finding_count"), + "max_finding_severity": labels.get("max_finding_severity"), + "remediation_required": labels.get("remediation_required"), + "redaction_applied": metadata.get("redaction_applied"), + "redaction_total_replacements": redaction_stats.get("total_replacements", 0), + "redaction_classes": redaction_classes if isinstance(redaction_classes, list) else [], + "token_prompt": token_usage.get("prompt", 0), + "token_completion": token_usage.get("completion", 0), + "token_total": token_usage.get("total", 0), + "event_sha256": event.get("event_sha256"), + "prev_event_sha256": event.get("prev_event_sha256"), + } + + +def _validate_eval_rows(repo_root: str, rows: List[Dict[str, Any]], source_path: str) -> List[str]: + registry = SchemaRegistry(repo_root) + schema_uri = "https://specdev.local/schema/trinity/eval_export_row.schema.json" + schema = registry.load(schema_uri) + reg = _registry_for(registry) + validator = Draft202012Validator( + schema, + registry=reg, + format_checker=Draft202012Validator.FORMAT_CHECKER, + ) + + errors: List[str] = [] + for idx, row in enumerate(rows, start=1): + row_errors = sorted(validator.iter_errors(row), key=lambda e: list(e.path)) + for e in row_errors: + path = "/".join(map(str, e.path)) + errors.append(f"{source_path}:row[{idx}]:{path}: {e.message}") + return errors + + +def export_eval_rows( + repo_root: str, + session_log_path: str, + out_path: str | None = None, +) -> Tuple[List[Dict[str, Any]], List[str]]: + """ + Convert a validated Trinity session log into eval-export rows. + + Returns: + (rows, errors) + """ + runtime_errors = validate_runtime_file(repo_root, session_log_path, "session_event") + if runtime_errors: + return [], runtime_errors + + events, parse_errors = _load_session_events(session_log_path) + if parse_errors: + return [], parse_errors + + rows = [_event_to_eval_row(repo_root, session_log_path, event) for event in events] + row_errors = _validate_eval_rows(repo_root, rows, session_log_path) + if row_errors: + return rows, row_errors + + if out_path: + out_abs = os.path.abspath(out_path) + os.makedirs(os.path.dirname(out_abs), exist_ok=True) + with open(out_abs, "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + return rows, [] diff --git a/tools/specdev_tools/trinity_eval_publish.py b/tools/specdev_tools/trinity_eval_publish.py new file mode 100644 index 00000000..bc82464c --- /dev/null +++ b/tools/specdev_tools/trinity_eval_publish.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import datetime as dt +import glob +import json +import os +import urllib.error +import urllib.request +from typing import Any, Dict, List, Optional, Tuple + + +def _utc_now_iso() -> str: + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _load_json(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _load_jsonl(path: str) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as f: + for raw_line in f: + stripped = raw_line.strip() + if not stripped: + continue + rows.append(json.loads(stripped)) + return rows + + +def _ci_context() -> Dict[str, str]: + keys = [ + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_REF", + "GITHUB_RUN_ID", + "GITHUB_RUN_ATTEMPT", + "GITHUB_WORKFLOW", + "GITHUB_ACTOR", + ] + out: Dict[str, str] = {} + for key in keys: + value = os.getenv(key) + if value: + out[key] = value + return out + + +def _summarize_replay_report(path: str, report: Dict[str, Any]) -> Dict[str, Any]: + summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {} + artifact = report.get("artifact_verification", {}) if isinstance(report.get("artifact_verification"), dict) else {} + warnings = report.get("warnings", []) + errors = report.get("errors", []) + warning_count = len(warnings) if isinstance(warnings, list) else 0 + error_count = len(errors) if isinstance(errors, list) else 0 + return { + "path": path, + "status": report.get("status", "unknown"), + "run_id": summary.get("run_id"), + "total_events": int(summary.get("total_events", 0) or 0), + "warning_count": warning_count, + "error_count": error_count, + "artifact_checked": int(artifact.get("checked", 0) or 0), + "artifact_mismatch": int(artifact.get("mismatch", 0) or 0), + "artifact_missing": int(artifact.get("missing", 0) or 0), + } + + +def _default_summary(rows: List[Dict[str, Any]], replay_summaries: List[Dict[str, Any]]) -> Dict[str, Any]: + token_prompt = 0 + token_completion = 0 + token_total = 0 + capture_level_counts: Dict[str, int] = {} + replay_status_counts: Dict[str, int] = {} + replay_warnings = 0 + replay_errors = 0 + replay_events = 0 + artifact_checked = 0 + artifact_mismatch = 0 + artifact_missing = 0 + + for row in rows: + capture = str(row.get("capture_level", "unknown")) + capture_level_counts[capture] = capture_level_counts.get(capture, 0) + 1 + token_prompt += int(row.get("token_prompt", 0) or 0) + token_completion += int(row.get("token_completion", 0) or 0) + token_total += int(row.get("token_total", 0) or 0) + + for replay in replay_summaries: + status = str(replay.get("status", "unknown")) + replay_status_counts[status] = replay_status_counts.get(status, 0) + 1 + replay_warnings += int(replay.get("warning_count", 0) or 0) + replay_errors += int(replay.get("error_count", 0) or 0) + replay_events += int(replay.get("total_events", 0) or 0) + artifact_checked += int(replay.get("artifact_checked", 0) or 0) + artifact_mismatch += int(replay.get("artifact_mismatch", 0) or 0) + artifact_missing += int(replay.get("artifact_missing", 0) or 0) + + return { + "totals": { + "sessions": len(replay_summaries), + "eval_rows": len(rows), + "replay_events": replay_events, + "token_prompt": token_prompt, + "token_completion": token_completion, + "token_total": token_total, + "replay_warnings": replay_warnings, + "replay_errors": replay_errors, + "artifact_checked": artifact_checked, + "artifact_mismatch": artifact_mismatch, + "artifact_missing": artifact_missing, + }, + "capture_level_counts": capture_level_counts, + "replay_status_counts": replay_status_counts, + } + + +def _resolve_env_or_value(value: Optional[str], env_name: Optional[str]) -> Optional[str]: + if isinstance(value, str) and value.strip(): + return value.strip() + if isinstance(env_name, str) and env_name.strip(): + env_value = os.getenv(env_name.strip(), "").strip() + if env_value: + return env_value + return None + + +def _publish_json( + endpoint: str, + payload: Dict[str, Any], + auth_token: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[Optional[int], Optional[str], List[str]]: + errors: List[str] = [] + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = { + "Content-Type": "application/json", + "User-Agent": "specdev-tools/trinity-publish-eval", + } + if isinstance(auth_token, str) and auth_token: + headers["Authorization"] = f"Bearer {auth_token}" + + request = urllib.request.Request(endpoint, data=body, headers=headers, method="POST") + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + status = response.getcode() + response_body = response.read(4096).decode("utf-8", errors="replace") + if status >= 400: + errors.append(f"publish failed with HTTP status {status}") + return status, response_body, errors + except urllib.error.HTTPError as e: + response_body = e.read(4096).decode("utf-8", errors="replace") + errors.append(f"publish failed with HTTP status {e.code}") + return e.code, response_body, errors + except urllib.error.URLError as e: + errors.append(f"publish failed ({e})") + return None, None, errors + except Exception as e: + errors.append(f"publish failed ({e})") + return None, None, errors + + +def publish_eval_bundle( + rows_glob: str, + replay_glob: str, + dashboard_json: Optional[str] = None, + out_path: Optional[str] = None, + source: Optional[str] = None, + max_rows: int = 50000, + endpoint: Optional[str] = None, + endpoint_env: Optional[str] = None, + auth_token: Optional[str] = None, + auth_token_env: Optional[str] = None, + require_publish: bool = False, + timeout_seconds: int = 20, +) -> Tuple[Dict[str, Any], Dict[str, Any], List[str]]: + errors: List[str] = [] + + row_files = sorted(glob.glob(rows_glob)) + replay_files = sorted(glob.glob(replay_glob)) + + rows: List[Dict[str, Any]] = [] + rows_truncated = False + for path in row_files: + for row in _load_jsonl(path): + if len(rows) >= max_rows: + rows_truncated = True + break + rows.append(row) + if rows_truncated: + break + + replay_summaries: List[Dict[str, Any]] = [] + for path in replay_files: + report = _load_json(path) + replay_summaries.append(_summarize_replay_report(path, report)) + + dashboard: Optional[Dict[str, Any]] = None + if dashboard_json: + dashboard_path = os.path.abspath(dashboard_json) + if os.path.exists(dashboard_path): + dashboard = _load_json(dashboard_path) + else: + errors.append(f"dashboard JSON not found: {dashboard_path}") + + summary = dashboard if isinstance(dashboard, dict) else _default_summary(rows, replay_summaries) + bundle: Dict[str, Any] = { + "schema_version": "trinity-eval-export-v1", + "generated_at": _utc_now_iso(), + "source": source or "local", + "ci_context": _ci_context(), + "files": { + "row_files": row_files, + "replay_files": replay_files, + "dashboard_json": os.path.abspath(dashboard_json) if dashboard_json else None, + }, + "summary": summary, + "row_count_exported": len(rows), + "rows_truncated": rows_truncated, + "rows": rows, + "replay_reports": replay_summaries, + } + + resolved_endpoint = _resolve_env_or_value(endpoint, endpoint_env) + resolved_auth = _resolve_env_or_value(auth_token, auth_token_env) + publish_result: Dict[str, Any] = { + "status": "skipped", + "endpoint": resolved_endpoint, + "http_status": None, + "response_body_preview": None, + } + + if resolved_endpoint: + status, response_preview, publish_errors = _publish_json( + endpoint=resolved_endpoint, + payload=bundle, + auth_token=resolved_auth, + timeout_seconds=timeout_seconds, + ) + publish_result["http_status"] = status + publish_result["response_body_preview"] = response_preview + if publish_errors: + publish_result["status"] = "failed" + errors.extend(publish_errors) + else: + publish_result["status"] = "published" + elif require_publish: + publish_result["status"] = "failed" + errors.append("publish endpoint is not configured") + + if out_path: + out_abs = os.path.abspath(out_path) + os.makedirs(os.path.dirname(out_abs), exist_ok=True) + payload = {"bundle": bundle, "publish_result": publish_result} + with open(out_abs, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + return bundle, publish_result, errors diff --git a/tools/specdev_tools/trinity_remediation.py b/tools/specdev_tools/trinity_remediation.py new file mode 100644 index 00000000..c44d7bb1 --- /dev/null +++ b/tools/specdev_tools/trinity_remediation.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import datetime as dt +import json +import os +from typing import Any, Dict, List, Optional, Tuple + +from .trinity_runtime_validate import validate_runtime_file + +PROTO_VER = "trinity-runtime-v1" + + +def _utc_now_iso() -> str: + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _load_json(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _load_jsonl(path: str) -> List[Dict[str, Any]]: + events: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as f: + for raw_line in f: + stripped = raw_line.strip() + if not stripped: + continue + events.append(json.loads(stripped)) + return events + + +def _infer_phase(phase_id: Any) -> Optional[str]: + if not isinstance(phase_id, str): + return None + for candidate in ("16a", "16b", "16c"): + if candidate in phase_id: + return candidate + return None + + +def _find_latest_spawn_task_input_ref(events: List[Dict[str, Any]]) -> Optional[str]: + for event in reversed(events): + if event.get("event_type") != "SPAWN": + continue + content = event.get("content", {}) + if isinstance(content, dict): + ref = content.get("task_input_artifact_ref") + if isinstance(ref, str) and ref: + return ref + return None + + +def _create_actions_from_replay_report(report: Dict[str, Any]) -> List[Dict[str, Any]]: + actions: List[Dict[str, Any]] = [] + warnings = report.get("warnings", []) + errors = report.get("errors", []) + issues = [] + if isinstance(warnings, list): + issues.extend([("warning", w) for w in warnings if isinstance(w, str)]) + if isinstance(errors, list): + issues.extend([("error", e) for e in errors if isinstance(e, str)]) + + for severity, issue in issues: + if "missing" in issue and "artifact" in issue: + actions.append( + { + "type": "restore_artifact", + "severity": severity, + "issue": issue, + "operation": "recreate_or_restore_missing_artifact", + } + ) + elif "hash mismatch" in issue: + actions.append( + { + "type": "artifact_hash_mismatch", + "severity": severity, + "issue": issue, + "operation": "regenerate_artifact_and_refresh_hash_binding", + } + ) + elif "capture_level" in issue or "capture_policy" in issue: + actions.append( + { + "type": "capture_policy_violation", + "severity": severity, + "issue": issue, + "operation": "adjust_capture_policy_or_event_capture_fields", + } + ) + elif "redaction" in issue: + actions.append( + { + "type": "redaction_mismatch", + "severity": severity, + "issue": issue, + "operation": "re-run_redaction_pipeline_and_update_stats", + } + ) + else: + actions.append( + { + "type": "manual_review", + "severity": severity, + "issue": issue, + "operation": "manual_triage", + } + ) + + if not actions: + actions.append( + { + "type": "no_action", + "severity": "info", + "issue": "Replay report has no warnings/errors.", + "operation": "continue", + } + ) + return actions + + +def _resolve_ref(base_path: str, repo_root: str, ref: str) -> str: + if os.path.isabs(ref): + return ref + local = os.path.abspath(os.path.join(os.path.dirname(base_path), ref)) + if os.path.exists(local): + return local + return os.path.abspath(os.path.join(repo_root, ref)) + + +def _prepare_resume_task_input( + repo_root: str, + session_log_path: str, + report: Dict[str, Any], +) -> Tuple[Optional[Dict[str, Any]], List[str]]: + errors: List[str] = [] + events = _load_jsonl(session_log_path) + task_input_ref = _find_latest_spawn_task_input_ref(events) + if not task_input_ref: + return None, ["No spawn task_input_artifact_ref found in session log."] + + task_input_path = _resolve_ref(session_log_path, repo_root, task_input_ref) + if not os.path.exists(task_input_path): + return None, [f"Resume source task_input not found: {task_input_ref}"] + + payload = _load_json(task_input_path) + run_id = report.get("summary", {}).get("run_id", "run") + payload["child_id"] = f"resume-{run_id}" + payload["task_description"] = f"{payload.get('task_description', 'resume task')} [resume remediation]" + return payload, errors + + +def _prepare_session_state( + report: Dict[str, Any], resume_task_input_ref: Optional[str] +) -> Tuple[Optional[Dict[str, Any]], List[str]]: + errors: List[str] = [] + summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {} + run_id = summary.get("run_id") + if not isinstance(run_id, str) or not run_id: + errors.append("Resume lineage missing summary.run_id; cannot generate deterministic session_state.") + + timeline = report.get("timeline", []) if isinstance(report.get("timeline"), list) else [] + if not timeline or not isinstance(timeline[-1], dict): + errors.append("Resume lineage missing timeline events; cannot infer parent_id/phase/last_event_id.") + return None, errors + + last_event = timeline[-1] + parent_id = last_event.get("agent_id") + if not isinstance(parent_id, str) or not parent_id: + errors.append("Resume lineage missing last timeline agent_id; cannot generate deterministic parent_id.") + + step_ids = summary.get("step_ids", []) if isinstance(summary.get("step_ids"), list) else [] + step_id = step_ids[0] if step_ids and isinstance(step_ids[0], str) and step_ids[0] else None + if not step_id and isinstance(last_event.get("step_id"), str) and last_event.get("step_id"): + step_id = last_event.get("step_id") + if not isinstance(step_id, str) or not step_id: + errors.append("Resume lineage missing step_id; cannot generate deterministic session_state.") + + phase = _infer_phase(last_event.get("phase_id") if isinstance(last_event, dict) else None) + if not isinstance(phase, str): + errors.append("Resume lineage missing parseable phase_id (expected 16a/16b/16c token).") + + event_id = last_event.get("event_id") if isinstance(last_event, dict) else None + if errors: + return None, errors + + pending_child_id = f"resume-{run_id}" + # Session-state deep validation expects canonical spawn refs for resume lineage. + canonical_pending_spawn_ref = f".trinity/runtime/spawns/{pending_child_id}/task_input.json" + return { + "protocol_version": PROTO_VER, + "run_id": run_id, + "parent_id": parent_id, + "active_phase": phase, + "step_id": step_id, + "status": "resuming", + "pending_child_id": pending_child_id, + "pending_spawn_ref": canonical_pending_spawn_ref if isinstance(resume_task_input_ref, str) and resume_task_input_ref else None, + "spawn_log_ref": None, + "scratchpad_ref": None, + "last_event_id": event_id if isinstance(event_id, str) else None, + "retry_counters": {"planner": 0, "builder": 0, "verifier": 0, "milestone": 0}, + "updated_at": _utc_now_iso(), + }, [] + + +def _is_resume_source_issue(message: str) -> bool: + return ( + "No spawn task_input_artifact_ref found in session log." in message + or message.startswith("Resume source task_input not found:") + ) + + +def build_remediation_plan( + repo_root: str, + replay_report_path: str, + session_log_path: Optional[str] = None, + emit_session_state_path: Optional[str] = None, + emit_task_input_path: Optional[str] = None, + missing_resume_source_policy: str = "hard", +) -> Tuple[Dict[str, Any], List[str]]: + report = _load_json(replay_report_path) + actions = _create_actions_from_replay_report(report) + + errors: List[str] = [] + warnings: List[str] = [] + resume_task_input: Optional[Dict[str, Any]] = None + resume_task_input_ref: Optional[str] = None + + if missing_resume_source_policy not in {"soft", "hard"}: + errors.append( + f"invalid missing_resume_source_policy '{missing_resume_source_policy}' (expected 'soft' or 'hard')" + ) + + if session_log_path and emit_task_input_path: + resume_task_input, prep_errors = _prepare_resume_task_input(repo_root, session_log_path, report) + for issue in prep_errors: + if _is_resume_source_issue(issue) and missing_resume_source_policy == "soft": + warnings.append(issue) + else: + errors.append(issue) + if resume_task_input: + out_abs = os.path.abspath(emit_task_input_path) + os.makedirs(os.path.dirname(out_abs), exist_ok=True) + with open(out_abs, "w", encoding="utf-8") as f: + json.dump(resume_task_input, f, indent=2) + validation_errors = validate_runtime_file(repo_root, out_abs, "task_input") + if validation_errors: + errors.extend(validation_errors) + else: + resume_task_input_ref = out_abs + + session_state_payload: Optional[Dict[str, Any]] = None + if emit_session_state_path: + session_state_payload, lineage_errors = _prepare_session_state(report, resume_task_input_ref) + if lineage_errors: + errors.extend(lineage_errors) + elif session_state_payload: + out_abs = os.path.abspath(emit_session_state_path) + os.makedirs(os.path.dirname(out_abs), exist_ok=True) + with open(out_abs, "w", encoding="utf-8") as f: + json.dump(session_state_payload, f, indent=2) + validation_errors = validate_runtime_file(repo_root, out_abs, "session_state") + if validation_errors: + errors.extend(validation_errors) + + plan: Dict[str, Any] = { + "status": "needs_attention" if errors else ("ready_with_warnings" if warnings else "ready"), + "generated_at": _utc_now_iso(), + "source_replay_report": os.path.abspath(replay_report_path), + "source_session_log": os.path.abspath(session_log_path) if session_log_path else None, + "missing_resume_source_policy": missing_resume_source_policy, + "actions": actions, + "warnings": warnings, + "resume_outputs": { + "session_state_path": os.path.abspath(emit_session_state_path) if emit_session_state_path else None, + "task_input_path": os.path.abspath(emit_task_input_path) if emit_task_input_path else None, + }, + } + if session_state_payload is not None: + plan["resume_preview"] = session_state_payload + + return plan, errors diff --git a/tools/specdev_tools/trinity_replay.py b/tools/specdev_tools/trinity_replay.py new file mode 100644 index 00000000..931de3ab --- /dev/null +++ b/tools/specdev_tools/trinity_replay.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any, Dict, List, Optional, Tuple + +from .trinity_runtime_validate import validate_runtime_file + + +def _normalize_hash(value: Any) -> Optional[str]: + if not isinstance(value, str): + return None + raw = value.strip() + if raw.startswith("sha256:"): + raw = raw[len("sha256:") :] + if len(raw) != 64: + return None + try: + int(raw, 16) + except ValueError: + return None + return raw.lower() + + +def _sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def _sha256_canonical_json(path: str) -> Optional[str]: + try: + payload = json.load(open(path, "r", encoding="utf-8")) + except Exception: + return None + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _resolve_ref(session_log_path: str, repo_root: str, ref: str) -> str: + if os.path.isabs(ref): + return ref + local = os.path.abspath(os.path.join(os.path.dirname(session_log_path), ref)) + if os.path.exists(local): + return local + return os.path.abspath(os.path.join(repo_root, ref)) + + +def _load_session_events(session_log_path: str) -> Tuple[List[Dict[str, Any]], List[str]]: + events: List[Dict[str, Any]] = [] + errors: List[str] = [] + with open(session_log_path, "r", encoding="utf-8") as f: + for idx, raw_line in enumerate(f, start=1): + stripped = raw_line.strip() + if not stripped: + continue + try: + events.append(json.loads(stripped)) + except json.JSONDecodeError as e: + errors.append(f"{session_log_path}:{idx}: invalid json line ({e})") + return events, errors + + +def replay_session(repo_root: str, session_log_path: str, strict: bool = True) -> Dict[str, Any]: + """ + Standalone replay analyzer for a validated Trinity session log. + Verifies artifact/hash lineage and reconstructs timeline summary. + """ + report: Dict[str, Any] = { + "status": "ok", + "strict": strict, + "session_log": os.path.abspath(session_log_path), + "errors": [], + "warnings": [], + "summary": {}, + "timeline": [], + "artifact_verification": { + "checked": 0, + "matched": 0, + "missing": 0, + "mismatch": 0, + }, + } + + runtime_errors = validate_runtime_file(repo_root, session_log_path, "session_event") + if runtime_errors: + report["status"] = "failed" + report["errors"].extend(runtime_errors) + return report + + events, parse_errors = _load_session_events(session_log_path) + if parse_errors: + report["status"] = "failed" + report["errors"].extend(parse_errors) + return report + + event_type_counts: Dict[str, int] = {} + role_counts: Dict[str, int] = {} + phase_sequence: List[str] = [] + step_ids: List[str] = [] + agents: Dict[str, Dict[str, Any]] = {} + lineage_edges: set[Tuple[str, str]] = set() + + for event in events: + event_type = event.get("event_type", "UNKNOWN") + role = event.get("role", "UNKNOWN") + phase = event.get("phase_id") + step_id = event.get("step_id") + agent_id = event.get("agent_id") + parent_id = event.get("parent_id") + + event_type_counts[event_type] = event_type_counts.get(event_type, 0) + 1 + role_counts[role] = role_counts.get(role, 0) + 1 + if isinstance(phase, str) and phase and (not phase_sequence or phase_sequence[-1] != phase): + phase_sequence.append(phase) + if isinstance(step_id, str) and step_id and step_id not in step_ids: + step_ids.append(step_id) + + if isinstance(agent_id, str) and agent_id: + seq = event.get("event_sequence") + state = agents.setdefault( + agent_id, + { + "role": role, + "parent_id": parent_id, + "first_sequence": seq, + "last_sequence": seq, + "last_event_type": event_type, + }, + ) + state["last_sequence"] = seq + state["last_event_type"] = event_type + if isinstance(parent_id, str) and parent_id: + lineage_edges.add((parent_id, agent_id)) + + summary_text = None + content = event.get("content", {}) if isinstance(event.get("content"), dict) else {} + if isinstance(content.get("summary"), str): + summary_text = content.get("summary") + report["timeline"].append( + { + "event_sequence": event.get("event_sequence"), + "event_id": event.get("event_id"), + "event_type": event_type, + "agent_id": agent_id, + "parent_id": parent_id, + "phase_id": phase, + "step_id": step_id, + "summary": summary_text, + } + ) + + artifact_checks: List[Tuple[str, Any, str, str]] = [] + artifact_ref = event.get("artifact_ref") + artifact_sha = event.get("artifact_sha256") + if isinstance(artifact_ref, str) and artifact_ref and artifact_sha is not None: + artifact_checks.append(("artifact_ref", artifact_ref, artifact_sha, "file")) + + if isinstance(content, dict): + prompt_ref = content.get("prompt_artifact_ref") + prompt_sha = content.get("prompt_sha256") + if isinstance(prompt_ref, str) and prompt_ref and prompt_sha is not None: + artifact_checks.append(("prompt_artifact_ref", prompt_ref, prompt_sha, "file")) + + response_ref = content.get("response_artifact_ref") + response_sha = content.get("response_sha256") + if isinstance(response_ref, str) and response_ref and response_sha is not None: + artifact_checks.append(("response_artifact_ref", response_ref, response_sha, "file")) + + metadata = event.get("metadata", {}) if isinstance(event.get("metadata"), dict) else {} + policy_ref = metadata.get("capture_policy_ref") + policy_sha = metadata.get("capture_policy_sha256") + if isinstance(policy_ref, str) and policy_ref and policy_sha is not None: + artifact_checks.append(("capture_policy_ref", policy_ref, policy_sha, "json_canonical")) + + for label, ref, expected_sha_raw, mode in artifact_checks: + expected_sha = _normalize_hash(expected_sha_raw) + if expected_sha is None: + report["warnings"].append( + f"{session_log_path}: event {event.get('event_id')} has invalid hash format for {label}" + ) + continue + + report["artifact_verification"]["checked"] += 1 + resolved = _resolve_ref(session_log_path, repo_root, ref) + if not os.path.exists(resolved): + report["artifact_verification"]["missing"] += 1 + report["warnings"].append( + f"{session_log_path}: event {event.get('event_id')} missing {label} artifact: {ref}" + ) + continue + + if mode == "json_canonical": + actual_sha = _sha256_canonical_json(resolved) + if actual_sha is None: + report["artifact_verification"]["mismatch"] += 1 + report["warnings"].append( + f"{session_log_path}: event {event.get('event_id')} {label} is not valid JSON: {ref}" + ) + continue + else: + actual_sha = _sha256_file(resolved) + + if actual_sha != expected_sha: + report["artifact_verification"]["mismatch"] += 1 + report["warnings"].append( + f"{session_log_path}: event {event.get('event_id')} hash mismatch for {label}: {ref}" + ) + else: + report["artifact_verification"]["matched"] += 1 + + report["summary"] = { + "run_id": events[0].get("run_id") if events else None, + "total_events": len(events), + "event_type_counts": event_type_counts, + "role_counts": role_counts, + "phase_sequence": phase_sequence, + "step_ids": step_ids, + "agents": agents, + "lineage_edges": sorted([{"parent_id": p, "agent_id": a} for p, a in lineage_edges], key=lambda x: (x["parent_id"], x["agent_id"])), + } + + if strict and report["warnings"]: + report["status"] = "failed" + report["errors"].append("Strict replay failed due to warnings.") + elif report["warnings"]: + report["status"] = "warnings" + else: + report["status"] = "ok" + + return report diff --git a/tools/specdev_tools/trinity_runtime.py b/tools/specdev_tools/trinity_runtime.py new file mode 100644 index 00000000..7a90dc4a --- /dev/null +++ b/tools/specdev_tools/trinity_runtime.py @@ -0,0 +1,6459 @@ +from __future__ import annotations + +import copy +import fnmatch +import hashlib +import json +import os +import re +import shlex +import subprocess +import sys +import time +import uuid +import urllib.error +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +import yaml +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +from .registry import SchemaRegistry +from .trinity_runtime_validate import RUNTIME_SCHEMA_BY_TYPE, validate_runtime_file +from .validate import validate_file +from .seed_lint import lint_seeds +from .docs_lint import lint_docs +from .governance import check_commit_message + + +PROTO_VER = "trinity-runtime-v1" +SESSION_SCHEMA_VER = "trinity-session-log-v1" +TOOL_REQUEST_SCHEMA_URI = "https://specdev.local/schema/trinity/tool_call_request.schema.json" +TOOL_RESULT_SCHEMA_URI = "https://specdev.local/schema/trinity/tool_call_result.schema.json" +UTILITY_CALL_SCHEMA_URI = "https://specdev.local/schema/trinity/utility_call.schema.json" +UTILITY_RESULT_SCHEMA_URI = "https://specdev.local/schema/trinity/utility_result.schema.json" + +CORE_AUTHORITY_FILES = [ + "spec/04_fr_list.json", + "spec/05_interface_contracts.json", + "spec/06_invariants.json", + "spec/07_nfrs.json", + "spec/08_fixtures.json", + "spec/09_impl_plan.json", + "spec/10_governance.json", + "spec/11_redteam.json", + "spec/12_ci_gates.json", + "spec/13_extension_manifest.json", + "spec/13a_completeness_assessment.json", + "spec/14_roadmap.json", + "spec/15_scaffold.json", +] + +SPEC_FILE_BY_TYPE = { + "fr": "spec/04_fr_list.json", + "api": "spec/05_interface_contracts.json", + "inv": "spec/06_invariants.json", + "nfr": "spec/07_nfrs.json", + "fixture": "spec/08_fixtures.json", +} + +PROMPT_MAP = { + "16a": "prompts/prompt_16a_impl_planner.md", + "16b": "prompts/prompt_16b_impl_coder.md", + "16c": "prompts/prompt_16c_impl_reviewer.md", +} + +UTILITY_PROMPT_MAP = { + "Researcher": "prompts/trinity/70_researcher.md", + "ToolUser": "prompts/trinity/80_tool_usage.md", + "Summarizer": "prompts/trinity/90_summarizer.md", + "Auditor": "prompts/trinity/99_auditor.md", +} + +READONLY_GIT_SUBCOMMANDS = { + "branch", + "describe", + "diff", + "grep", + "log", + "ls-files", + "rev-parse", + "show", + "status", + "tag", +} + +WILDCARD_CHARS = set("*?[]") + +DEFAULT_CAPTURE_POLICY = { + "policy_id": "builtin-eval-default", + "version": "1", + "default_capture_level": "summary", + "always_full_on_event_types": ["ERROR"], + "sample_rate_by_event_type": { + "SPAWN": 0.0, + "MESSAGE": 0.2, + "TOOL_CALL": 0.0, + "TOOL_RESULT": 0.25, + "VALIDATION": 0.5, + "TERMINATE": 0.1, + "ERROR": 1.0, + }, + "max_full_events_per_run": 24, + "context_window_token_target": 80000, + "max_full_capture_context_fraction": 0.2, + "full_capture_token_budget_per_run": 20000, + "max_full_prompt_tokens_per_event": 12000, + "max_full_completion_tokens_per_event": 6000, + "oversize_fallback": "summary", + "full_capture_allowlist_roles": [ + "Orchestrator", + "Planner", + "Builder", + "Verifier", + "Worker", + "Researcher", + "Auditor", + "Summarizer", + "ToolUser", + ], + "require_redaction_before_full": True, + "sampling_salt": "builtin", + "operating_profile": { + "profile": "eval_default", + "tier": "balanced", + "budget_tier": "medium", + }, + "budgets": { + "context_window_token_target": 80000, + "full_capture_token_budget_per_run": 20000, + "max_full_prompt_tokens_per_event": 12000, + "max_full_completion_tokens_per_event": 6000, + }, + "retention": { + "session_log_days": 30, + "capture_artifact_days": 14, + "eval_export_days": 90, + }, +} + +SECRET_REDACTION_RULES: Tuple[Tuple[str, re.Pattern[str], str, float], ...] = ( + ( + "openai_key", + re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), + "[REDACTED_OPENAI_KEY]", + 0.98, + ), + ( + "aws_access_key", + re.compile(r"\bAKIA[0-9A-Z]{16}\b"), + "[REDACTED_AWS_ACCESS_KEY]", + 0.96, + ), + ( + "bearer_token", + re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{12,}\b", re.IGNORECASE), + "Bearer [REDACTED_TOKEN]", + 0.94, + ), + ( + "jwt", + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), + "[REDACTED_JWT]", + 0.92, + ), + ( + "private_key_block", + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----"), + "[REDACTED_PRIVATE_KEY_BLOCK]", + 0.99, + ), + ( + "api_secret_assignment", + re.compile( + r"(?i)\b(api[_-]?key|token|secret|password)\b\s*[:=]\s*([\"']?)[^\s\"';]{8,}\2" + ), + r"\1=[REDACTED_SECRET]", + 0.9, + ), +) + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sha256_file(path: str) -> str: + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + +def _canonical_sha(payload: dict) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return _sha256_text(canonical) + + +def _compute_event_sha(event: dict) -> str: + hash_payload = dict(event) + hash_payload["event_sha256"] = None + canonical = json.dumps(hash_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return _sha256_text(canonical) + + +def _read_json(path: str) -> dict: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_json_atomic(path: str, payload: dict) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + temp_path = f"{path}.tmp-{uuid.uuid4().hex}" + with open(temp_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False) + f.write("\n") + os.replace(temp_path, path) + + +def _append_jsonl(path: str, payload: dict) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "a", encoding="utf-8") as f: + f.write(json.dumps(payload, ensure_ascii=False)) + f.write("\n") + + +def _rel(repo_root: str, path: str) -> str: + return os.path.relpath(path, repo_root).replace("\\", "/") + + +def _path_within_root(root: str, candidate: str) -> bool: + root_real = os.path.realpath(root) + cand_real = os.path.realpath(candidate) + return cand_real == root_real or cand_real.startswith(root_real + os.sep) + + +def _normalize_rel_path(path_value: str) -> str: + raw = path_value.replace("\\", "/").strip() + if raw.startswith("./"): + raw = raw[2:] + normalized = os.path.normpath(raw or ".").replace("\\", "/") + if normalized in {"", "."}: + return "." + return normalized + + +def _is_escape_rel_path(path_value: str) -> bool: + normalized = _normalize_rel_path(path_value) + return normalized == ".." or normalized.startswith("../") + + +def _prompt_path_for(phase: str, role: Optional[str] = None) -> str: + if phase == "utility" and isinstance(role, str): + role_prompt = UTILITY_PROMPT_MAP.get(role) + if isinstance(role_prompt, str): + return role_prompt + return PROMPT_MAP.get(phase, "prompts/prompt_16_impl_context.md") + + +def _load_yaml(path: str) -> dict: + with open(path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + if not isinstance(data, dict): + raise ValueError(f"{path}: trinity config must be a YAML object") + return data + + +def _run_git(repo_root: str, args: List[str], check: bool = True) -> subprocess.CompletedProcess: + result = subprocess.run( + ["git"] + args, + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if check and result.returncode != 0: + msg = (result.stderr or result.stdout or "").strip() + raise RuntimeError(f"git {' '.join(args)} failed: {msg}") + return result + + +def _git_head(repo_root: str) -> Optional[str]: + result = _run_git(repo_root, ["rev-parse", "HEAD"], check=False) + if result.returncode != 0: + return None + out = (result.stdout or "").strip() + if re.fullmatch(r"[0-9a-f]{40}", out): + return out + return None + + +def _is_dirty_worktree(repo_root: str) -> bool: + result = _run_git(repo_root, ["status", "--porcelain"], check=False) + if result.returncode != 0: + return True + return bool((result.stdout or "").strip()) + + +def _normalize_test_command(entry: Any) -> Optional[str]: + if isinstance(entry, str): + cmd = entry.strip() + return cmd or None + if isinstance(entry, dict): + cmd = entry.get("command") + if isinstance(cmd, str): + cmd = cmd.strip() + return cmd or None + return None + + +def _looks_like_pattern(path_value: str) -> bool: + return any(ch in WILDCARD_CHARS for ch in path_value) + + +def _redact_sensitive_text(text: str) -> Tuple[str, dict]: + if not isinstance(text, str): + return "", { + "total_replacements": 0, + "by_class": {}, + "classes_detected": [], + "detectors_used": ["secret_scanner_v2"], + "min_confidence": 0.0, + "max_confidence": 0.0, + } + + redacted = text + by_class: Dict[str, int] = {} + confidence_hits: List[float] = [] + for cls, pattern, replacement, confidence in SECRET_REDACTION_RULES: + redacted, count = pattern.subn(replacement, redacted) + if count > 0: + by_class[cls] = by_class.get(cls, 0) + int(count) + confidence_hits.extend([confidence] * int(count)) + + total = sum(by_class.values()) + stats = { + "total_replacements": total, + "by_class": by_class, + "classes_detected": sorted(by_class.keys()), + "detectors_used": ["secret_scanner_v2"], + "min_confidence": min(confidence_hits) if confidence_hits else 0.0, + "max_confidence": max(confidence_hits) if confidence_hits else 0.0, + } + return redacted, stats + + +def _is_secret_dump_command(command: str) -> bool: + lowered = command.lower() + try: + tokens = shlex.split(command, posix=True) + except Exception: + # Preserve conservative behavior when parsing fails. + fallback_patterns = ( + "printenv", + "env |", + "cat ~/.ssh", + "cat $home/.ssh", + "cat ~/.aws", + "aws configure get", + "cat .env", + "grep -r secret", + "grep -r token", + ) + return any(p in lowered for p in fallback_patterns) + + if not tokens: + return False + + binary = tokens[0].lower() + if binary == "printenv": + return True + if binary == "env": + # Allow `env KEY=value ` but block output-dumping forms like + # `env`, `env -0`, or shell-piped forms that primarily print environment. + idx = 1 + while idx < len(tokens): + tok = tokens[idx] + if tok.startswith("-"): + idx += 1 + continue + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", tok): + idx += 1 + continue + break + if idx >= len(tokens): + return True + if tokens[idx] in {"|", ">", ">>"}: + return True + return False + if binary == "aws" and len(tokens) >= 3 and tokens[1].lower() == "configure" and tokens[2].lower() == "get": + return True + if binary == "cat": + sensitive_reads = {".env", "~/.ssh", "$home/.ssh", "~/.aws"} + for tok in tokens[1:]: + norm = tok.strip().lower() + if norm in sensitive_reads or norm.startswith("~/.ssh/") or norm.startswith("$home/.ssh/"): + return True + if binary == "grep" and any(tok.lower() == "-r" for tok in tokens[1:]): + targets = {tok.lower() for tok in tokens[1:] if isinstance(tok, str)} + if "secret" in targets or "token" in targets: + return True + return False + + +def _loop_evidence_present(payload: Any) -> bool: + if isinstance(payload, str): + return len(payload.strip()) >= 8 + if isinstance(payload, list): + return any(_loop_evidence_present(item) for item in payload) + if isinstance(payload, dict): + preferred_keys = ("evidence", "summary", "notes", "rationale", "decision", "artifact_ref") + for key in preferred_keys: + if _loop_evidence_present(payload.get(key)): + return True + for value in payload.values(): + if _loop_evidence_present(value): + return True + return False + + +def _extract_command_excerpt(stdout: str, stderr: str, exit_code: int) -> Tuple[str, bool]: + text = (stdout or "") + ("\n" + stderr if stderr else "") + lines = [ln for ln in text.splitlines() if ln.strip()] + marker_re = re.compile(r"(PASSED|passed|OK|SUCCESS|✓|0 (errors|failures?|failed)|\d+ passed)") + for line in lines: + if marker_re.search(line): + return line[:400], True + if lines: + return lines[-1][:400], False + return f"FAILED exit_code={exit_code}", False + + +def _stable_unit_interval(key: str) -> float: + digest = hashlib.sha256(key.encode("utf-8")).hexdigest() + return int(digest[:16], 16) / float(0xFFFFFFFFFFFFFFFF) + + +def _extract_json_object(text: str) -> Optional[dict]: + raw = (text or "").strip() + if not raw: + return None + try: + parsed = json.loads(raw) + return parsed if isinstance(parsed, dict) else None + except Exception: + pass + + fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", raw, flags=re.DOTALL) + if fenced: + try: + parsed = json.loads(fenced.group(1)) + return parsed if isinstance(parsed, dict) else None + except Exception: + pass + + start = raw.find("{") + if start == -1: + return None + depth = 0 + end = -1 + for i, ch in enumerate(raw[start:], start=start): + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + end = i + break + if end == -1: + return None + try: + parsed = json.loads(raw[start : end + 1]) + return parsed if isinstance(parsed, dict) else None + except Exception: + return None + + +@dataclass +class TrinityConfig: + llm_api_base: str + llm_model: str + llm_timeout: int + llm_api_key_env: str + llm_temperature: float + llm_top_p: float + llm_max_tokens: int + execution_mode: str + max_child_turns: int + max_loops: int + retry_cap_planner: int + retry_cap_builder: int + retry_cap_verifier: int + retry_cap_milestone: int + allow_dirty: bool + checkpoint_commits: bool + conformance_mode: bool + child_timeout_seconds: int + child_timeout_by_phase: Dict[str, int] + allow_bootstrap_authority_fallback: bool + allow_anchor_conflicts: bool + + @staticmethod + def load(repo_root: str) -> "TrinityConfig": + config_path = os.path.join(repo_root, ".trinity", "trinity.yaml") + if not os.path.exists(config_path): + raise RuntimeError(f"Missing Trinity config: {config_path}") + raw = _load_yaml(config_path) + llm = raw.get("llm", {}) if isinstance(raw.get("llm"), dict) else {} + limits = raw.get("limits", {}) if isinstance(raw.get("limits"), dict) else {} + runtime = raw.get("runtime", {}) if isinstance(raw.get("runtime"), dict) else {} + retry_caps = runtime.get("retry_caps", {}) if isinstance(runtime.get("retry_caps"), dict) else {} + timeout_by_phase_raw = runtime.get("child_timeout_by_phase", {}) + timeout_by_phase: Dict[str, int] = {} + if isinstance(timeout_by_phase_raw, dict): + for phase_name in ("16a", "16b", "16c", "utility"): + value = timeout_by_phase_raw.get(phase_name) + if value is None: + continue + ivalue = int(value) + if ivalue < 0: + raise RuntimeError(f"runtime.child_timeout_by_phase.{phase_name} must be >= 0") + timeout_by_phase[phase_name] = ivalue + + def _cap(name: str, fallback: int) -> int: + value = int(retry_caps.get(name, fallback)) + if value < 1: + raise RuntimeError(f"runtime.retry_caps.{name} must be >= 1") + return value + + milestone_default = int(limits.get("max_loops", 10)) + if milestone_default < 1: + raise RuntimeError("limits.max_loops must be >= 1") + return TrinityConfig( + llm_api_base=str(llm.get("api_base", "http://localhost:1234/v1")), + llm_model=str(llm.get("model", "input-model")), + llm_timeout=int(llm.get("timeout", 300)), + llm_api_key_env=str(llm.get("api_key_env", "OPENAI_API_KEY")), + llm_temperature=float(llm.get("temperature", 0.2)), + llm_top_p=float(llm.get("top_p", 0.9)), + llm_max_tokens=int(llm.get("max_tokens", 4096)), + execution_mode=str(runtime.get("execution_mode", "llm")).strip().lower() or "llm", + max_child_turns=int(runtime.get("max_child_turns", 12)), + max_loops=milestone_default, + retry_cap_planner=_cap("planner", 10), + retry_cap_builder=_cap("builder", 10), + retry_cap_verifier=_cap("verifier", 10), + retry_cap_milestone=_cap("milestone", milestone_default), + allow_dirty=bool(runtime.get("allow_dirty", False)), + checkpoint_commits=bool(runtime.get("checkpoint_commits", True)), + conformance_mode=bool(runtime.get("conformance_mode", True)), + child_timeout_seconds=max(0, int(runtime.get("child_timeout_seconds", 21600))), + child_timeout_by_phase=timeout_by_phase, + allow_bootstrap_authority_fallback=bool(runtime.get("allow_bootstrap_authority_fallback", False)), + allow_anchor_conflicts=bool(runtime.get("allow_anchor_conflicts", False)), + ) + + +class SessionLogger: + def __init__( + self, + repo_root: str, + run_id: str, + root_task_id: str, + step_id: str, + model: str, + *, + decoding_temperature: float = 0.2, + decoding_top_p: float = 0.9, + decoding_max_tokens: int = 4096, + log_path: Optional[str] = None, + ) -> None: + self.repo_root = repo_root + self.run_id = run_id + self.step_id = step_id + self.model = model + self.decoding_temperature = decoding_temperature + self.decoding_top_p = decoding_top_p + self.decoding_max_tokens = decoding_max_tokens + if isinstance(log_path, str) and log_path.strip(): + self.path = log_path if os.path.isabs(log_path) else os.path.join(repo_root, log_path) + else: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + self.path = os.path.join(repo_root, ".trinity", "sessions", f"{ts}_{root_task_id}.jsonl") + os.makedirs(os.path.dirname(self.path), exist_ok=True) + if not os.path.exists(self.path): + with open(self.path, "w", encoding="utf-8"): + pass + self.sequence = 0 + self.prev_hash: Optional[str] = None + self.schema_registry = SchemaRegistry(repo_root) + self.request_schema_sha = _canonical_sha(self.schema_registry.load(TOOL_REQUEST_SCHEMA_URI)) + self.result_schema_sha = _canonical_sha(self.schema_registry.load(TOOL_RESULT_SCHEMA_URI)) + self._session_event_validator = self._build_session_event_validator() + self.catalog_ref = ".trinity/runtime/tools/catalog.json" + self.catalog_sha = self._ensure_tool_catalog() + self.toolkit_version = self._detect_toolkit_version() + self.git_head = _git_head(repo_root) or "unknown" + ( + self.capture_policy_ref, + self.capture_policy_sha256, + self.capture_policy, + self.capture_policy_profile, + self.capture_policy_fallback_warnings, + ) = self._load_capture_policy() + self._sampled_full_counts: Dict[str, int] = {} + self._full_capture_tokens: Dict[str, int] = {} + self._scan_offset = 0 + self.sync_from_disk() + + def _build_session_event_validator(self) -> Draft202012Validator: + store = {uri: Resource.from_contents(schema) for uri, schema in self.schema_registry.store.items()} + registry = Registry().with_resources(store.items()) + schema = self.schema_registry.load(RUNTIME_SCHEMA_BY_TYPE["session_event"]) + return Draft202012Validator( + schema, + registry=registry, + format_checker=Draft202012Validator.FORMAT_CHECKER, + ) + + def sync_from_disk(self) -> None: + if not os.path.exists(self.path): + self.sequence = 0 + self.prev_hash = None + self._scan_offset = 0 + return + file_size = os.path.getsize(self.path) + if self._scan_offset < 0 or self._scan_offset > file_size: + self.sequence = 0 + self.prev_hash = None + self._scan_offset = 0 + + last_sequence = self.sequence + last_hash: Optional[str] = self.prev_hash + with open(self.path, "r", encoding="utf-8") as f: + if self._scan_offset: + f.seek(self._scan_offset) + for raw in f: + line = raw.strip() + if not line: + continue + try: + event = json.loads(line) + except Exception: + continue + seq = event.get("event_sequence") + sha = event.get("event_sha256") + if isinstance(seq, int) and seq > 0 and seq >= last_sequence: + last_sequence = seq + if isinstance(sha, str) and re.fullmatch(r"[a-f0-9]{64}", sha): + last_hash = sha + self._scan_offset = f.tell() + self.sequence = last_sequence + self.prev_hash = last_hash + + def _detect_toolkit_version(self) -> str: + pyproject_path = os.path.join(self.repo_root, "tools", "pyproject.toml") + if not os.path.exists(pyproject_path): + return "unknown" + with open(pyproject_path, "r", encoding="utf-8") as f: + content = f.read() + m = re.search(r'(?m)^version\s*=\s*"([^"]+)"', content) + return m.group(1) if m else "unknown" + + def _normalize_capture_policy(self, payload: dict) -> Tuple[dict, dict, List[str]]: + effective = copy.deepcopy(DEFAULT_CAPTURE_POLICY) + warnings: List[str] = [] + if not isinstance(payload, dict): + warnings.append("capture policy payload was not an object; using builtin defaults") + return effective, dict(effective.get("operating_profile", {})), warnings + + for key in ( + "policy_id", + "version", + "default_capture_level", + "always_full_on_event_types", + "sample_rate_by_event_type", + "max_full_events_per_run", + "context_window_token_target", + "max_full_capture_context_fraction", + "full_capture_token_budget_per_run", + "max_full_prompt_tokens_per_event", + "max_full_completion_tokens_per_event", + "oversize_fallback", + "full_capture_allowlist_roles", + "require_redaction_before_full", + "sampling_salt", + ): + if key in payload: + effective[key] = payload[key] + + profile_in = payload.get("operating_profile") + profile = copy.deepcopy(effective.get("operating_profile", {})) + if isinstance(profile_in, dict): + for pkey in ("profile", "tier", "budget_tier"): + if isinstance(profile_in.get(pkey), str) and profile_in[pkey]: + profile[pkey] = profile_in[pkey] + else: + warnings.append(f"capture policy operating_profile.{pkey} missing; default applied") + else: + warnings.append("capture policy operating_profile missing; default profile applied") + + retention_in = payload.get("retention") + retention = copy.deepcopy(effective.get("retention", {})) + if isinstance(retention_in, dict): + for rkey in ("session_log_days", "capture_artifact_days", "eval_export_days"): + rval = retention_in.get(rkey) + if isinstance(rval, int) and rval >= 1: + retention[rkey] = rval + else: + warnings.append(f"capture policy retention.{rkey} missing or invalid; default applied") + else: + warnings.append("capture policy retention missing; default retention applied") + + budgets_in = payload.get("budgets") + budgets = copy.deepcopy(effective.get("budgets", {})) + budget_map = { + "context_window_token_target": 1024, + "full_capture_token_budget_per_run": 0, + "max_full_prompt_tokens_per_event": 0, + "max_full_completion_tokens_per_event": 0, + } + for bkey, minimum in budget_map.items(): + chosen = None + if isinstance(budgets_in, dict): + chosen = budgets_in.get(bkey) + if not (isinstance(chosen, int) and chosen >= minimum): + chosen = payload.get(bkey) + if isinstance(chosen, int) and chosen >= minimum: + budgets[bkey] = chosen + else: + warnings.append(f"capture policy budget '{bkey}' missing or invalid; default applied") + + effective["operating_profile"] = profile + effective["retention"] = retention + effective["budgets"] = budgets + effective["context_window_token_target"] = budgets["context_window_token_target"] + effective["full_capture_token_budget_per_run"] = budgets["full_capture_token_budget_per_run"] + effective["max_full_prompt_tokens_per_event"] = budgets["max_full_prompt_tokens_per_event"] + effective["max_full_completion_tokens_per_event"] = budgets["max_full_completion_tokens_per_event"] + return effective, profile, warnings + + def _load_capture_policy(self) -> Tuple[Optional[str], Optional[str], Optional[dict], dict, List[str]]: + rel = ".trinity/logging/log_capture_policy.json" + path = os.path.join(self.repo_root, rel) + if not os.path.exists(path): + default_policy = copy.deepcopy(DEFAULT_CAPTURE_POLICY) + return ( + None, + None, + default_policy, + copy.deepcopy(default_policy.get("operating_profile", {})), + [], + ) + errs = validate_runtime_file(self.repo_root, path, "log_capture_policy") + if errs: + raise RuntimeError("Invalid Trinity log capture policy: " + "; ".join(errs)) + payload = _read_json(path) + effective, profile, warnings = self._normalize_capture_policy(payload) + return rel, _canonical_sha(payload), effective, profile, warnings + + def _capture_decision( + self, + *, + event_type: str, + role: str, + event_id: str, + event_sequence: int, + prompt_tokens: int, + completion_tokens: int, + ) -> Tuple[str, str]: + if not isinstance(self.capture_policy, dict): + return "summary", "policy:default" + + policy = self.capture_policy + policy_id = str(policy.get("policy_id", "policy")) + policy_run_key = f"{policy_id}|{self.run_id}" + default_capture_level = str(policy.get("default_capture_level", "summary")) + always_full_events = set(policy.get("always_full_on_event_types", [])) + allowlist_roles = policy.get("full_capture_allowlist_roles", []) + role_allowed_for_full = (not allowlist_roles) or (role in allowlist_roles) + sampling_salt = str(policy.get("sampling_salt", "default")) + + expected_capture_level = default_capture_level + expected_reason_prefix = "policy:default" + + is_always_full = role_allowed_for_full and event_type in always_full_events + if is_always_full: + expected_capture_level = "full" + expected_reason_prefix = "policy:always_full" + else: + sample_rates = policy.get("sample_rate_by_event_type", {}) + sample_rate = sample_rates.get(event_type, 0.0) if isinstance(sample_rates, dict) else 0.0 + sample_rate = sample_rate if isinstance(sample_rate, (int, float)) else 0.0 + sampled_for_full = False + if role_allowed_for_full and sample_rate > 0: + sample_key = f"{policy_id}|{sampling_salt}|{self.run_id}|{event_id}|{event_sequence}" + sampled_for_full = _stable_unit_interval(sample_key) < float(sample_rate) + if sampled_for_full: + max_full_events = policy.get("max_full_events_per_run", 0) + max_full_events = max_full_events if isinstance(max_full_events, int) else 0 + used = self._sampled_full_counts.get(policy_run_key, 0) + if used < max_full_events: + expected_capture_level = "full" + expected_reason_prefix = "policy:sampled" + self._sampled_full_counts[policy_run_key] = used + 1 + else: + expected_capture_level = str(policy.get("oversize_fallback", "summary")) + expected_reason_prefix = "policy:capped" + + if expected_capture_level == "full": + max_prompt_tokens = policy.get("max_full_prompt_tokens_per_event") + max_prompt_tokens = max_prompt_tokens if isinstance(max_prompt_tokens, int) and max_prompt_tokens >= 0 else None + max_completion_tokens = policy.get("max_full_completion_tokens_per_event") + max_completion_tokens = ( + max_completion_tokens if isinstance(max_completion_tokens, int) and max_completion_tokens >= 0 else None + ) + explicit_budget = policy.get("full_capture_token_budget_per_run") + explicit_budget = explicit_budget if isinstance(explicit_budget, int) and explicit_budget >= 0 else None + window_target = policy.get("context_window_token_target") + window_target = window_target if isinstance(window_target, int) and window_target >= 0 else None + window_fraction_raw = policy.get("max_full_capture_context_fraction") + derived_budget: Optional[int] = None + if ( + isinstance(window_fraction_raw, (int, float)) + and window_fraction_raw > 0 + and window_fraction_raw <= 1 + and isinstance(window_target, int) + ): + derived_budget = int(window_target * float(window_fraction_raw)) + + effective_budget: Optional[int] = None + for candidate in (explicit_budget, derived_budget): + if candidate is None: + continue + effective_budget = candidate if effective_budget is None else min(effective_budget, candidate) + + if isinstance(max_prompt_tokens, int) and prompt_tokens > max_prompt_tokens: + expected_capture_level = str(policy.get("oversize_fallback", "summary")) + expected_reason_prefix = "policy:token_guard_prompt" + elif isinstance(max_completion_tokens, int) and completion_tokens > max_completion_tokens: + expected_capture_level = str(policy.get("oversize_fallback", "summary")) + expected_reason_prefix = "policy:token_guard_completion" + else: + used_tokens = self._full_capture_tokens.get(policy_run_key, 0) + total_tokens = prompt_tokens + completion_tokens + if isinstance(effective_budget, int) and (used_tokens + total_tokens > effective_budget): + expected_capture_level = str(policy.get("oversize_fallback", "summary")) + expected_reason_prefix = "policy:token_budget" + else: + self._full_capture_tokens[policy_run_key] = used_tokens + total_tokens + + return expected_capture_level, expected_reason_prefix + + def _write_capture_artifact(self, *, event_id: str, kind: str, content: str) -> Tuple[str, str]: + capture_dir = os.path.join(self.repo_root, ".trinity", "captures") + os.makedirs(capture_dir, exist_ok=True) + rel = f".trinity/captures/{kind}_{event_id}.txt" + abs_path = os.path.join(self.repo_root, rel) + with open(abs_path, "w", encoding="utf-8") as f: + f.write(content) + if not content.endswith("\n"): + f.write("\n") + return rel, _sha256_file(abs_path) + + def _ensure_tool_catalog(self) -> str: + catalog_path = os.path.join(self.repo_root, self.catalog_ref) + catalog = { + "schema_version": "trinity-tool-catalog-v1", + "tools": [ + {"tool_name": "read_file", "required_args": ["path"]}, + {"tool_name": "write_file", "required_args": ["path", "content"]}, + {"tool_name": "edit_file", "required_args": ["path", "edits"]}, + {"tool_name": "apply_patch", "required_args": ["patch"]}, + {"tool_name": "move_file", "required_args": ["src_path", "dst_path"]}, + {"tool_name": "remove_file", "required_args": ["path"]}, + {"tool_name": "list_dir", "required_args": ["path"]}, + {"tool_name": "glob_match", "required_args": ["path", "patterns"]}, + {"tool_name": "search_text", "required_args": ["pattern", "paths"]}, + {"tool_name": "git_head", "required_args": []}, + {"tool_name": "git_show", "required_args": ["rev"]}, + {"tool_name": "git_diff", "required_args": []}, + {"tool_name": "exec_cmd", "required_args": ["command", "mode"]}, + {"tool_name": "validate_json", "required_args": ["path"]}, + {"tool_name": "checkpoint_branch", "required_args": ["branch_name"]}, + {"tool_name": "checkpoint_commit", "required_args": ["message"]}, + ], + } + _write_json_atomic(catalog_path, catalog) + return _canonical_sha(catalog) + + def _validate_event_before_persist(self, event: dict) -> None: + errors = sorted(self._session_event_validator.iter_errors(event), key=lambda e: list(e.path)) + if errors: + rendered = [] + for err in errors[:5]: + path = "/".join(map(str, err.path)) + rendered.append(f"{path}: {err.message}" if path else str(err.message)) + raise RuntimeError("Session event schema validation failed before persistence: " + "; ".join(rendered)) + + def append( + self, + event_type: str, + *, + role: str, + phase_id: str, + loop_id: str, + agent_id: str, + parent_id: Optional[str], + summary: str, + prompt_template_id: str, + step_id: Optional[str], + content_extra: Optional[dict] = None, + metadata_extra: Optional[dict] = None, + tool_call_id: Optional[str] = None, + result_id: Optional[str] = None, + artifact_ref: Optional[str] = None, + artifact_sha256: Optional[str] = None, + diff_ref: Optional[str] = None, + prompt_material_override: Optional[str] = None, + response_material_override: Optional[str] = None, + ) -> dict: + self.sync_from_disk() + next_sequence = self.sequence + 1 + event_id = str(uuid.uuid4()) + prompt_template_path = None + if isinstance(prompt_template_id, str) and prompt_template_id: + candidate = prompt_template_id + if not os.path.isabs(candidate): + candidate = os.path.join(self.repo_root, candidate) + if os.path.exists(candidate): + prompt_template_path = candidate + if prompt_template_path is None: + prompt_template_path = os.path.join(self.repo_root, _prompt_path_for(phase_id, role)) + prompt_sha = "" + prompt_text = prompt_template_id + if os.path.exists(prompt_template_path): + with open(prompt_template_path, "r", encoding="utf-8") as f: + prompt_text = f.read() + prompt_sha = _sha256_text(prompt_text) + else: + prompt_sha = _sha256_text(prompt_template_id) + prompt_capture_material = prompt_material_override if isinstance(prompt_material_override, str) else prompt_text + + content = { + "summary": summary, + "capture_level": "summary", + "capture_decision_reason": "policy:default", + "prompt_artifact_ref": None, + "prompt_sha256": None, + "response_artifact_ref": None, + "response_sha256": None, + } + if content_extra: + content.update(content_extra) + + completion_material = summary + if content_extra: + completion_material = completion_material + "\n" + json.dumps(content_extra, sort_keys=True, ensure_ascii=False) + if isinstance(response_material_override, str): + completion_material = response_material_override + prompt_tokens = max(1, len(prompt_capture_material) // 4) + completion_tokens = max(1, len(completion_material) // 4) + capture_level, capture_reason = self._capture_decision( + event_type=event_type, + role=role, + event_id=event_id, + event_sequence=next_sequence, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + + redaction_applied = False + redaction_stats = { + "total_replacements": 0, + "by_class": {}, + "classes_detected": [], + "detectors_used": ["secret_scanner_v2"], + "min_confidence": 0.0, + "max_confidence": 0.0, + } + prompt_capture_ref = None + prompt_capture_sha = None + response_capture_ref = None + response_capture_sha = None + if capture_level == "full": + prompt_payload = prompt_capture_material + response_payload = completion_material + if isinstance(self.capture_policy, dict) and bool(self.capture_policy.get("require_redaction_before_full")): + redaction_applied = True + prompt_payload, prompt_stats = _redact_sensitive_text(prompt_payload) + response_payload, response_stats = _redact_sensitive_text(response_payload) + merged_by_class: Dict[str, int] = {} + for stats_obj in (prompt_stats, response_stats): + by_class = stats_obj.get("by_class", {}) if isinstance(stats_obj.get("by_class"), dict) else {} + for cls, count in by_class.items(): + if isinstance(cls, str) and isinstance(count, int) and count > 0: + merged_by_class[cls] = merged_by_class.get(cls, 0) + count + total_replacements = sum(merged_by_class.values()) + confidence_values: List[float] = [] + for stats_obj in (prompt_stats, response_stats): + min_conf = stats_obj.get("min_confidence") + max_conf = stats_obj.get("max_confidence") + if isinstance(min_conf, (int, float)) and min_conf > 0: + confidence_values.append(float(min_conf)) + if isinstance(max_conf, (int, float)) and max_conf > 0: + confidence_values.append(float(max_conf)) + redaction_stats = { + "total_replacements": total_replacements, + "by_class": merged_by_class, + "classes_detected": sorted(merged_by_class.keys()), + "detectors_used": ["secret_scanner_v2"], + "min_confidence": min(confidence_values) if confidence_values else 0.0, + "max_confidence": max(confidence_values) if confidence_values else 0.0, + } + _, residual_prompt_stats = _redact_sensitive_text(prompt_payload) + _, residual_response_stats = _redact_sensitive_text(response_payload) + residual_hits = int(residual_prompt_stats.get("total_replacements", 0)) + int( + residual_response_stats.get("total_replacements", 0) + ) + if residual_hits > 0: + raise RuntimeError( + "capture policy requires redaction before full capture, but residual sensitive patterns remained" + ) + + prompt_capture_ref, prompt_capture_sha = self._write_capture_artifact( + event_id=event_id, + kind="prompt", + content=prompt_payload, + ) + response_capture_ref, response_capture_sha = self._write_capture_artifact( + event_id=event_id, + kind="response", + content=response_payload, + ) + content["capture_level"] = "full" + content["capture_decision_reason"] = capture_reason + content["prompt_artifact_ref"] = prompt_capture_ref + content["prompt_sha256"] = prompt_capture_sha + content["response_artifact_ref"] = response_capture_ref + content["response_sha256"] = response_capture_sha + else: + content["capture_level"] = capture_level + content["capture_decision_reason"] = capture_reason + + metadata = { + "toolkit_version": self.toolkit_version, + "schema_version": "v1", + "git_head": self.git_head, + "prompt_template_id": prompt_template_id, + "prompt_template_sha256": prompt_sha, + "redaction_profile": "eval", + "redaction_applied": redaction_applied, + "capture_policy_ref": self.capture_policy_ref, + "capture_policy_sha256": self.capture_policy_sha256, + "capture_policy_profile": self.capture_policy_profile if isinstance(self.capture_policy_profile, dict) else None, + "capture_policy_fallback_applied": bool(self.capture_policy_fallback_warnings), + "capture_policy_fallback_reasons": list(self.capture_policy_fallback_warnings), + "redaction_stats": redaction_stats, + "decoding": { + "temperature": self.decoding_temperature, + "top_p": self.decoding_top_p, + "max_tokens": self.decoding_max_tokens, + }, + "token_usage": { + "prompt": prompt_tokens, + "completion": completion_tokens, + "total": prompt_tokens + completion_tokens, + }, + } + if metadata_extra: + metadata.update(metadata_extra) + + event = { + "schema_version": SESSION_SCHEMA_VER, + "timestamp": _utc_now(), + "event_type": event_type, + "event_id": event_id, + "event_sequence": next_sequence, + "prev_event_sha256": self.prev_hash, + "event_sha256": None, + "run_id": self.run_id, + "phase_id": phase_id, + "loop_id": loop_id, + "agent_id": agent_id, + "parent_id": parent_id, + "role": role, + "step_id": step_id, + "tool_call_id": tool_call_id, + "result_id": result_id, + "artifact_ref": artifact_ref, + "artifact_sha256": artifact_sha256, + "diff_ref": diff_ref, + "model": self.model, + "content": content, + "metadata": metadata, + } + event["event_sha256"] = _compute_event_sha(event) + self._validate_event_before_persist(event) + _append_jsonl(self.path, event) + self.sequence = next_sequence + self.prev_hash = event["event_sha256"] + return event + + def tool_schema_context(self, tool_name: str) -> dict: + return { + "tool_schema_context": { + "mode": "catalog_plus_on_demand", + "catalog_ref": self.catalog_ref, + "catalog_sha256": self.catalog_sha, + "expanded_tool_names": [tool_name], + "request_schema_uri": TOOL_REQUEST_SCHEMA_URI, + "request_schema_sha256": self.request_schema_sha, + "result_schema_uri": TOOL_RESULT_SCHEMA_URI, + "result_schema_sha256": self.result_schema_sha, + } + } + + +class OpenAICompatibleClient: + def __init__( + self, + *, + api_base: str, + model: str, + timeout_seconds: int, + api_key_env: str, + temperature: float, + top_p: float, + max_tokens: int, + ) -> None: + base = (api_base or "").strip() + if not base: + raise RuntimeError("llm.api_base is required for llm execution mode") + if base.endswith("/chat/completions"): + self.url = base + else: + self.url = base.rstrip("/") + "/chat/completions" + self.model = model + self.timeout_seconds = timeout_seconds + self.temperature = temperature + self.top_p = top_p + self.max_tokens = max_tokens + key_name = (api_key_env or "").strip() + self.api_key = os.environ.get(key_name) if key_name else None + + def chat(self, messages: List[dict]) -> Tuple[str, dict]: + headers = {"Content-Type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + payload = { + "model": self.model, + "messages": messages, + "temperature": self.temperature, + "top_p": self.top_p, + "max_tokens": self.max_tokens, + } + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request(self.url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=self.timeout_seconds) as resp: + body = resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as e: + err_body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"LLM HTTP {e.code}: {err_body}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"LLM endpoint unreachable at {self.url}: {e}") from e + except Exception as e: # noqa: BLE001 + raise RuntimeError(f"LLM request failed: {e}") from e + + try: + parsed = json.loads(body) + except Exception as e: # noqa: BLE001 + raise RuntimeError(f"LLM response was not valid JSON: {e}") from e + + choices = parsed.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError("LLM response missing choices") + first = choices[0] if isinstance(choices[0], dict) else {} + message = first.get("message", {}) if isinstance(first.get("message"), dict) else {} + content = message.get("content") + if isinstance(content, str): + text = content + elif isinstance(content, list): + parts: List[str] = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text" and isinstance(part.get("text"), str): + parts.append(part["text"]) + text = "\n".join(parts) + else: + text = "" + usage = parsed.get("usage", {}) if isinstance(parsed.get("usage"), dict) else {} + return text, usage + + +class ToolExecutor: + def __init__( + self, + repo_root: str, + logger: SessionLogger, + run_id: str, + *, + agent_id: str, + phase: str, + step_id: str, + allowed_read_paths: List[str], + allowed_write_paths: List[str], + target_file_patterns: Optional[List[str]] = None, + docs_policy: Optional[dict] = None, + protected_write_paths: Optional[List[str]] = None, + enable_checkpoints: bool = False, + ) -> None: + self.repo_root = repo_root + self.logger = logger + self.run_id = run_id + self.agent_id = agent_id + self.phase = phase + self.step_id = step_id + self.allowed_read_paths = allowed_read_paths + self.allowed_write_paths = allowed_write_paths + self.target_file_patterns = [p for p in (target_file_patterns or []) if isinstance(p, str) and p] + self.docs_policy = docs_policy if isinstance(docs_policy, dict) else {} + self.protected_write_paths = [p for p in (protected_write_paths or []) if isinstance(p, str) and p] + self.enable_checkpoints = enable_checkpoints + self.tools_dir = os.path.join(repo_root, ".trinity", "runtime", "tools") + os.makedirs(self.tools_dir, exist_ok=True) + + def _is_allowed_path(self, rel_path: str, allowlist: List[str]) -> bool: + normalized = _normalize_rel_path(rel_path) + if _is_escape_rel_path(normalized): + return False + for allowed in allowlist: + if not isinstance(allowed, str) or not allowed: + continue + allowed_norm = allowed.replace("\\", "/").strip() + if allowed_norm.startswith("./"): + allowed_norm = allowed_norm[2:] + allowed_norm = allowed_norm.rstrip("/") or "." + if _is_escape_rel_path(allowed_norm): + continue + if allowed_norm == ".": + return True + if normalized == allowed_norm or normalized.startswith(allowed_norm + "/"): + return True + if fnmatch.fnmatch(normalized, allowed_norm): + return True + return False + + def _resolve_path(self, path_value: str) -> Tuple[str, str]: + if os.path.isabs(path_value): + abs_path = path_value + else: + abs_path = os.path.abspath(os.path.join(self.repo_root, path_value)) + real = os.path.realpath(abs_path) + root_real = os.path.realpath(self.repo_root) + rel = os.path.relpath(real, root_real).replace("\\", "/") + return real, rel + + def _matches_any_pattern(self, rel_path: str, patterns: List[str]) -> bool: + normalized = _normalize_rel_path(rel_path) + if _is_escape_rel_path(normalized): + return False + for pattern in patterns: + if not isinstance(pattern, str) or not pattern: + continue + pnorm = pattern.replace("\\", "/").strip() + if pnorm.startswith("./"): + pnorm = pnorm[2:] + pnorm = pnorm or "." + if _is_escape_rel_path(pnorm): + continue + if fnmatch.fnmatch(normalized, pnorm): + return True + return False + + def _is_protected_write_path(self, rel_path: str) -> bool: + return self._is_allowed_path(rel_path, self.protected_write_paths) + + def _is_allowed_write_target(self, rel_path: str) -> bool: + if not self._is_allowed_path(rel_path, self.allowed_write_paths): + return False + if not self.target_file_patterns: + return True + if self._matches_any_pattern(rel_path, self.target_file_patterns): + return True + doc_paths = self.docs_policy.get("doc_paths", []) if isinstance(self.docs_policy, dict) else [] + if isinstance(doc_paths, list) and self._matches_any_pattern(rel_path, [str(p) for p in doc_paths]): + return True + return False + + def _assert_write_allowed(self, rel_path: str, tool_name: str) -> None: + if self.phase in {"16b", "16c"} and self._is_protected_write_path(rel_path): + raise PermissionError(f"{tool_name} blocked by seed-authority guard: {rel_path}") + if not self._is_allowed_write_target(rel_path): + raise PermissionError(f"{tool_name} blocked by write scope: {rel_path}") + + def _worktree_snapshot(self) -> Set[str]: + status = _run_git(self.repo_root, ["status", "--porcelain", "--untracked-files=all"], check=False) + if status.returncode != 0: + raise PermissionError("exec_cmd blocked: unable to inspect worktree state for readonly guard") + ignored_prefixes = (".trinity/runtime/home", ".trinity/runtime/tmp") + snapshot: Set[str] = set() + for line in (status.stdout or "").splitlines(): + if not line.strip(): + continue + path_fragment = line[3:] if len(line) > 3 else line + if " -> " in path_fragment: + path_fragment = path_fragment.split(" -> ", 1)[1] + normalized = path_fragment.strip().replace("\\", "/") + if any(normalized == prefix or normalized.startswith(prefix + "/") for prefix in ignored_prefixes): + continue + snapshot.add(line.rstrip()) + return snapshot + + def _fallback_result(self, tool_name: str, args: dict) -> dict: + head = _git_head(self.repo_root) or ("0" * 40) + default_path = args.get("path") if isinstance(args.get("path"), str) and args.get("path").strip() else "." + if tool_name == "read_file": + start = args.get("start_line", 1) + start_line = start if isinstance(start, int) and start > 0 else 1 + end = args.get("end_line") + end_line = end if isinstance(end, int) and end > 0 else None + return {"path": default_path, "line_start": start_line, "line_end": end_line, "bytes_read": 0, "content": "", "truncated": False} + if tool_name == "write_file": + return {"path": default_path, "bytes_written": 0, "content_sha256": "sha256:" + _sha256_text("")} + if tool_name == "edit_file": + return {"path": default_path, "edits_applied": 0, "content_sha256": "sha256:" + _sha256_text("")} + if tool_name == "apply_patch": + return {"files_changed": 0, "hunks_applied": 0} + if tool_name == "move_file": + src_path = args.get("src_path") if isinstance(args.get("src_path"), str) and args.get("src_path").strip() else default_path + dst_path = args.get("dst_path") if isinstance(args.get("dst_path"), str) and args.get("dst_path").strip() else default_path + return {"src_path": src_path, "dst_path": dst_path, "content_sha256": "sha256:" + _sha256_text("")} + if tool_name == "remove_file": + return {"path": default_path, "removed": False, "previously_missing": True} + if tool_name == "list_dir": + return {"path": default_path, "entries": []} + if tool_name == "glob_match": + patterns_raw = args.get("patterns", []) + patterns = [p for p in patterns_raw if isinstance(p, str) and p] if isinstance(patterns_raw, list) else [] + if not patterns: + patterns = ["*"] + return {"path": default_path, "patterns": patterns, "matches": []} + if tool_name == "search_text": + pattern = args.get("pattern") if isinstance(args.get("pattern"), str) and args.get("pattern") else "" + return {"pattern": pattern, "paths_scanned": 0, "matches": []} + if tool_name == "git_head": + return {"head": head} + if tool_name == "git_show": + rev = args.get("rev") if isinstance(args.get("rev"), str) and args.get("rev") else "HEAD" + return {"rev": rev, "content_excerpt": "", "truncated": False} + if tool_name == "git_diff": + return {"base_rev": None, "head_rev": None, "diff_excerpt": "", "truncated": False} + if tool_name == "exec_cmd": + command = args.get("command") if isinstance(args.get("command"), str) and args.get("command") else "" + mode = args.get("mode") if args.get("mode") in {"standard", "summarized"} else "summarized" + return {"command": command, "mode": mode} + if tool_name == "validate_json": + return {"path": default_path, "valid": False, "errors": []} + if tool_name == "checkpoint_branch": + branch = args.get("branch_name") if isinstance(args.get("branch_name"), str) and args.get("branch_name") else f"trinity/{self.step_id or 'default'}" + return {"branch_name": branch, "head": head} + if tool_name == "checkpoint_commit": + message = args.get("message") if isinstance(args.get("message"), str) and args.get("message") else "checkpoint blocked" + return {"commit_sha": head, "message": message} + return {} + + def call( + self, + tool_name: str, + args: dict, + *, + role: str = "Orchestrator", + parent_id: Optional[str] = None, + loop_id: str = "l1", + ) -> dict: + call_id = f"tool-{uuid.uuid4().hex[:12]}" + created_at = _utc_now() + request = { + "protocol_version": PROTO_VER, + "run_id": self.run_id, + "call_id": call_id, + "agent_id": self.agent_id, + "parent_id": parent_id, + "role": role, + "phase": self.phase, + "step_id": self.step_id, + "tool_name": tool_name, + "args": args, + "working_dir": self.repo_root, + "timeout_seconds": 120, + "created_at": created_at, + } + request_path = os.path.join(self.tools_dir, "tool_call_request.json") + _write_json_atomic(request_path, request) + request_errors = validate_runtime_file(self.repo_root, request_path, "tool_call_request") + if request_errors: + raise RuntimeError("; ".join(request_errors)) + + self.logger.append( + "TOOL_CALL", + role=role, + phase_id=self.phase, + loop_id=loop_id, + agent_id=self.agent_id, + parent_id=parent_id, + summary=f"{tool_name} request", + prompt_template_id="tool_protocol", + step_id=self.step_id, + tool_call_id=call_id, + content_extra={"tool_call": {"name": tool_name, "args": args}}, + metadata_extra=self.logger.tool_schema_context(tool_name), + ) + + started = time.monotonic() + status = "success" + exit_code: Optional[int] = None + stdout_excerpt: Optional[str] = None + stderr_excerpt: Optional[str] = None + truncated = False + artifact_ref: Optional[str] = None + artifact_sha: Optional[str] = None + result: dict = {} + error_payload: Optional[dict] = None + + try: + if tool_name == "exec_cmd": + result, exit_code, stdout_excerpt, stderr_excerpt, truncated = self._exec_cmd(args) + elif tool_name == "read_file": + result = self._read_file(args) + elif tool_name == "write_file": + result, artifact_ref, artifact_sha = self._write_file(args) + elif tool_name == "edit_file": + result, artifact_ref, artifact_sha = self._edit_file(args) + elif tool_name == "move_file": + result, artifact_ref, artifact_sha = self._move_file(args) + elif tool_name == "remove_file": + result, artifact_ref, artifact_sha = self._remove_file(args) + elif tool_name == "list_dir": + result = self._list_dir(args) + elif tool_name == "glob_match": + result = self._glob_match(args) + elif tool_name == "search_text": + result = self._search_text(args) + elif tool_name == "git_head": + result = self._tool_git_head() + elif tool_name == "git_show": + result = self._git_show(args) + elif tool_name == "git_diff": + result = self._git_diff(args) + elif tool_name == "validate_json": + result = self._validate_json(args) + elif tool_name == "checkpoint_branch": + result = self._checkpoint_branch(args) + elif tool_name == "checkpoint_commit": + result = self._checkpoint_commit(args) + elif tool_name == "apply_patch": + result, artifact_ref, artifact_sha = self._apply_patch(args) + else: + raise RuntimeError(f"unsupported tool '{tool_name}'") + except TimeoutError as e: + status = "timeout" + error_payload = {"code": "timeout", "message": str(e)} + except PermissionError as e: + status = "blocked" + error_payload = {"code": "blocked", "message": str(e)} + except Exception as e: # noqa: BLE001 + status = "error" + error_payload = {"code": "error", "message": str(e)} + + if not result: + result = self._fallback_result(tool_name, args) + if tool_name == "exec_cmd" and not isinstance(exit_code, int): + exit_code = 1 + + duration_ms = int((time.monotonic() - started) * 1000) + result_id = f"result-{uuid.uuid4().hex[:12]}" + summary = f"{tool_name} {status}" + result_payload = { + "protocol_version": PROTO_VER, + "run_id": self.run_id, + "call_id": call_id, + "result_id": result_id, + "agent_id": self.agent_id, + "role": role, + "phase": self.phase, + "step_id": self.step_id, + "tool_name": tool_name, + "status": status, + "summary": summary, + "result": result, + "duration_ms": duration_ms, + "exit_code": exit_code, + "working_dir": self.repo_root, + "stdout_excerpt": stdout_excerpt, + "stderr_excerpt": stderr_excerpt, + "truncated": bool(truncated), + "artifact_ref": artifact_ref, + "artifact_sha256": artifact_sha, + "finished_at": _utc_now(), + } + if error_payload: + result_payload["error"] = error_payload + + result_path = os.path.join(self.tools_dir, "tool_call_result.json") + _write_json_atomic(result_path, result_payload) + result_errors = validate_runtime_file(self.repo_root, result_path, "tool_call_result") + if result_errors: + raise RuntimeError("; ".join(result_errors)) + + self.logger.append( + "TOOL_RESULT", + role=role, + phase_id=self.phase, + loop_id=loop_id, + agent_id=self.agent_id, + parent_id=parent_id, + summary=summary, + prompt_template_id="tool_protocol", + step_id=self.step_id, + tool_call_id=call_id, + result_id=result_id, + artifact_ref=artifact_ref, + artifact_sha256=artifact_sha, + content_extra={ + "tool_result": { + "command": result_payload.get("result", {}).get("command", tool_name), + "exit_code": int(exit_code) if isinstance(exit_code, int) else 0, + "duration_ms": duration_ms, + "working_dir": self.repo_root, + "stdout_excerpt": stdout_excerpt or "", + "stderr_excerpt": stderr_excerpt or "", + "truncated": bool(truncated), + } + }, + metadata_extra=self.logger.tool_schema_context(tool_name), + ) + return result_payload + + def _read_file(self, args: dict) -> dict: + path = args.get("path") + if not isinstance(path, str) or not path: + raise RuntimeError("read_file: missing path") + abs_path, rel = self._resolve_path(path) + if not self._is_allowed_path(rel, self.allowed_read_paths): + raise PermissionError(f"read_file blocked by allowlist: {rel}") + if not os.path.exists(abs_path): + raise RuntimeError(f"file not found: {rel}") + start_line = int(args.get("start_line", 1)) + end_line = args.get("end_line") + with open(abs_path, "r", encoding="utf-8", errors="ignore") as f: + lines = f.readlines() + if isinstance(end_line, int): + chunk = lines[start_line - 1 : end_line] + line_end: Optional[int] = end_line + else: + chunk = lines[start_line - 1 :] + line_end = None + content = "".join(chunk) + max_chars = args.get("max_chars") + truncated = False + if isinstance(max_chars, int) and max_chars > 0 and len(content) > max_chars: + content = content[:max_chars] + truncated = True + return { + "path": rel, + "line_start": start_line, + "line_end": line_end, + "bytes_read": len("".join(chunk).encode("utf-8")), + "content": content, + "truncated": truncated, + } + + def _write_file(self, args: dict) -> Tuple[dict, str, str]: + path = args.get("path") + content = args.get("content") + if not isinstance(path, str) or not path: + raise RuntimeError("write_file: missing path") + if not isinstance(content, str): + raise RuntimeError("write_file: missing content") + abs_path, rel = self._resolve_path(path) + self._assert_write_allowed(rel, "write_file") + create_parents = bool(args.get("create_parents", True)) + mode = args.get("mode", "overwrite") + if create_parents: + os.makedirs(os.path.dirname(abs_path), exist_ok=True) + if mode == "append": + with open(abs_path, "a", encoding="utf-8") as f: + f.write(content) + elif mode == "create_new": + if os.path.exists(abs_path): + raise RuntimeError(f"write_file create_new target exists: {rel}") + with open(abs_path, "x", encoding="utf-8") as f: + f.write(content) + else: + with open(abs_path, "w", encoding="utf-8") as f: + f.write(content) + sha = "sha256:" + _sha256_file(abs_path) + return {"path": rel, "bytes_written": len(content.encode("utf-8")), "content_sha256": sha}, rel, sha + + def _edit_file(self, args: dict) -> Tuple[dict, str, str]: + path = args.get("path") + edits = args.get("edits") + if not isinstance(path, str) or not path: + raise RuntimeError("edit_file: missing path") + if not isinstance(edits, list) or not edits: + raise RuntimeError("edit_file: missing edits") + abs_path, rel = self._resolve_path(path) + self._assert_write_allowed(rel, "edit_file") + if not os.path.exists(abs_path): + raise RuntimeError(f"edit_file target missing: {rel}") + with open(abs_path, "r", encoding="utf-8") as f: + content = f.read() + applied = 0 + for edit in edits: + if not isinstance(edit, dict): + continue + search = edit.get("search") + replace = edit.get("replace") + regex = bool(edit.get("regex", False)) + occurrence = edit.get("occurrence") + if not isinstance(search, str) or not isinstance(replace, str): + continue + if regex: + count = 0 if not isinstance(occurrence, int) else 1 + new_content, n = re.subn(search, replace, content, count=count) + if n > 0: + content = new_content + applied += n + else: + if isinstance(occurrence, int) and occurrence > 0: + idx = -1 + start = 0 + for _ in range(occurrence): + idx = content.find(search, start) + if idx == -1: + break + start = idx + len(search) + if idx != -1: + content = content[:idx] + replace + content[idx + len(search) :] + applied += 1 + else: + n = content.count(search) + if n > 0: + content = content.replace(search, replace) + applied += n + with open(abs_path, "w", encoding="utf-8") as f: + f.write(content) + sha = "sha256:" + _sha256_text(content) + return {"path": rel, "edits_applied": applied, "content_sha256": sha}, rel, sha + + def _move_file(self, args: dict) -> Tuple[dict, str, str]: + src_path = args.get("src_path") + dst_path = args.get("dst_path") + overwrite = bool(args.get("overwrite", False)) + create_parents = bool(args.get("create_parents", True)) + if not isinstance(src_path, str) or not src_path.strip(): + raise RuntimeError("move_file: missing src_path") + if not isinstance(dst_path, str) or not dst_path.strip(): + raise RuntimeError("move_file: missing dst_path") + + src_abs, src_rel = self._resolve_path(src_path) + dst_abs, dst_rel = self._resolve_path(dst_path) + self._assert_write_allowed(src_rel, "move_file") + self._assert_write_allowed(dst_rel, "move_file") + + if not os.path.exists(src_abs): + raise RuntimeError(f"move_file source missing: {src_rel}") + if os.path.isdir(src_abs): + raise RuntimeError(f"move_file source must be a file: {src_rel}") + if os.path.exists(dst_abs) and not overwrite: + raise RuntimeError(f"move_file destination exists: {dst_rel}") + if create_parents: + os.makedirs(os.path.dirname(dst_abs), exist_ok=True) + os.replace(src_abs, dst_abs) + sha = "sha256:" + _sha256_file(dst_abs) + return {"src_path": src_rel, "dst_path": dst_rel, "content_sha256": sha}, dst_rel, sha + + def _remove_file(self, args: dict) -> Tuple[dict, str, str]: + path = args.get("path") + missing_ok = bool(args.get("missing_ok", False)) + if not isinstance(path, str) or not path.strip(): + raise RuntimeError("remove_file: missing path") + abs_path, rel = self._resolve_path(path) + self._assert_write_allowed(rel, "remove_file") + + if not os.path.exists(abs_path): + if not missing_ok: + raise RuntimeError(f"remove_file target missing: {rel}") + artifact_sha = "sha256:" + _sha256_text(f"missing:{rel}") + return {"path": rel, "removed": False, "previously_missing": True}, rel, artifact_sha + if os.path.isdir(abs_path): + raise RuntimeError(f"remove_file target must be file: {rel}") + prior_sha = "sha256:" + _sha256_file(abs_path) + os.remove(abs_path) + return {"path": rel, "removed": True, "previously_missing": False}, rel, prior_sha + + def _apply_patch(self, args: dict) -> Tuple[dict, str, str]: + patch = args.get("patch") + if not isinstance(patch, str) or not patch.strip(): + raise RuntimeError("apply_patch: missing patch") + + touched_paths: List[str] = [] + for line in patch.splitlines(): + if not (line.startswith("+++ ") or line.startswith("--- ")): + continue + path = line[4:].strip() + if path == "/dev/null": + continue + if path.startswith("b/") or path.startswith("a/"): + path = path[2:] + if path and path not in touched_paths: + touched_paths.append(path) + + if not touched_paths: + raise RuntimeError("apply_patch: no target paths detected") + + for rel in touched_paths: + self._assert_write_allowed(rel, "apply_patch") + + patch_path = os.path.join(self.tools_dir, f"patch-{uuid.uuid4().hex[:12]}.diff") + with open(patch_path, "w", encoding="utf-8") as f: + f.write(patch) + if not patch.endswith("\n"): + f.write("\n") + + try: + result = _run_git(self.repo_root, ["apply", "--whitespace=nowarn", patch_path], check=False) + if result.returncode != 0: + msg = (result.stderr or result.stdout or "").strip() + raise RuntimeError(msg or "git apply failed") + finally: + if os.path.exists(patch_path): + os.remove(patch_path) + + artifact_ref = touched_paths[0] + artifact_abs, artifact_rel = self._resolve_path(artifact_ref) + if os.path.exists(artifact_abs): + patch_sha = "sha256:" + _sha256_file(artifact_abs) + else: + patch_sha = "sha256:" + _sha256_text(patch) + hunk_count = sum(1 for line in patch.splitlines() if line.startswith("@@")) + return {"files_changed": len(touched_paths), "hunks_applied": hunk_count}, artifact_rel, patch_sha + + def _list_dir(self, args: dict) -> dict: + path = args.get("path") + recursive = bool(args.get("recursive", False)) + include_hidden = bool(args.get("include_hidden", False)) + if not isinstance(path, str) or not path: + raise RuntimeError("list_dir: missing path") + abs_path, rel = self._resolve_path(path) + if not self._is_allowed_path(rel, self.allowed_read_paths): + raise PermissionError(f"list_dir blocked by allowlist: {rel}") + if not os.path.isdir(abs_path): + raise RuntimeError(f"directory not found: {rel}") + entries: List[str] = [] + if recursive: + for root, dirs, files in os.walk(abs_path): + for name in dirs + files: + if not include_hidden and name.startswith("."): + continue + entries.append(_rel(self.repo_root, os.path.join(root, name))) + else: + for name in sorted(os.listdir(abs_path)): + if not include_hidden and name.startswith("."): + continue + entries.append(_rel(self.repo_root, os.path.join(abs_path, name))) + return {"path": rel, "entries": entries} + + def _glob_match(self, args: dict) -> dict: + path = args.get("path") + patterns = args.get("patterns") + if not isinstance(path, str) or not path: + raise RuntimeError("glob_match: missing path") + if not isinstance(patterns, list) or not patterns: + raise RuntimeError("glob_match: missing patterns") + abs_path, rel = self._resolve_path(path) + if not self._is_allowed_path(rel, self.allowed_read_paths): + raise PermissionError(f"glob_match blocked by allowlist: {rel}") + matches: List[str] = [] + for pattern in patterns: + if not isinstance(pattern, str) or not pattern: + continue + for matched in Path(abs_path).glob(pattern): + matches.append(_rel(self.repo_root, str(matched))) + return {"path": rel, "patterns": patterns, "matches": sorted(set(matches))} + + def _search_text(self, args: dict) -> dict: + pattern = args.get("pattern") + paths = args.get("paths") + if not isinstance(pattern, str) or not pattern: + raise RuntimeError("search_text: missing pattern") + if not isinstance(paths, list) or not paths: + raise RuntimeError("search_text: missing paths") + regex = bool(args.get("use_regex", args.get("regex", False))) + case_sensitive = bool(args.get("case_sensitive", True)) + results: List[dict] = [] + paths_scanned = 0 + flags = 0 if case_sensitive else re.IGNORECASE + compiled = re.compile(pattern, flags=flags) if regex else None + for path in paths: + if not isinstance(path, str) or not path: + continue + abs_path, rel = self._resolve_path(path) + if not self._is_allowed_path(rel, self.allowed_read_paths): + continue + if not os.path.isfile(abs_path): + continue + paths_scanned += 1 + with open(abs_path, "r", encoding="utf-8", errors="ignore") as f: + for i, line in enumerate(f, start=1): + hit = bool(compiled.search(line)) if compiled else (pattern in line) + if hit: + results.append({"path": rel, "line": i, "text_excerpt": line.rstrip("\n")[:400]}) + return {"pattern": pattern, "paths_scanned": paths_scanned, "matches": results} + + def _tool_git_head(self) -> dict: + head = _git_head(self.repo_root) + if not head: + raise RuntimeError("git_head unavailable") + return {"head": head} + + def _git_show(self, args: dict) -> dict: + rev = args.get("rev") + if not isinstance(rev, str) or not rev: + raise RuntimeError("git_show: missing rev") + result = _run_git(self.repo_root, ["show", rev], check=False) + content = (result.stdout or "") + ("\n" + (result.stderr or "") if result.stderr else "") + excerpt = content[:4000] + return {"rev": rev, "content_excerpt": excerpt, "truncated": len(content) > len(excerpt)} + + def _git_diff(self, args: dict) -> dict: + base_rev = args.get("base_rev", args.get("rev_a")) + head_rev = args.get("head_rev", args.get("rev_b")) + context_lines = args.get("context_lines") + paths = args.get("paths") + cmd = ["diff"] + if isinstance(context_lines, int) and context_lines >= 0: + cmd.extend(["-U", str(context_lines)]) + if isinstance(base_rev, str) and base_rev and isinstance(head_rev, str) and head_rev: + cmd.append(f"{base_rev}..{head_rev}") + elif isinstance(base_rev, str) and base_rev: + cmd.append(base_rev) + elif isinstance(head_rev, str) and head_rev: + cmd.append(head_rev) + if isinstance(paths, list): + normalized_paths = [p for p in paths if isinstance(p, str) and p] + if normalized_paths: + cmd.append("--") + cmd.extend(normalized_paths) + result = _run_git(self.repo_root, cmd, check=False) + content = (result.stdout or "") + ("\n" + (result.stderr or "") if result.stderr else "") + excerpt = content[:4000] + return { + "base_rev": base_rev if isinstance(base_rev, str) else None, + "head_rev": head_rev if isinstance(head_rev, str) else None, + "diff_excerpt": excerpt, + "truncated": len(content) > len(excerpt), + } + + def _validate_json(self, args: dict) -> dict: + path = args.get("path") + if not isinstance(path, str) or not path: + raise RuntimeError("validate_json: missing path") + abs_path, rel = self._resolve_path(path) + errs = validate_file(self.repo_root, abs_path) + return {"path": rel, "valid": not errs, "errors": errs} + + def _checkpoint_branch(self, args: dict) -> dict: + branch = args.get("branch_name", args.get("branch")) + if not isinstance(branch, str) or not branch: + raise RuntimeError("checkpoint_branch: missing branch") + if not branch.startswith("trinity/"): + raise RuntimeError("checkpoint_branch: branch must start with 'trinity/'") + if not self.enable_checkpoints: + head = _git_head(self.repo_root) + if not head: + raise RuntimeError("checkpoint_branch skipped but git head unavailable") + return {"branch_name": branch, "head": head} + _run_git(self.repo_root, ["switch", "main"]) + exists = _run_git(self.repo_root, ["rev-parse", "--verify", branch], check=False).returncode == 0 + if exists: + _run_git(self.repo_root, ["switch", branch]) + else: + _run_git(self.repo_root, ["switch", "-c", branch]) + head = _git_head(self.repo_root) + if not head: + raise RuntimeError("checkpoint_branch completed but git head unavailable") + return {"branch_name": branch, "head": head} + + def _checkpoint_commit(self, args: dict) -> dict: + message = args.get("message") + if not isinstance(message, str) or not message.strip(): + raise RuntimeError("checkpoint_commit: missing message") + if not self.enable_checkpoints: + head = _git_head(self.repo_root) + if not head: + raise RuntimeError("checkpoint_commit skipped but git head unavailable") + return {"commit_sha": head, "message": message} + _run_git(self.repo_root, ["add", "-A"]) + commit = _run_git(self.repo_root, ["commit", "-m", message], check=False) + if commit.returncode != 0 and "nothing to commit" in (commit.stderr or commit.stdout): + head = _git_head(self.repo_root) + if not head: + raise RuntimeError("checkpoint_commit noop but git head unavailable") + return {"commit_sha": head, "message": message} + if commit.returncode != 0: + raise RuntimeError((commit.stderr or commit.stdout or "").strip()) + head = _git_head(self.repo_root) + if not head: + raise RuntimeError("checkpoint_commit completed but git head unavailable") + return {"commit_sha": head, "message": message} + + def _exec_cmd(self, args: dict) -> Tuple[dict, int, str, str, bool]: + command = args.get("command") + mode = args.get("mode") + if not isinstance(command, str) or not command.strip(): + raise RuntimeError("exec_cmd: missing command") + if mode not in {"standard", "summarized"}: + raise RuntimeError("exec_cmd: mode must be 'standard' or 'summarized'") + # Unattended safety gate: block common secret-dumping commands. + if _is_secret_dump_command(command): + raise PermissionError("exec_cmd blocked by unattended command safety policy") + + readonly_snapshot: Optional[Set[str]] = None + if self.phase in {"16c", "utility"}: + self._assert_exec_cmd_readonly(command) + readonly_snapshot = self._worktree_snapshot() + + timeout_seconds = int(args.get("timeout_seconds", 300)) + started = time.monotonic() + env = os.environ.copy() + runtime_home = os.path.join(self.repo_root, ".trinity", "runtime", "home") + runtime_tmp = os.path.join(self.repo_root, ".trinity", "runtime", "tmp") + os.makedirs(runtime_home, exist_ok=True) + os.makedirs(runtime_tmp, exist_ok=True) + env["HOME"] = runtime_home + env["TMPDIR"] = runtime_tmp + proc = subprocess.run( + command, + shell=True, + cwd=self.repo_root, + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + env=env, + ) + duration_ms = int((time.monotonic() - started) * 1000) + stdout = proc.stdout or "" + stderr = proc.stderr or "" + truncated = False + if mode == "summarized": + stdout_lines = stdout.splitlines() + stderr_lines = stderr.splitlines() + stdout_out = "\n".join(stdout_lines[:40]) + stderr_out = "\n".join(stderr_lines[:20]) + truncated = len(stdout_lines) > 40 or len(stderr_lines) > 20 + else: + stdout_out = stdout + stderr_out = stderr + if readonly_snapshot is not None: + after_snapshot = self._worktree_snapshot() + if after_snapshot != readonly_snapshot: + raise PermissionError("exec_cmd blocked: readonly command produced worktree side effects") + return ( + { + "command": command, + "mode": mode, + }, + proc.returncode, + stdout_out, + stderr_out, + truncated, + ) + + def _assert_exec_cmd_readonly(self, command: str) -> None: + lowered = command.lower() + write_operator_patterns = [ + re.compile(r"(^|[^<])>(>|)?"), + re.compile(r"<<\s*", lowered): + raise PermissionError("exec_cmd blocked: output redirection is not allowed in readonly mode") + + try: + tokens = shlex.split(command, posix=True) + except Exception: + raise PermissionError("exec_cmd blocked: command parsing failed for readonly policy") + if not tokens: + raise PermissionError("exec_cmd blocked: empty command") + + binary = tokens[0].lower() + mutating_bins = { + "mv", + "cp", + "rm", + "touch", + "mkdir", + "rmdir", + "chmod", + "chown", + "ln", + "install", + "truncate", + "dd", + } + if binary in mutating_bins: + raise PermissionError(f"exec_cmd blocked: '{binary}' is disallowed in readonly mode") + + if binary == "sed" and "-i" in tokens: + raise PermissionError("exec_cmd blocked: 'sed -i' is disallowed in readonly mode") + + if binary == "git": + sub = tokens[1].lower() if len(tokens) > 1 else "" + if sub not in READONLY_GIT_SUBCOMMANDS: + raise PermissionError(f"exec_cmd blocked: git subcommand '{sub or ''}' is not readonly") + + if binary in {"python", "python3"} and "-c" in tokens: + idx = tokens.index("-c") + snippet = tokens[idx + 1] if idx + 1 < len(tokens) else "" + if re.search(r"\b(open|Path)\s*\(", snippet) and re.search(r"\b(write|append|touch|mkdir|unlink|remove|rename)\b", snippet): + raise PermissionError("exec_cmd blocked: python -c snippet includes mutating filesystem operations") + + if binary in {"bash", "sh", "zsh"} and any(flag in tokens for flag in ("-c", "-lc")): + idx = max(i for i, t in enumerate(tokens) if t in {"-c", "-lc"}) + nested = tokens[idx + 1] if idx + 1 < len(tokens) else "" + if re.search(r"(^|[^<])>(>|)?|<< None: + self.repo_root = repo_root + self.step_id = step_id + self.milestone_path = milestone_path + self.anchor_path = anchor_path + self.allow_authority_fallback = allow_authority_fallback + self.seed_manifest_rel = "spec/common/seed_manifest.json" + self.seed_manifest_path = os.path.join(repo_root, self.seed_manifest_rel) + if not os.path.exists(self.seed_manifest_path): + raise RuntimeError(f"seed manifest missing: {self.seed_manifest_rel}") + self.seed_manifest = _read_json(self.seed_manifest_path) + self.git_head = _git_head(repo_root) + if not self.git_head: + raise RuntimeError("git head commit not available; required for spec_ref grounding") + self._bootstrap_selection_trace: List[dict] = [] + + def seed_files_ordered(self, phase: str) -> List[str]: + seeds = self.seed_manifest.get("seeds", []) + seed_path_by_id: Dict[str, str] = {} + for seed in seeds: + if isinstance(seed, dict): + sid = seed.get("seed_id") + spath = seed.get("path") + if isinstance(sid, str) and isinstance(spath, str): + seed_path_by_id[sid] = spath + + ordered: List[str] = [] + global_seed_order = self.seed_manifest.get("global_seed_order", []) + if isinstance(global_seed_order, list): + for sid in global_seed_order: + if isinstance(sid, str) and sid in seed_path_by_id and seed_path_by_id[sid] not in ordered: + ordered.append(seed_path_by_id[sid]) + + req = self.seed_manifest.get("step_requirements", {}) + phase_seed_ids = req.get(phase, []) if isinstance(req, dict) else [] + if isinstance(phase_seed_ids, list): + for sid in phase_seed_ids: + if isinstance(sid, str) and sid in seed_path_by_id and seed_path_by_id[sid] not in ordered: + ordered.append(seed_path_by_id[sid]) + return ordered + + def docs_policy(self) -> dict: + policy = self.seed_manifest.get("docs_policy", {}) + if not isinstance(policy, dict): + return {"doc_paths": ["docs/**", "README.md", "CHANGELOG.md"], "readme_required": True, "root_readme_required": True} + doc_paths = policy.get("doc_paths") + if not isinstance(doc_paths, list) or not doc_paths: + doc_paths = ["docs/**", "README.md", "CHANGELOG.md"] + return { + "doc_paths": [p for p in doc_paths if isinstance(p, str) and p], + "readme_required": bool(policy.get("readme_required", True)), + "root_readme_required": bool(policy.get("root_readme_required", True)), + } + + def _collect_json_ids(self, payload: Any, out: List[str]) -> None: + if isinstance(payload, dict): + for k, v in payload.items(): + if isinstance(v, str) and (k == "id" or k.endswith("_id")) and v: + out.append(v) + self._collect_json_ids(v, out) + elif isinstance(payload, list): + for item in payload: + self._collect_json_ids(item, out) + + def _resolve_line_range(self, rel_path: str, item_id: str) -> Optional[str]: + abs_path = os.path.join(self.repo_root, rel_path) + if not os.path.exists(abs_path): + return None + with open(abs_path, "r", encoding="utf-8", errors="ignore") as f: + lines = f.readlines() + patterns = [ + re.compile(rf'"id"\s*:\s*"{re.escape(item_id)}"'), + re.compile(rf'"[a-z_]*id"\s*:\s*"{re.escape(item_id)}"'), + re.compile(rf'"{re.escape(item_id)}"'), + ] + for i, line in enumerate(lines, start=1): + for p in patterns: + if p.search(line): + return f"L{i}-L{i}" + return None + + def resolve_spec_ref(self, spec_type: str, item_id: str) -> dict: + preferred = SPEC_FILE_BY_TYPE.get(spec_type) + candidates: List[str] = [] + if preferred: + candidates.append(preferred) + candidates.extend([p for p in CORE_AUTHORITY_FILES if p not in candidates]) + for rel_path in candidates: + abs_path = os.path.join(self.repo_root, rel_path) + if not os.path.exists(abs_path): + continue + try: + payload = _read_json(abs_path) + except Exception: + continue + ids: List[str] = [] + self._collect_json_ids(payload, ids) + if item_id not in ids: + continue + line_range = self._resolve_line_range(rel_path, item_id) + if not line_range: + continue + return { + "type": spec_type, + "id": item_id, + "path": rel_path, + "line_range": line_range, + "commit_hash": self.git_head, + } + raise RuntimeError(f"Unable to resolve grounded spec ref: {spec_type}:{item_id}") + + def _tokenize_candidate_ids(self, value: str, source: str) -> List[dict]: + out: List[dict] = [] + if not isinstance(value, str): + return out + for token in re.findall(r"[A-Za-z0-9_-]+", value): + normalized = token.strip().lower() + if not normalized or "-" not in normalized: + continue + if normalized.startswith(("fr-", "api-", "nfr-", "inv-", "fixture-")): + ref_type = normalized.split("-", 1)[0] + out.append({"type_hint": ref_type, "item_id": normalized, "source": source, "candidate_mode": "tokenized"}) + else: + out.append({"type_hint": None, "item_id": normalized, "source": source, "candidate_mode": "tokenized"}) + return out + + def _roadmap_bootstrap_candidates(self) -> List[dict]: + roadmap_path = os.path.join(self.repo_root, "spec", "14_roadmap.json") + if not os.path.exists(roadmap_path): + return [] + try: + roadmap = _read_json(roadmap_path) + except Exception: + return [] + milestones = roadmap.get("milestones", []) + if not isinstance(milestones, list): + return [] + current = None + for milestone in milestones: + if isinstance(milestone, dict) and milestone.get("milestone_id") == self.step_id: + current = milestone + break + if not isinstance(current, dict): + return [] + values: List[Tuple[str, str]] = [] + + candidates: List[dict] = [] + seen: Set[Tuple[Optional[str], str]] = set() + + def _add_candidate(type_hint: Optional[str], item_id: str, source: str, candidate_mode: str) -> None: + key = (type_hint, item_id) + if key in seen: + return + seen.add(key) + candidates.append( + { + "type_hint": type_hint, + "item_id": item_id, + "source": source, + "candidate_mode": candidate_mode, + } + ) + + deliverables = current.get("deliverables", []) + if isinstance(deliverables, list): + for idx, deliverable in enumerate(deliverables): + if not isinstance(deliverable, dict): + continue + dtype = deliverable.get("type") + did = deliverable.get("id") + if dtype in {"fr", "api", "nfr", "inv", "fixture"} and isinstance(did, str) and did: + _add_candidate(dtype, did, f"roadmap.milestones[{self.step_id}].deliverables[{idx}]", "structured") + + tasks = current.get("tasks") + if isinstance(tasks, list): + for task_idx, task in enumerate(tasks): + if not isinstance(task, dict): + continue + acceptance = task.get("acceptance_criteria") + if isinstance(acceptance, list): + for crit_idx, criterion in enumerate(acceptance): + if not isinstance(criterion, dict): + continue + fixture_ref = criterion.get("fixture_ref") + if isinstance(fixture_ref, str) and fixture_ref: + _add_candidate( + "fixture", + fixture_ref, + f"roadmap.milestones[{self.step_id}].tasks[{task_idx}].acceptance_criteria[{crit_idx}].fixture_ref", + "structured", + ) + + for key in ("name", "user_story", "milestone_id"): + v = current.get(key) + if isinstance(v, str) and v.strip(): + values.append((v, f"roadmap.milestones[{self.step_id}].{key}")) + for key in ("deliverables", "source_milestones"): + arr = current.get(key) + if isinstance(arr, list): + for idx, entry in enumerate(arr): + if isinstance(entry, str) and entry.strip(): + values.append((entry, f"roadmap.milestones[{self.step_id}].{key}[{idx}]")) + if isinstance(tasks, list): + for task_idx, task in enumerate(tasks): + if not isinstance(task, dict): + continue + task_id = task.get("task_id") + desc = task.get("description") + if isinstance(task_id, str) and task_id.strip(): + values.append((task_id, f"roadmap.milestones[{self.step_id}].tasks[{task_idx}].task_id")) + if isinstance(desc, str) and desc.strip(): + values.append((desc, f"roadmap.milestones[{self.step_id}].tasks[{task_idx}].description")) + + for value, source in values: + for candidate in self._tokenize_candidate_ids(value, source): + type_hint = candidate.get("type_hint") + item_id = candidate.get("item_id") + if isinstance(item_id, str) and item_id: + _add_candidate( + type_hint if isinstance(type_hint, str) else None, + item_id, + str(candidate.get("source", source)), + str(candidate.get("candidate_mode", "tokenized")), + ) + return candidates + + def _first_available_ref_for_type(self, spec_type: str) -> Optional[dict]: + rel_path = SPEC_FILE_BY_TYPE.get(spec_type) + if not isinstance(rel_path, str): + return None + abs_path = os.path.join(self.repo_root, rel_path) + if not os.path.exists(abs_path): + return None + try: + payload = _read_json(abs_path) + except Exception: + return None + ids: List[str] = [] + self._collect_json_ids(payload, ids) + for item_id in ids: + try: + return self.resolve_spec_ref(spec_type, item_id) + except Exception: + continue + return None + + def _bootstrap_required_spec_refs(self) -> List[dict]: + refs: List[dict] = [] + seen: Set[Tuple[str, str]] = set() + trace: List[dict] = [] + self._bootstrap_selection_trace = [] + + def _try_add(spec_type: str, item_id: str, source: str, selection_mode: str) -> None: + key = (spec_type, item_id) + if key in seen: + return + preferred_path = SPEC_FILE_BY_TYPE.get(spec_type) + try: + ref = self.resolve_spec_ref(spec_type, item_id) + except Exception: + return + if isinstance(preferred_path, str) and ref.get("path") != preferred_path: + return + refs.append(ref) + seen.add(key) + trace.append( + { + "spec_type": spec_type, + "id": item_id, + "selected_from": source, + "selection_mode": selection_mode, + "path": str(ref.get("path") or ""), + "line_range": str(ref.get("line_range") or ""), + } + ) + + for candidate in self._roadmap_bootstrap_candidates(): + maybe_type = candidate.get("type_hint") if isinstance(candidate.get("type_hint"), str) else None + item_id = candidate.get("item_id") + source = candidate.get("source") + candidate_mode = candidate.get("candidate_mode") + if not isinstance(item_id, str) or not item_id: + continue + source_label = source if isinstance(source, str) and source else "roadmap" + selection_mode = candidate_mode if isinstance(candidate_mode, str) and candidate_mode else "tokenized" + if isinstance(maybe_type, str): + _try_add(maybe_type, item_id, source_label, selection_mode) + else: + for spec_type in ("fr", "api", "nfr", "inv", "fixture"): + _try_add(spec_type, item_id, source_label, "tokenized") + if len(refs) >= 8: + break + + if not refs and self.allow_authority_fallback: + for spec_type in ("fr", "api", "nfr", "inv", "fixture"): + candidate = self._first_available_ref_for_type(spec_type) + if isinstance(candidate, dict): + candidate_type = candidate.get("type") + candidate_id = candidate.get("id") + if isinstance(candidate_type, str) and isinstance(candidate_id, str): + key = (candidate_type, candidate_id) + else: + key = None + if key is not None and key not in seen: + refs.append(candidate) + seen.add(key) + trace.append( + { + "spec_type": candidate_type, + "id": candidate_id, + "selected_from": f"authority:{SPEC_FILE_BY_TYPE.get(spec_type, '')}", + "selection_mode": "authority_fallback", + "path": str(candidate.get("path") or ""), + "line_range": str(candidate.get("line_range") or ""), + } + ) + if refs: + break + self._bootstrap_selection_trace = trace + return refs + + def bootstrap_selection_trace(self) -> List[dict]: + return [dict(item) for item in self._bootstrap_selection_trace if isinstance(item, dict)] + + def required_spec_refs(self, phase: str, milestone_payload: Optional[dict]) -> List[dict]: + refs: List[dict] = [] + seen: set[Tuple[str, str]] = set() + self._bootstrap_selection_trace = [] + checklist = [] + if isinstance(milestone_payload, dict): + plan = milestone_payload.get("plan", {}) + if isinstance(plan, dict): + spec_alignment = plan.get("spec_alignment", {}) + if isinstance(spec_alignment, dict): + checklist = spec_alignment.get("checklist", []) + if isinstance(checklist, list): + for item in checklist: + if not isinstance(item, dict): + continue + ref = item.get("spec_ref") + if not isinstance(ref, dict): + continue + rtype = ref.get("type") + rid = ref.get("id") + if isinstance(rtype, str) and isinstance(rid, str): + key = (rtype, rid) + if key in seen: + continue + try: + resolved = self.resolve_spec_ref(rtype, rid) + except Exception: + continue + refs.append(resolved) + seen.add(key) + if not refs and phase == "16a": + refs = self._bootstrap_required_spec_refs() + return refs + + def _pattern_roots(self, patterns: List[str]) -> List[str]: + roots: List[str] = [] + for pattern in patterns: + if not isinstance(pattern, str) or not pattern: + continue + root = pattern.split("*", 1)[0].rstrip("/") + if not root: + continue + if root not in roots: + roots.append(root) + return roots + + def context_pack( + self, + phase: str, + milestone_payload: Optional[dict], + target_file_patterns: List[str], + test_commands: Optional[List[Any]] = None, + ) -> dict: + seed_files = self.seed_files_ordered(phase) + refs = self.required_spec_refs(phase, milestone_payload) + read_paths: List[str] = [] + # Phase-differentiated read paths (C-1 finding): + # - .trinity/logging is never needed by children (capture policy is parent-owned) + # - .trinity/runtime/tools is excluded from utility agents (they use inline protocol) + phase_runtime_paths: List[str] = [".trinity/runtime/spawns"] + if phase != "utility": + phase_runtime_paths.append(".trinity/runtime/tools") + for rel in seed_files + CORE_AUTHORITY_FILES + [ + self.seed_manifest_rel, + _rel(self.repo_root, self.milestone_path), + _rel(self.repo_root, self.anchor_path), + ] + phase_runtime_paths: + if rel not in read_paths: + read_paths.append(rel) + for root in self._pattern_roots(target_file_patterns): + if root not in read_paths: + read_paths.append(root) + + write_paths = [ + ".trinity/runtime/workspace", + "spec/impl_context", + "spec/16_impl_context.json", + ] + for root in self._pattern_roots(target_file_patterns): + if root not in write_paths: + write_paths.append(root) + docs = self.docs_policy() + for root in self._pattern_roots(docs.get("doc_paths", [])): + if root not in write_paths: + write_paths.append(root) + + payload: dict = { + "protocol_version": PROTO_VER, + "phase": phase, + "step_id": self.step_id, + "seed_manifest_path": self.seed_manifest_rel, + "seed_files_ordered": seed_files, + "required_spec_refs": refs, + "artifact_refs": { + "milestone_context_path": _rel(self.repo_root, self.milestone_path), + "anchor_path": _rel(self.repo_root, self.anchor_path), + "workspace_refs": [f".trinity/workspace/{self.step_id}/"], + }, + "allowed_read_paths": read_paths, + "allowed_write_paths": write_paths, + "target_file_patterns": target_file_patterns, + "docs_policy": docs, + } + bootstrap_trace = self.bootstrap_selection_trace() + if phase == "16a" and bootstrap_trace: + payload["bootstrap_ref_trace"] = bootstrap_trace + if phase in {"16b", "16c"}: + tcmds = test_commands or [] + payload["test_contract"] = { + "test_commands": tcmds, + "success_markers": ["PASSED", "passed", "OK", "SUCCESS", "✓", "0 failures", "0 failed", "0 errors"], + } + return payload + + +class TrinityRuntime: + def __init__( + self, + repo_root: str, + config: TrinityConfig, + *, + step_id: Optional[str], + resume: bool = False, + answers: Optional[List[str]] = None, + resume_run_id: Optional[str] = None, + ) -> None: + self.repo_root = os.path.abspath(repo_root) + self.config = config + self.step_id = step_id + self.resume = resume + self.resume_answers = [a.strip() for a in (answers or []) if isinstance(a, str) and a.strip()] + self.resume_run_id = resume_run_id.strip() if isinstance(resume_run_id, str) and resume_run_id.strip() else None + self.run_id = f"run-{uuid.uuid4().hex[:12]}" + self.root_agent_id = f"orchestrator-{uuid.uuid4().hex[:12]}" + self.retry_caps = { + "planner": int(self.config.retry_cap_planner), + "builder": int(self.config.retry_cap_builder), + "verifier": int(self.config.retry_cap_verifier), + "milestone": int(self.config.retry_cap_milestone), + } + mode = (self.config.execution_mode or "").strip().lower() + if mode not in {"llm", "deterministic"}: + raise RuntimeError("runtime.execution_mode must be one of: llm, deterministic") + self.execution_mode = mode + self._llm_client_instance: Optional[OpenAICompatibleClient] = None + self._runtime_schema_registry = SchemaRegistry(self.repo_root) + store = {uri: Resource.from_contents(schema) for uri, schema in self._runtime_schema_registry.store.items()} + self._runtime_schema_refs = Registry().with_resources(store.items()) + self._runtime_validator_cache: Dict[str, Draft202012Validator] = {} + + def _llm_client(self) -> OpenAICompatibleClient: + if self._llm_client_instance is None: + self._llm_client_instance = OpenAICompatibleClient( + api_base=self.config.llm_api_base, + model=self.config.llm_model, + timeout_seconds=max(1, int(self.config.llm_timeout)), + api_key_env=self.config.llm_api_key_env, + temperature=self.config.llm_temperature, + top_p=self.config.llm_top_p, + max_tokens=max(128, int(self.config.llm_max_tokens)), + ) + return self._llm_client_instance + + def _runtime_schema_validator(self, schema_uri: str) -> Draft202012Validator: + cached = self._runtime_validator_cache.get(schema_uri) + if cached is not None: + return cached + schema = self._runtime_schema_registry.load(schema_uri) + validator = Draft202012Validator( + schema, + registry=self._runtime_schema_refs, + format_checker=Draft202012Validator.FORMAT_CHECKER, + ) + self._runtime_validator_cache[schema_uri] = validator + return validator + + def _validate_payload_against_schema(self, schema_uri: str, payload: Any) -> List[str]: + validator = self._runtime_schema_validator(schema_uri) + errors = sorted(validator.iter_errors(payload), key=lambda e: list(e.path)) + rendered: List[str] = [] + for err in errors: + path = "/".join(str(p) for p in err.path) + rendered.append(f"{path}: {err.message}" if path else str(err.message)) + return rendered + + def _log_utility_schema_validation( + self, + *, + logger: SessionLogger, + utility_role: str, + utility_child_id: str, + parent_child_id: str, + loop_id: str, + turn: int, + schema_label: str, + passed: bool, + errors: Optional[List[str]] = None, + ) -> None: + details = "" + if errors: + details = ": " + "; ".join(errors[:3]) + logger.append( + "VALIDATION", + role=utility_role, + phase_id="utility", + loop_id=f"{loop_id}-utility-{turn}", + agent_id=utility_child_id, + parent_id=parent_child_id, + summary=f"{utility_role} {schema_label} schema {'pass' if passed else 'fail'}{details}", + prompt_template_id=_prompt_path_for("utility", utility_role), + step_id=self.step_id, + content_extra=self._phase_validation_content( + passed=passed, + schema_status="pass" if passed else "fail", + deep_status="pass" if passed else "fail", + governance_status="n/a", + ), + ) + + def _load_prompt_text(self, phase: str, *, role: Optional[str] = None) -> str: + rel = _prompt_path_for(phase, role) + path = os.path.join(self.repo_root, rel) + if not os.path.exists(path): + raise RuntimeError(f"Prompt source missing for phase {phase}: {rel}") + with open(path, "r", encoding="utf-8") as f: + return f.read() + + def _normalize_llm_task_result( + self, + *, + action_obj: dict, + child_id: str, + role: str, + phase: str, + milestone_path: str, + ) -> dict: + raw = action_obj.get("task_result", {}) + result = raw if isinstance(raw, dict) else {} + status = result.get("status") + if status not in {"success", "blocked", "failed", "questions"}: + status = "failed" + + normalized = { + "protocol_version": PROTO_VER, + "child_id": child_id, + "role": role, + "phase": phase, + "step_id": self.step_id, + "status": status, + "summary": str(result.get("summary") or action_obj.get("summary") or f"{role} {status}"), + "artifacts": result.get("artifacts") if isinstance(result.get("artifacts"), list) else [], + } + milestone_rel = _rel(self.repo_root, milestone_path) + if status == "success" and phase in {"16a", "16b", "16c"} and not normalized["artifacts"]: + normalized["artifacts"] = [milestone_rel] + + findings = result.get("findings") + if status in {"blocked", "failed"}: + if isinstance(findings, list) and findings: + normalized["findings"] = findings + else: + normalized["findings"] = [ + self._finding( + f"llm-{phase}-missing-findings", + "gap", + "blocking", + "LLM returned blocked/failed status without findings payload.", + ) + ] + + if status == "questions": + questions = result.get("questions") + if isinstance(questions, list) and questions: + normalized["questions"] = [str(q) for q in questions if isinstance(q, str) and q.strip()] + if not normalized.get("questions"): + normalized["questions"] = ["Clarification required by LLM phase output."] + normalized["artifacts"] = [] + + return normalized + + def _validate_loop_checkpoint(self, action_obj: dict, phase: str) -> Optional[str]: + if phase not in {"16a", "16b", "16c"}: + return None + checkpoint = action_obj.get("loop_checkpoint") + if not isinstance(checkpoint, dict): + return ( + "final_result must include loop_checkpoint object with draft/review/refine evidence " + "for phases 16a/16b/16c." + ) + missing: List[str] = [] + for stage in ("draft", "review", "refine"): + if not _loop_evidence_present(checkpoint.get(stage)): + missing.append(stage) + if missing: + return ( + "loop_checkpoint is missing required evidence for: " + + ", ".join(missing) + + ". Provide concrete Draft/Review/Refine evidence." + ) + return None + + def _validate_utility_loop_checkpoint(self, action_obj: dict, utility_role: str) -> Optional[str]: + if utility_role not in {"Researcher", "Summarizer", "Auditor"}: + return None + checkpoint = action_obj.get("loop_checkpoint") + if not isinstance(checkpoint, dict): + return ( + "utility final_result must include loop_checkpoint object with draft/review/refine evidence " + f"for utility role '{utility_role}'." + ) + missing: List[str] = [] + for stage in ("draft", "review", "refine"): + if not _loop_evidence_present(checkpoint.get(stage)): + missing.append(stage) + if missing: + return ( + "utility loop_checkpoint is missing required evidence for: " + + ", ".join(missing) + + ". Provide concrete Draft/Review/Refine evidence." + ) + return None + + def _run_utility_role( + self, + *, + llm_client: OpenAICompatibleClient, + logger: SessionLogger, + parent_child_id: str, + parent_phase: str, + utility_role: str, + utility_call: dict, + context_pack: dict, + milestone_path: str, + loop_id: str, + ) -> dict: + if utility_role not in UTILITY_PROMPT_MAP: + return { + "status": "blocked", + "summary": f"Unsupported utility role '{utility_role}'", + "findings": [self._finding("utility-role-unsupported", "policy", "blocking", f"Unsupported utility role '{utility_role}'.")], + "utility_payload": {}, + } + + utility_child_id = f"{utility_role.lower()}-{uuid.uuid4().hex[:8]}" + utility_tools = ToolExecutor( + self.repo_root, + logger, + self.run_id, + agent_id=utility_child_id, + phase="utility", + step_id=self.step_id, + allowed_read_paths=context_pack.get("allowed_read_paths", []), + allowed_write_paths=context_pack.get("allowed_write_paths", []), + target_file_patterns=context_pack.get("target_file_patterns", []), + docs_policy=context_pack.get("docs_policy", {}), + protected_write_paths=( + ([context_pack.get("seed_manifest_path")] if isinstance(context_pack.get("seed_manifest_path"), str) else []) + + ( + context_pack.get("seed_files_ordered", []) + if isinstance(context_pack.get("seed_files_ordered"), list) + else [] + ) + ), + enable_checkpoints=False, + ) + + objective = utility_call.get("objective") + if not isinstance(objective, str) or not objective.strip(): + objective = utility_call.get("summary") + if not isinstance(objective, str) or not objective.strip(): + objective = f"Utility support for {parent_phase}" + utility_input = utility_call.get("input") if isinstance(utility_call.get("input"), dict) else {} + utility_prompt = self._load_prompt_text("utility", role=utility_role) + utility_protocol = ( + "Return JSON only. Supported actions:\n" + "1) Tool call:\n" + '{"action":"tool_call","summary":"...","tool_call":{"tool_name":"","args":{...}}}\n' + "2) Final result:\n" + '{"action":"final_result","summary":"...","loop_checkpoint":{"draft":"...","review":"...","refine":"..."},"utility_result":{"status":"ready|questions|blocked","summary":"...","open_questions":[...],"errors":[...],"findings":[...]}}\n' + "Rules:\n" + "- Never fabricate evidence.\n" + "- Stay within context_pack constraints.\n" + "- Keep output assumption-free." + ) + utility_payload = { + "protocol_version": PROTO_VER, + "role": utility_role, + "phase": "utility", + "step_id": self.step_id, + "objective": objective, + "input": utility_input, + "context_pack": context_pack, + "milestone_artifact_ref": _rel(self.repo_root, milestone_path), + "tool_catalog_ref": ".trinity/runtime/tools/catalog.json", + } + messages: List[dict] = [ + {"role": "system", "content": utility_prompt}, + {"role": "system", "content": utility_protocol}, + {"role": "user", "content": json.dumps(utility_payload, ensure_ascii=False)}, + ] + max_turns = max(1, min(4, int(self.config.max_child_turns))) + for turn in range(1, max_turns + 1): + request_material = json.dumps({"messages": messages}, ensure_ascii=False) + try: + llm_text, llm_usage = llm_client.chat(messages) + except Exception as e: # noqa: BLE001 + return { + "status": "blocked", + "summary": f"{utility_role} blocked: LLM request failed", + "findings": [self._finding("utility-llm-request-failed", "policy", "blocking", str(e))], + "utility_payload": {}, + } + + metadata_extra = None + if isinstance(llm_usage, dict): + prompt_toks = llm_usage.get("prompt_tokens") + completion_toks = llm_usage.get("completion_tokens") + total_toks = llm_usage.get("total_tokens") + if all(isinstance(v, int) for v in (prompt_toks, completion_toks, total_toks)): + metadata_extra = { + "token_usage": { + "prompt": int(prompt_toks), + "completion": int(completion_toks), + "total": int(total_toks), + } + } + + logger.append( + "MESSAGE", + role=utility_role, + phase_id="utility", + loop_id=f"{loop_id}-utility-{turn}", + agent_id=utility_child_id, + parent_id=parent_child_id, + summary=f"{utility_role} utility turn {turn}", + prompt_template_id=_prompt_path_for("utility", utility_role), + step_id=self.step_id, + metadata_extra=metadata_extra, + prompt_material_override=request_material, + response_material_override=llm_text, + ) + + action_obj = _extract_json_object(llm_text) + if not isinstance(action_obj, dict): + return { + "status": "blocked", + "summary": f"{utility_role} blocked: invalid utility JSON action", + "findings": [ + self._finding( + "utility-invalid-json", + "policy", + "blocking", + "Utility role output could not be parsed as a JSON action object.", + ) + ], + "utility_payload": {}, + } + action = action_obj.get("action") + if action == "tool_call": + tool_call = action_obj.get("tool_call", {}) + tool_name = tool_call.get("tool_name") if isinstance(tool_call, dict) else None + args = tool_call.get("args") if isinstance(tool_call, dict) else None + if not isinstance(tool_name, str) or not isinstance(args, dict): + return { + "status": "blocked", + "summary": f"{utility_role} blocked: malformed utility tool_call", + "findings": [ + self._finding( + "utility-malformed-tool-call", + "policy", + "blocking", + "Utility role returned tool_call without valid tool_name/args.", + ) + ], + "utility_payload": {}, + } + tool_result = utility_tools.call( + tool_name, + args, + role=utility_role, + parent_id=parent_child_id, + loop_id=f"{loop_id}-utility-{turn}", + ) + messages.append({"role": "assistant", "content": json.dumps(action_obj, ensure_ascii=False)}) + messages.append( + { + "role": "user", + "content": json.dumps( + { + "tool_result": tool_result, + "instruction": "Continue with next action or final_result.", + }, + ensure_ascii=False, + ), + } + ) + continue + + if action == "final_result": + checkpoint_error = self._validate_utility_loop_checkpoint(action_obj, utility_role) + if checkpoint_error: + return { + "status": "blocked", + "summary": f"{utility_role} blocked: missing utility loop checkpoint evidence", + "findings": [ + self._finding( + "utility-loop-checkpoint-missing", + "policy", + "blocking", + checkpoint_error, + ) + ], + "utility_payload": {}, + } + utility_result = action_obj.get("utility_result", {}) + if not isinstance(utility_result, dict): + self._log_utility_schema_validation( + logger=logger, + utility_role=utility_role, + utility_child_id=utility_child_id, + parent_child_id=parent_child_id, + loop_id=loop_id, + turn=turn, + schema_label="utility_result", + passed=False, + errors=["utility_result must be an object"], + ) + return { + "status": "blocked", + "summary": f"{utility_role} blocked: malformed utility_result payload", + "findings": [ + self._finding( + "utility-malformed-result", + "policy", + "blocking", + "Utility role returned final_result without utility_result object.", + ) + ], + "utility_payload": {}, + } + schema_errors = self._validate_payload_against_schema(UTILITY_RESULT_SCHEMA_URI, utility_result) + self._log_utility_schema_validation( + logger=logger, + utility_role=utility_role, + utility_child_id=utility_child_id, + parent_child_id=parent_child_id, + loop_id=loop_id, + turn=turn, + schema_label="utility_result", + passed=not schema_errors, + errors=schema_errors, + ) + if schema_errors: + return { + "status": "blocked", + "summary": f"{utility_role} blocked: utility_result schema validation failed", + "findings": [ + self._finding( + "utility-result-schema-invalid", + "policy", + "blocking", + "; ".join(schema_errors[:3]), + ) + ], + "utility_payload": utility_result, + } + utility_status = utility_result.get("status") + mapped_status = ( + "success" + if utility_status == "ready" + else "questions" + if utility_status == "questions" + else "blocked" + ) + findings = utility_result.get("findings") + questions = utility_result.get("open_questions") + return { + "status": mapped_status, + "summary": str(utility_result.get("summary") or action_obj.get("summary") or f"{utility_role} completed"), + "findings": findings if isinstance(findings, list) else [], + "questions": [str(q) for q in questions if isinstance(q, str)] if isinstance(questions, list) else [], + "utility_payload": utility_result, + } + + return { + "status": "blocked", + "summary": f"{utility_role} blocked: unsupported utility action", + "findings": [ + self._finding( + "utility-unsupported-action", + "policy", + "blocking", + f"Unsupported utility action '{action}'.", + ) + ], + "utility_payload": {}, + } + + return { + "status": "blocked", + "summary": f"{utility_role} blocked: max utility turns exceeded", + "findings": [ + self._finding( + "utility-turn-cap", + "policy", + "blocking", + "Utility role exceeded max turns without final_result.", + ) + ], + "utility_payload": {}, + } + + def _llm_phase_handler(self, milestone_path: str, logger: SessionLogger, *, phase: str, role: str): + def _handler(task_input: dict, context_pack: dict, child_id: str) -> dict: + llm_client = self._llm_client() + tools = ToolExecutor( + self.repo_root, + logger, + self.run_id, + agent_id=child_id, + phase=phase, + step_id=self.step_id, + allowed_read_paths=context_pack.get("allowed_read_paths", []), + allowed_write_paths=context_pack.get("allowed_write_paths", []), + target_file_patterns=context_pack.get("target_file_patterns", []), + docs_policy=context_pack.get("docs_policy", {}), + protected_write_paths=( + ([context_pack.get("seed_manifest_path")] if isinstance(context_pack.get("seed_manifest_path"), str) else []) + + ( + context_pack.get("seed_files_ordered", []) + if isinstance(context_pack.get("seed_files_ordered"), list) + else [] + ) + ), + enable_checkpoints=self.config.checkpoint_commits, + ) + + prompt_text = self._load_prompt_text(phase, role=role) + protocol_instructions = ( + "Return JSON only. Supported actions:\n" + "1) Tool call:\n" + '{"action":"tool_call","summary":"...","tool_call":{"tool_name":"","args":{...}}}\n' + "2) Final result:\n" + '{"action":"final_result","summary":"...","loop_checkpoint":{"draft":"...","review":"...","refine":"..."},"task_result":{"status":"success|blocked|failed|questions","summary":"...","artifacts":[...],"findings":[...],"questions":[...]}}\n' + "3) Utility role invocation (16a/16b/16c only):\n" + '{"action":"utility_call","summary":"...","utility_call":{"role":"Researcher|ToolUser|Summarizer|Auditor","objective":"...","input":{...}}}\n' + "Rules:\n" + "- Never fabricate files or test outcomes.\n" + "- Use only listed tools when needed.\n" + "- For phases 16a/16b/16c, final_result MUST include loop_checkpoint with draft/review/refine evidence.\n" + "- For success in phases 16a/16b/16c, include milestone artifact in artifacts.\n" + "- Keep all outputs assumption-free and schema-compliant." + ) + + initial_payload = { + "task_input": task_input, + "context_pack": context_pack, + "milestone_artifact_ref": _rel(self.repo_root, milestone_path), + "tool_catalog_ref": ".trinity/runtime/tools/catalog.json", + "result_schema_hint": "schema/trinity/task_result.schema.json", + } + messages: List[dict] = [ + {"role": "system", "content": prompt_text}, + {"role": "system", "content": protocol_instructions}, + {"role": "user", "content": json.dumps(initial_payload, ensure_ascii=False)}, + ] + + max_turns = max(1, int(self.config.max_child_turns)) + for turn in range(1, max_turns + 1): + request_material = json.dumps({"messages": messages}, ensure_ascii=False) + try: + llm_text, llm_usage = llm_client.chat(messages) + except Exception as e: # noqa: BLE001 + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: LLM request failed", + artifacts=[], + findings=[self._finding(f"{phase}-llm-request-failed", "policy", "blocking", str(e))], + ) + + metadata_extra = None + if isinstance(llm_usage, dict): + prompt_toks = llm_usage.get("prompt_tokens") + completion_toks = llm_usage.get("completion_tokens") + total_toks = llm_usage.get("total_tokens") + if all(isinstance(v, int) for v in (prompt_toks, completion_toks, total_toks)): + metadata_extra = { + "token_usage": { + "prompt": int(prompt_toks), + "completion": int(completion_toks), + "total": int(total_toks), + } + } + + logger.append( + "MESSAGE", + role=role, + phase_id=phase, + loop_id=f"l2-{turn}", + agent_id=child_id, + parent_id=self.root_agent_id, + summary=f"{role} LLM turn {turn}", + prompt_template_id=_prompt_path_for(phase, role), + step_id=self.step_id, + metadata_extra=metadata_extra, + prompt_material_override=request_material, + response_material_override=llm_text, + ) + + action_obj = _extract_json_object(llm_text) + if not isinstance(action_obj, dict): + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: invalid LLM JSON action", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-invalid-json", + "policy", + "blocking", + "LLM output could not be parsed as a JSON action object.", + ) + ], + ) + + action = action_obj.get("action") + if action == "tool_call": + tool_call = action_obj.get("tool_call", {}) + tool_name = tool_call.get("tool_name") if isinstance(tool_call, dict) else None + args = tool_call.get("args") if isinstance(tool_call, dict) else None + if not isinstance(tool_name, str) or not isinstance(args, dict): + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: malformed tool_call action", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-malformed-tool-call", + "policy", + "blocking", + "LLM returned tool_call action without valid tool_name/args.", + ) + ], + ) + tool_result = tools.call( + tool_name, + args, + role=role, + parent_id=self.root_agent_id, + loop_id=f"l2-{turn}", + ) + messages.append({"role": "assistant", "content": json.dumps(action_obj, ensure_ascii=False)}) + messages.append( + { + "role": "user", + "content": json.dumps( + { + "tool_result": tool_result, + "instruction": "Continue with next action or final_result.", + }, + ensure_ascii=False, + ), + } + ) + continue + + if action == "utility_call": + if phase not in {"16a", "16b", "16c"}: + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: utility_call unsupported in phase {phase}", + artifacts=[], + findings=[ + self._finding( + f"{phase}-utility-call-unsupported", + "policy", + "blocking", + "utility_call is only supported for 16a/16b/16c phase handlers.", + ) + ], + ) + utility_call = action_obj.get("utility_call", {}) + utility_call_schema_errors = self._validate_payload_against_schema( + UTILITY_CALL_SCHEMA_URI, + utility_call, + ) + logger.append( + "VALIDATION", + role=role, + phase_id=phase, + loop_id=f"l{turn}", + agent_id=child_id, + parent_id=self.root_agent_id, + summary=( + "utility_call schema pass" + if not utility_call_schema_errors + else "utility_call schema fail: " + "; ".join(utility_call_schema_errors[:3]) + ), + prompt_template_id=_prompt_path_for(phase, role), + step_id=self.step_id, + content_extra=self._phase_validation_content( + passed=not utility_call_schema_errors, + schema_status="pass" if not utility_call_schema_errors else "fail", + deep_status="pass" if not utility_call_schema_errors else "fail", + governance_status="n/a", + ), + ) + if utility_call_schema_errors: + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: utility_call schema validation failed", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-invalid-utility-call-schema", + "policy", + "blocking", + "; ".join(utility_call_schema_errors[:3]), + ) + ], + ) + utility_role = utility_call.get("role") if isinstance(utility_call, dict) else None + if not isinstance(utility_role, str) or utility_role not in UTILITY_PROMPT_MAP: + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: malformed utility_call action", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-malformed-utility-call", + "policy", + "blocking", + "LLM returned utility_call without valid role.", + ) + ], + ) + utility_result = self._run_utility_role( + llm_client=llm_client, + logger=logger, + parent_child_id=child_id, + parent_phase=phase, + utility_role=utility_role, + utility_call=utility_call, + context_pack=context_pack, + milestone_path=milestone_path, + loop_id=f"l2-{turn}", + ) + messages.append({"role": "assistant", "content": json.dumps(action_obj, ensure_ascii=False)}) + messages.append( + { + "role": "user", + "content": json.dumps( + { + "utility_result": utility_result, + "instruction": "Incorporate utility result and continue with next action or final_result.", + }, + ensure_ascii=False, + ), + } + ) + continue + + if action == "final_result": + checkpoint_error = self._validate_loop_checkpoint(action_obj, phase) + if checkpoint_error: + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: missing loop checkpoint evidence", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-loop-checkpoint-missing", + "policy", + "blocking", + checkpoint_error, + ) + ], + ) + return self._normalize_llm_task_result( + action_obj=action_obj, + child_id=child_id, + role=role, + phase=phase, + milestone_path=milestone_path, + ) + + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: unsupported LLM action", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-unsupported-action", + "policy", + "blocking", + f"Unsupported LLM action '{action}'.", + ) + ], + ) + + return self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: max LLM turns exceeded", + artifacts=[], + findings=[ + self._finding( + f"{phase}-llm-turn-cap", + "policy", + "blocking", + f"Phase exceeded max child turns ({max_turns}) without final_result.", + ) + ], + ) + + return _handler + + def _roadmap_path(self) -> str: + return os.path.join(self.repo_root, "spec", "14_roadmap.json") + + def _load_roadmap(self) -> dict: + path = self._roadmap_path() + if not os.path.exists(path): + raise RuntimeError(f"Roadmap missing: {_rel(self.repo_root, path)}") + payload = _read_json(path) + if payload.get("$schema") != "https://specdev.local/schema/14_roadmap.schema.json": + raise RuntimeError(f"Roadmap schema mismatch in {_rel(self.repo_root, path)}") + return payload + + def _session_state_candidates(self) -> List[str]: + runtime_dir = os.path.join(self.repo_root, ".trinity", "runtime") + if not os.path.isdir(runtime_dir): + return [] + candidates: List[str] = [] + for name in os.listdir(runtime_dir): + if not (name.startswith("session_state_") and name.endswith(".json")): + continue + candidates.append(os.path.join(runtime_dir, name)) + candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True) + return candidates + + def _select_resume_state_path(self) -> str: + candidates = self._session_state_candidates() + if not candidates: + raise RuntimeError("Resume requested but no session_state_*.json found under .trinity/runtime") + + matched: List[Tuple[str, dict]] = [] + for path in candidates: + try: + payload = _read_json(path) + except Exception: + continue + if not isinstance(payload, dict): + continue + run_id = payload.get("run_id") + step_id = payload.get("step_id") + if self.resume_run_id and run_id != self.resume_run_id: + continue + if self.step_id and step_id != self.step_id: + continue + matched.append((path, payload)) + + if not matched: + details: List[str] = [] + if self.resume_run_id: + details.append(f"run_id='{self.resume_run_id}'") + if self.step_id: + details.append(f"step_id='{self.step_id}'") + suffix = " for " + ", ".join(details) if details else "" + raise RuntimeError(f"Resume requested but no matching session state found{suffix}") + + if self.resume_run_id: + run_matches = [path for path, _ in matched] + if len(run_matches) > 1: + raise RuntimeError( + "Resume run_id matched multiple session states; clean stale states or provide --step-id for disambiguation" + ) + return run_matches[0] + + if self.step_id: + step_matches = [path for path, _ in matched] + if len(step_matches) > 1: + raise RuntimeError( + f"Resume for step_id '{self.step_id}' is ambiguous ({len(step_matches)} states). " + "Provide --resume-run-id to select a specific run." + ) + return step_matches[0] + + if len(matched) > 1: + raise RuntimeError( + "Resume is ambiguous because multiple session states exist. Provide --step-id or --resume-run-id." + ) + return matched[0][0] + + def _load_resume_state(self) -> Optional[dict]: + if not self.resume: + return None + state_path = self._select_resume_state_path() + errs = validate_runtime_file(self.repo_root, state_path, "session_state") + if errs: + raise RuntimeError("Resume state is invalid: " + "; ".join(errs)) + state = _read_json(state_path) + state_step_id = state.get("step_id") + if not isinstance(state_step_id, str) or not state_step_id: + raise RuntimeError("Resume state missing step_id") + if self.step_id and self.step_id != state_step_id: + raise RuntimeError( + f"Resume state step_id '{state_step_id}' does not match requested step_id '{self.step_id}'" + ) + self.step_id = state_step_id + return state + + def _pick_step_id(self, roadmap: dict) -> str: + milestones = roadmap.get("milestones", []) + if not isinstance(milestones, list) or not milestones: + raise RuntimeError("Roadmap has no milestones") + milestone_ids = [m.get("milestone_id") for m in milestones if isinstance(m, dict)] + if self.step_id: + if self.step_id not in milestone_ids: + raise RuntimeError(f"step_id '{self.step_id}' not found in roadmap milestones") + return self.step_id + + status_by_id = { + m.get("milestone_id"): m.get("status", "pending") + for m in milestones + if isinstance(m, dict) and isinstance(m.get("milestone_id"), str) + } + dependencies = roadmap.get("dependencies", []) + dep_ids = [] + if isinstance(dependencies, list): + dep_ids = [ + d.get("id") + for d in dependencies + if isinstance(d, dict) and d.get("type") == "milestone" and isinstance(d.get("id"), str) + ] + for m in milestones: + if not isinstance(m, dict): + continue + mid = m.get("milestone_id") + status = m.get("status", "pending") + if not isinstance(mid, str): + continue + if status == "done": + continue + deps_ok = all(status_by_id.get(dep) == "done" for dep in dep_ids if dep in status_by_id) + if deps_ok: + return mid + raise RuntimeError("No eligible milestone found with satisfied dependencies") + + def _ensure_branch(self, tools: ToolExecutor) -> None: + branch = f"trinity/{self.step_id}" + tools.call("checkpoint_branch", {"branch_name": branch}, loop_id="l1") + + def _phase_validation_content( + self, + *, + task_input_ref: Optional[str] = None, + task_result_ref: Optional[str] = None, + passed: Optional[bool] = None, + schema_status: Optional[str] = None, + deep_status: Optional[str] = None, + governance_status: str = "n/a", + seed_lint_status: str = "n/a", + docs_lint_status: str = "n/a", + ) -> dict: + status = "pass" if bool(passed) else "fail" + schema = schema_status if schema_status in {"pass", "fail", "n/a"} else status + deep = deep_status if deep_status in {"pass", "fail", "n/a"} else status + content = { + "validation": { + "schema": schema, + "deep_validator": deep, + "governance": governance_status if governance_status in {"pass", "fail", "n/a"} else "n/a", + "seed_lint": seed_lint_status if seed_lint_status in {"pass", "fail", "n/a"} else "n/a", + "docs_lint": docs_lint_status if docs_lint_status in {"pass", "fail", "n/a"} else "n/a", + } + } + if task_input_ref: + content["task_input_artifact_ref"] = task_input_ref + if task_result_ref: + content["task_result_artifact_ref"] = task_result_ref + return content + + def _phase_attempt_cap(self, phase: str) -> int: + if phase == "16a": + return self.retry_caps["planner"] + if phase == "16b": + return self.retry_caps["builder"] + if phase == "16c": + return self.retry_caps["verifier"] + return self.retry_caps["milestone"] + + def _normalize_checklist_scope(self, checklist_scope: Optional[List[str]]) -> List[str]: + if not isinstance(checklist_scope, list): + return [] + return sorted({x for x in checklist_scope if isinstance(x, str) and x.strip()}) + + def _next_spawn_attempt(self, spawn_log_path: str, role: str, phase: str, checklist_scope: Optional[List[str]]) -> int: + if not os.path.exists(spawn_log_path): + return 1 + try: + payload = _read_json(spawn_log_path) + except Exception: + return 1 + if payload.get("run_id") != self.run_id: + return 1 + entries = payload.get("entries", []) + if not isinstance(entries, list): + return 1 + purpose = f"{role} {phase}" + count = 0 + for entry in entries: + if not isinstance(entry, dict): + continue + if entry.get("step_id") != self.step_id: + continue + if entry.get("phase") != phase: + continue + if entry.get("purpose") != purpose: + continue + entry_scope = self._normalize_checklist_scope(entry.get("checklist_scope")) + if entry_scope != self._normalize_checklist_scope(checklist_scope): + continue + count += 1 + return count + 1 + + def _child_timeout_for_phase(self, phase: str) -> int: + override = self.config.child_timeout_by_phase.get(phase) + if isinstance(override, int) and override >= 0: + return override + return max(0, int(self.config.child_timeout_seconds)) + + def _run_child_subprocess( + self, + *, + phase: str, + role: str, + child_id: str, + milestone_path: str, + task_input_path: str, + context_pack_path: str, + task_result_path: str, + session_log_path: str, + ) -> Optional[str]: + module_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + cmd = [ + sys.executable, + "-m", + "specdev_tools.cli", + "trinity-child", + "--repo-root", + self.repo_root, + "--step-id", + self.step_id, + "--phase", + phase, + "--role", + role, + "--child-id", + child_id, + "--milestone-path", + _rel(self.repo_root, milestone_path), + "--task-input", + _rel(self.repo_root, task_input_path), + "--context-pack", + _rel(self.repo_root, context_pack_path), + "--task-result", + _rel(self.repo_root, task_result_path), + "--session-log", + _rel(self.repo_root, session_log_path), + "--run-id", + self.run_id, + "--parent-id", + self.root_agent_id, + "--mode", + self.execution_mode, + ] + env = os.environ.copy() + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + module_root + if not existing_pythonpath + else module_root + os.pathsep + existing_pythonpath + ) + env["SPECDEV_SKIP_VENV_CHECK"] = "1" + timeout_seconds = self._child_timeout_for_phase(phase) + timeout_arg = timeout_seconds if timeout_seconds > 0 else None + try: + proc = subprocess.run( + cmd, + cwd=self.repo_root, + capture_output=True, + text=True, + check=False, + env=env, + timeout=timeout_arg, + ) + except subprocess.TimeoutExpired: + if timeout_seconds > 0: + return f"child subprocess timeout after {timeout_seconds}s in phase {phase}" + return f"child subprocess timeout in phase {phase}" + if proc.returncode == 0: + return None + stderr = (proc.stderr or "").strip() + stdout = (proc.stdout or "").strip() + msg = stderr or stdout or f"child subprocess failed with exit {proc.returncode}" + return msg + + def _ingest_phase_result( + self, + *, + logger: SessionLogger, + role: str, + phase: str, + child_id: str, + attempt: int, + milestone_path: str, + session_state_path: str, + spawn_log_path: str, + scratchpad_path: str, + retry_counters: dict, + canonical_task_input_ref: str, + canonical_task_result_ref: str, + checklist_scope: List[str], + task_result: dict, + task_result_errors: List[str], + ) -> Tuple[dict, List[str]]: + logger.append( + "VALIDATION", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"{role} task_result validation {'pass' if not task_result_errors else 'fail'}", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra=self._phase_validation_content(task_result_ref=canonical_task_result_ref, passed=not task_result_errors), + ) + logger.append( + "TERMINATE", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"Terminate {role} with {task_result.get('status')}", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra={"task_result_artifact_ref": canonical_task_result_ref}, + ) + self._update_spawn_log( + spawn_log_path=spawn_log_path, + child_id=child_id, + role=role, + phase=phase, + attempt=attempt, + checklist_scope=checklist_scope, + task_input_ref=canonical_task_input_ref, + task_result_ref=canonical_task_result_ref, + task_result=task_result, + ) + + status = task_result.get("status") + if status == "questions": + questions_raw = task_result.get("questions", []) + questions = [q for q in questions_raw if isinstance(q, str) and q.strip()] if isinstance(questions_raw, list) else [] + self._write_session_state( + session_state_path=session_state_path, + active_phase=phase, + status="awaiting_input", + pending_child_id=child_id, + pending_spawn_ref=canonical_task_input_ref, + pending_questions=questions, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters=retry_counters, + ) + self._write_scratchpad( + scratchpad_path, + phase=phase, + next_action_ref=f"phase:{phase}:awaiting_input", + state_summary=f"{phase} awaiting clarification", + checklist_scope=self._checklist_ids(milestone_path), + validation_gate={ + "schema": "pass" if not task_result_errors else "fail", + "deep_validator": "pass" if not task_result_errors else "fail", + "governance": "n/a", + }, + ) + return task_result, task_result_errors + + self._write_session_state( + session_state_path=session_state_path, + active_phase=phase, + status="resuming", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters=retry_counters, + ) + self._write_scratchpad( + scratchpad_path, + phase=phase, + next_action_ref=f"phase:{phase}:ingest", + state_summary=f"{phase} completed with status {task_result.get('status')}", + checklist_scope=self._checklist_ids(milestone_path), + validation_gate={ + "schema": "pass" if not task_result_errors else "fail", + "deep_validator": "pass" if not task_result_errors else "fail", + "governance": "n/a", + }, + ) + return task_result, task_result_errors + + def _spawn_phase( + self, + *, + logger: SessionLogger, + resolver: ContextResolver, + phase: str, + role: str, + phase_label: str, + milestone_path: str, + anchor_path: str, + session_state_path: str, + spawn_log_path: str, + scratchpad_path: str, + retry_counters: dict, + ) -> Tuple[dict, List[str]]: + checklist_scope = self._checklist_ids(milestone_path) + attempt = self._next_spawn_attempt(spawn_log_path, role, phase, checklist_scope) + if attempt > self._phase_attempt_cap(phase): + finding = self._finding( + f"{phase}-spawn-loop-cap", + "policy", + "blocking", + f"Spawn loop cap exceeded for {role} {phase} (attempt {attempt}).", + ) + return ( + self._task_result( + child_id=f"{role.lower()}-blocked", + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: spawn loop cap exceeded", + artifacts=[], + findings=[finding], + ), + [finding["description"]], + ) + + milestone_payload = _read_json(milestone_path) if os.path.exists(milestone_path) else None + phase_target_patterns = self._target_patterns_for_phase(phase, milestone_payload, milestone_path, anchor_path) + phase_test_commands = self._phase_test_commands(milestone_payload) + context_pack = resolver.context_pack(phase, milestone_payload, phase_target_patterns, phase_test_commands) + bootstrap_trace = context_pack.get("bootstrap_ref_trace") if isinstance(context_pack.get("bootstrap_ref_trace"), list) else None + if phase == "16a" and not isinstance(milestone_payload, dict): + try: + milestone_payload = self._ensure_milestone_artifact(milestone_path, context_pack=context_pack) + except Exception as e: # noqa: BLE001 + finding = self._finding( + "planner-bootstrap-failed", + "policy", + "blocking", + str(e), + ) + blocked_result = self._task_result( + child_id=f"{role.lower()}-blocked", + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: milestone bootstrap failed", + artifacts=[], + findings=[finding], + ) + return blocked_result, [finding["description"]] + phase_target_patterns = self._target_patterns_for_phase(phase, milestone_payload, milestone_path, anchor_path) + phase_test_commands = self._phase_test_commands(milestone_payload) + context_pack = resolver.context_pack(phase, milestone_payload, phase_target_patterns, phase_test_commands) + if bootstrap_trace and not isinstance(context_pack.get("bootstrap_ref_trace"), list): + context_pack["bootstrap_ref_trace"] = bootstrap_trace + checklist_scope = self._checklist_ids(milestone_path) + + child_id = f"{role.lower()}-{uuid.uuid4().hex[:8]}" + spawn_dir = os.path.join(self.repo_root, ".trinity", "runtime", "spawns", child_id) + os.makedirs(spawn_dir, exist_ok=True) + + # S-1: Write-ahead marker for transaction atomicity. + # If runtime crashes between here and marker removal, recovery + # detects the orphaned .wal file and cleans up partial artifacts. + wal_dir = os.path.join(self.repo_root, ".trinity", "runtime", "wal") + os.makedirs(wal_dir, exist_ok=True) + wal_path = os.path.join(wal_dir, f"{child_id}.wal") + with open(wal_path, "w") as wf: + wf.write(child_id) + + context_pack_path = os.path.join(spawn_dir, "context_pack.json") + _write_json_atomic(context_pack_path, context_pack) + context_errors = validate_runtime_file(self.repo_root, context_pack_path, "context_pack") + + task_input = { + "protocol_version": PROTO_VER, + "child_id": child_id, + "parent_id": self.root_agent_id, + "role": role, + "phase": phase, + "step_id": self.step_id, + "task_description": f"{phase_label} execution for milestone {self.step_id}", + "expected_output_schema": RUNTIME_SCHEMA_BY_TYPE["task_result"], + "context_pack_ref": _rel(self.repo_root, context_pack_path), + "target_files": self._target_files_for_task_input( + milestone_payload=milestone_payload, + milestone_path=milestone_path, + anchor_path=anchor_path, + phase_target_patterns=phase_target_patterns, + ), + "spec_refs": context_pack.get("required_spec_refs", []), + "role_metadata": { + "prompt_source": _prompt_path_for(phase, role), + "persona_goal": f"Execute {phase} for milestone {self.step_id}", + "stop_conditions": ["questions", "blocked", "success"], + }, + } + task_input_path = os.path.join(spawn_dir, "task_input.json") + _write_json_atomic(task_input_path, task_input) + task_input_errors = validate_runtime_file(self.repo_root, task_input_path, "task_input") + + task_result_path = os.path.join(spawn_dir, "task_result.json") + canonical_task_input_ref = f".trinity/runtime/spawns/{child_id}/task_input.json" + canonical_task_result_ref = f".trinity/runtime/spawns/{child_id}/task_result.json" + validation_ok = not context_errors and not task_input_errors + if not validation_ok: + logger.append( + "VALIDATION", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"{role} task_input validation fail", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra=self._phase_validation_content(task_input_ref=canonical_task_input_ref, passed=False), + ) + blocked_result = self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: invalid spawn artifacts", + artifacts=[], + findings=[ + self._finding( + f"{phase}-spawn-validation-failed", + "policy", + "blocking", + "; ".join(context_errors + task_input_errors), + ) + ], + ) + self._update_spawn_log( + spawn_log_path=spawn_log_path, + child_id=child_id, + role=role, + phase=phase, + attempt=attempt, + checklist_scope=checklist_scope, + task_input_ref=canonical_task_input_ref, + task_result_ref=None, + task_result=blocked_result, + ) + self._write_session_state( + session_state_path=session_state_path, + active_phase=phase, + status="blocked", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters=retry_counters, + ) + return blocked_result, context_errors + task_input_errors + + self._write_session_state( + session_state_path=session_state_path, + active_phase=phase, + status="waiting_child", + pending_child_id=child_id, + pending_spawn_ref=canonical_task_input_ref, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters=retry_counters, + ) + logger.append( + "SPAWN", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"Spawn {role}", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra={"task_input_artifact_ref": canonical_task_input_ref}, + ) + logger.append( + "VALIDATION", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"{role} task_input validation pass", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra=self._phase_validation_content(task_input_ref=canonical_task_input_ref, passed=True), + ) + + child_error = self._run_child_subprocess( + phase=phase, + role=role, + child_id=child_id, + milestone_path=milestone_path, + task_input_path=task_input_path, + context_pack_path=context_pack_path, + task_result_path=task_result_path, + session_log_path=logger.path, + ) + if child_error and not os.path.exists(task_result_path): + blocked_result = self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: child execution failed", + artifacts=[], + findings=[self._finding(f"{phase}-child-process-failed", "policy", "blocking", child_error)], + ) + _write_json_atomic(task_result_path, blocked_result) + + if not os.path.exists(task_result_path): + missing_result = self._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: child did not produce task_result", + artifacts=[], + findings=[self._finding(f"{phase}-missing-task-result", "policy", "blocking", "Child produced no task_result.json")], + ) + _write_json_atomic(task_result_path, missing_result) + + logger.sync_from_disk() + task_result_errors = validate_runtime_file(self.repo_root, task_result_path, "task_result") + task_result = _read_json(task_result_path) + ingest_result = self._ingest_phase_result( + logger=logger, + role=role, + phase=phase, + child_id=child_id, + attempt=attempt, + milestone_path=milestone_path, + session_state_path=session_state_path, + spawn_log_path=spawn_log_path, + scratchpad_path=scratchpad_path, + retry_counters=retry_counters, + canonical_task_input_ref=canonical_task_input_ref, + canonical_task_result_ref=canonical_task_result_ref, + checklist_scope=checklist_scope, + task_result=task_result, + task_result_errors=task_result_errors, + ) + # S-1: Remove WAL marker after successful transaction closure. + if os.path.exists(wal_path): + os.remove(wal_path) + return ingest_result + + def _resume_pending_spawn( + self, + *, + logger: SessionLogger, + session_state_path: str, + spawn_log_path: str, + scratchpad_path: str, + milestone_path: str, + retry_counters: dict, + resume_state: dict, + ) -> Optional[dict]: + pending_child_id = resume_state.get("pending_child_id") + pending_spawn_ref = resume_state.get("pending_spawn_ref") + if not (isinstance(pending_child_id, str) and pending_child_id and isinstance(pending_spawn_ref, str) and pending_spawn_ref): + return None + task_input_path = os.path.join(self.repo_root, pending_spawn_ref) + if not os.path.exists(task_input_path): + return None + task_input_errors = validate_runtime_file(self.repo_root, task_input_path, "task_input") + if task_input_errors: + return { + "status": "blocked", + "phase": "16a", + "task_result": self._task_result( + child_id=pending_child_id, + role="Planner", + phase="16a", + status="blocked", + summary="Pending spawn invalid", + artifacts=[], + findings=[self._finding("resume-invalid-task-input", "policy", "blocking", "; ".join(task_input_errors))], + ), + "errors": task_input_errors, + } + task_input = _read_json(task_input_path) + phase = task_input.get("phase") + role = task_input.get("role") + if phase not in {"16a", "16b", "16c", "utility"} or not isinstance(role, str): + return None + context_pack_ref = task_input.get("context_pack_ref") + if not isinstance(context_pack_ref, str) or not context_pack_ref: + return None + context_pack_path = os.path.join(self.repo_root, context_pack_ref) + if not os.path.exists(context_pack_path): + return None + spawn_dir = os.path.dirname(task_input_path) + task_result_path = os.path.join(spawn_dir, "task_result.json") + canonical_task_input_ref = pending_spawn_ref + canonical_task_result_ref = _rel(self.repo_root, task_result_path) + resume_status = resume_state.get("status") + if resume_status == "awaiting_input": + if not self.resume_answers: + pending_questions = resume_state.get("pending_questions") + questions = ( + [q for q in pending_questions if isinstance(q, str) and q.strip()] + if isinstance(pending_questions, list) + else [] + ) + if not questions and os.path.exists(task_result_path): + existing = _read_json(task_result_path) + q = existing.get("questions") + if isinstance(q, list): + questions = [x for x in q if isinstance(x, str) and x.strip()] + return { + "status": "questions", + "phase": phase, + "questions": questions, + "pending_spawn_ref": pending_spawn_ref, + } + clarifications = "\n".join(f"- {a}" for a in self.resume_answers) + existing_desc = task_input.get("task_description", "") + task_input["task_description"] = f"{existing_desc}\n\nUser clarifications:\n{clarifications}".strip() + _write_json_atomic(task_input_path, task_input) + refreshed_errors = validate_runtime_file(self.repo_root, task_input_path, "task_input") + if refreshed_errors: + return { + "status": "blocked", + "phase": phase, + "task_result": self._task_result( + child_id=pending_child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: invalid resumed task_input", + artifacts=[], + findings=[self._finding("resume-invalid-updated-task-input", "policy", "blocking", "; ".join(refreshed_errors))], + ), + "errors": refreshed_errors, + } + if os.path.exists(task_result_path): + os.remove(task_result_path) + self._write_session_state( + session_state_path=session_state_path, + active_phase=phase, + status="waiting_child", + pending_child_id=pending_child_id, + pending_spawn_ref=pending_spawn_ref, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters=retry_counters, + ) + + checklist_scope = self._checklist_ids(milestone_path) + attempt = self._next_spawn_attempt(spawn_log_path, role, phase, checklist_scope) + if attempt > self._phase_attempt_cap(phase): + return { + "status": "blocked", + "phase": phase, + "task_result": self._task_result( + child_id=pending_child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: spawn loop cap exceeded", + artifacts=[], + findings=[self._finding(f"{phase}-resume-loop-cap", "policy", "blocking", "Resume spawn loop cap exceeded.")], + ), + "errors": [f"{phase} resume spawn loop cap exceeded"], + } + + logger.append( + "SPAWN", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"Resume {role}", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra={"task_input_artifact_ref": canonical_task_input_ref}, + ) + logger.append( + "VALIDATION", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=f"{role} task_input validation pass", + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra=self._phase_validation_content(task_input_ref=canonical_task_input_ref, passed=True), + ) + if not os.path.exists(task_result_path): + child_error = self._run_child_subprocess( + phase=phase, + role=role, + child_id=pending_child_id, + milestone_path=milestone_path, + task_input_path=task_input_path, + context_pack_path=context_pack_path, + task_result_path=task_result_path, + session_log_path=logger.path, + ) + if child_error and not os.path.exists(task_result_path): + blocked = self._task_result( + child_id=pending_child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: child execution failed on resume", + artifacts=[], + findings=[self._finding(f"{phase}-resume-child-failed", "policy", "blocking", child_error)], + ) + _write_json_atomic(task_result_path, blocked) + logger.sync_from_disk() + if not os.path.exists(task_result_path): + return { + "status": "blocked", + "phase": phase, + "task_result": self._task_result( + child_id=pending_child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: missing task_result after resume", + artifacts=[], + findings=[self._finding(f"{phase}-resume-missing-task-result", "policy", "blocking", "Resume produced no task_result")], + ), + "errors": [f"{phase} missing task_result after resume"], + } + task_result_errors = validate_runtime_file(self.repo_root, task_result_path, "task_result") + task_result = _read_json(task_result_path) + ingested_result, ingested_errors = self._ingest_phase_result( + logger=logger, + role=role, + phase=phase, + child_id=pending_child_id, + attempt=attempt, + milestone_path=milestone_path, + session_state_path=session_state_path, + spawn_log_path=spawn_log_path, + scratchpad_path=scratchpad_path, + retry_counters=retry_counters, + canonical_task_input_ref=canonical_task_input_ref, + canonical_task_result_ref=canonical_task_result_ref, + checklist_scope=checklist_scope, + task_result=task_result, + task_result_errors=task_result_errors, + ) + return {"status": "replayed", "phase": phase, "task_result": ingested_result, "errors": ingested_errors} + + def _checklist_ids(self, milestone_path: str) -> List[str]: + if not os.path.exists(milestone_path): + return [] + payload = _read_json(milestone_path) + checklist = payload.get("plan", {}).get("spec_alignment", {}).get("checklist", []) + if not isinstance(checklist, list): + return [] + ids: List[str] = [] + for item in checklist: + if isinstance(item, dict) and isinstance(item.get("id"), str): + ids.append(item["id"]) + return ids + + def _target_patterns_for_phase( + self, + phase: str, + milestone_payload: Optional[dict], + milestone_path: str, + anchor_path: str, + ) -> List[str]: + milestone_rel = _rel(self.repo_root, milestone_path) + anchor_rel = _rel(self.repo_root, anchor_path) + patterns: List[str] = [milestone_rel, anchor_rel, "spec/impl_context/*.json", "spec/16_impl_context.json"] + if isinstance(milestone_payload, dict): + plan = milestone_payload.get("plan", {}) + summary = plan.get("summary", {}) if isinstance(plan, dict) else {} + targets = summary.get("target_file_patterns", []) if isinstance(summary, dict) else [] + if isinstance(targets, list): + for p in targets: + if isinstance(p, str) and p and p not in patterns: + patterns.append(p) + if phase == "16a" and "docs/**" not in patterns: + patterns.append("docs/**") + return patterns + + def _phase_test_commands(self, milestone_payload: Optional[dict]) -> List[Any]: + if not isinstance(milestone_payload, dict): + return [] + plan = milestone_payload.get("plan", {}) + if not isinstance(plan, dict): + return [] + req = plan.get("review_requirements", {}) + if not isinstance(req, dict): + return [] + cmds = req.get("test_commands", []) + if isinstance(cmds, list) and cmds: + return cmds + return [] + + def _target_files_for_task_input( + self, + *, + milestone_payload: Optional[dict], + milestone_path: str, + anchor_path: str, + phase_target_patterns: List[str], + ) -> List[str]: + files: List[str] = [ + _rel(self.repo_root, milestone_path), + _rel(self.repo_root, anchor_path), + ] + for candidate in phase_target_patterns: + if isinstance(candidate, str) and candidate and not _looks_like_pattern(candidate): + files.append(candidate) + if isinstance(milestone_payload, dict): + plan = milestone_payload.get("plan", {}) + if isinstance(plan, dict): + docs_impact = plan.get("docs_impact", {}) + if isinstance(docs_impact, dict): + docs_touched = docs_impact.get("docs_touched", []) + if isinstance(docs_touched, list): + for doc_path in docs_touched: + if isinstance(doc_path, str) and doc_path and not _looks_like_pattern(doc_path): + files.append(doc_path) + checklist = plan.get("spec_alignment", {}).get("checklist", []) if isinstance(plan.get("spec_alignment"), dict) else [] + if isinstance(checklist, list): + for item in checklist: + if not isinstance(item, dict): + continue + impl = item.get("implementation", {}) + if isinstance(impl, dict): + touched = impl.get("files_touched", []) + if isinstance(touched, list): + for touched_path in touched: + if isinstance(touched_path, str) and touched_path and not _looks_like_pattern(touched_path): + files.append(touched_path) + actions = impl.get("actions", []) + if isinstance(actions, list): + for action in actions: + if not isinstance(action, dict): + continue + target = action.get("target") + if isinstance(target, str) and target and not _looks_like_pattern(target): + files.append(target) + return list(dict.fromkeys(files)) + + def _bootstrap_seed_refs(self) -> List[dict]: + manifest_path = os.path.join(self.repo_root, "spec", "common", "seed_manifest.json") + if not os.path.exists(manifest_path): + return [] + try: + manifest = _read_json(manifest_path) + except Exception: + return [] + step_requirements = manifest.get("step_requirements", {}) + required_seed_ids: List[str] = [] + if isinstance(step_requirements, dict): + for phase in ("16a", "16"): + ids = step_requirements.get(phase, []) + if isinstance(ids, list): + for seed_id in ids: + if isinstance(seed_id, str) and seed_id and seed_id not in required_seed_ids: + required_seed_ids.append(seed_id) + return [{"seed_id": seed_id} for seed_id in required_seed_ids] + + def _bootstrap_milestone_artifact(self, milestone_path: str, required_spec_refs: List[dict]) -> dict: + refs: List[dict] = [] + for ref in required_spec_refs: + if not isinstance(ref, dict): + continue + rtype = ref.get("type") + rid = ref.get("id") + line_range = ref.get("line_range") + commit_hash = ref.get("commit_hash") + if all(isinstance(v, str) and v for v in (rtype, rid, line_range, commit_hash)): + refs.append( + { + "type": rtype, + "id": rid, + "line_range": line_range, + "commit_hash": commit_hash, + } + ) + if not refs: + raise RuntimeError("Planner bootstrap failed: no grounded spec refs available for milestone initialization") + + bootstrap_cmd = 'python3 -c "print(\'SUCCESS TRINITY_BOOTSTRAP\')"' + checklist: List[dict] = [] + for idx, ref in enumerate(refs[:5], start=1): + checklist.append( + { + "id": f"CHK_BOOTSTRAP_{idx:03d}", + "spec_ref": ref, + "description": ( + f"Bootstrap planner contract for grounded reference {ref['type']}:{ref['id']} " + "using deterministic evidence collection." + ), + "type": "validation", + "layer": "integration", + "checklist_status": "active", + "linked_test_expectation": bootstrap_cmd, + "nfr_refs": ["nfr-bootstrap-safety"], + "fixture_ref": "fixture-bootstrap-smoke", + "implementation": { + "status": "pending", + "files_touched": [], + "actions": [ + { + "type": "run_command", + "description": "Execute deterministic bootstrap smoke command to initialize evidence contract.", + "command": bootstrap_cmd, + }, + { + "type": "manual_verification", + "description": "Confirm success marker from bootstrap smoke command output.", + }, + ], + }, + } + ) + owner = "api" + roadmap_path = os.path.join(self.repo_root, "spec", "14_roadmap.json") + if os.path.exists(roadmap_path): + try: + roadmap = _read_json(roadmap_path) + if isinstance(roadmap.get("owner"), str) and roadmap["owner"]: + owner = roadmap["owner"] + except Exception: + owner = "api" + + payload = { + "$schema": "https://specdev.local/schema/16_impl_context.schema.json", + "id": self.step_id, + "owner": owner, + "created_at": _utc_now(), + "seed_refs": self._bootstrap_seed_refs() or [{"seed_id": "seed-overview"}], + "plan": { + "status": "active", + "summary": { + "functional_summary": ( + f"Bootstrap implementation context for milestone '{self.step_id}' generated from governed seeds and roadmap." + ), + "scope_in": [f"bootstrap:{self.step_id}"], + "scope_out": [], + "target_file_patterns": [ + _rel(self.repo_root, milestone_path), + "spec/16_impl_context.json", + "README.md", + ], + }, + "docs_impact": { + "status": "required", + "rationale": "Bootstrap writes Step 16 planning artifacts and must keep documentation traceability updated.", + "docs_touched": ["README.md"], + }, + "spec_alignment": {"checklist": checklist}, + "review_requirements": { + "guidelines": "Bootstrap milestone context until planner emits refined plan.", + "test_commands": [bootstrap_cmd], + }, + }, + } + _write_json_atomic(milestone_path, payload) + errs = validate_file(self.repo_root, milestone_path) + if errs: + raise RuntimeError("Planner bootstrap produced invalid milestone artifact: " + "; ".join(errs)) + return payload + + def _ensure_milestone_artifact(self, milestone_path: str, *, context_pack: Optional[dict] = None) -> dict: + if not os.path.exists(milestone_path): + refs = context_pack.get("required_spec_refs", []) if isinstance(context_pack, dict) else [] + if not isinstance(refs, list): + refs = [] + self._bootstrap_milestone_artifact(milestone_path, refs) + payload = _read_json(milestone_path) + if payload.get("$schema") != "https://specdev.local/schema/16_impl_context.schema.json": + raise RuntimeError(f"{_rel(self.repo_root, milestone_path)} is not a Step 16 milestone artifact") + return payload + + def _regenerate_anchor( + self, + milestone_path: str, + anchor_path: str, + *, + logger: Optional[SessionLogger] = None, + phase: str = "16a", + ) -> dict: + existing_extensions = {} + if os.path.exists(anchor_path): + try: + anchor = _read_json(anchor_path) + ext = anchor.get("extensions") + if isinstance(ext, dict): + existing_extensions = ext + except Exception: + existing_extensions = {} + + roadmap_active_ids: Set[str] = set() + roadmap_path = self._roadmap_path() + if os.path.exists(roadmap_path): + try: + roadmap = _read_json(roadmap_path) + milestones = roadmap.get("milestones", []) + if isinstance(milestones, list): + for milestone in milestones: + if not isinstance(milestone, dict): + continue + milestone_id = milestone.get("milestone_id") + status = milestone.get("status", "pending") + if isinstance(milestone_id, str) and status != "done": + roadmap_active_ids.add(milestone_id) + except Exception: + roadmap_active_ids = set() + + impl_dir = os.path.join(self.repo_root, "spec", "impl_context") + candidate_paths: List[str] = [] + if os.path.isdir(impl_dir): + for entry in sorted(os.listdir(impl_dir)): + if not entry.endswith(".json"): + continue + candidate_paths.append(os.path.join(impl_dir, entry)) + if os.path.exists(milestone_path) and milestone_path not in candidate_paths: + candidate_paths.append(milestone_path) + + milestone_payloads: List[dict] = [] + for candidate in candidate_paths: + if not os.path.exists(candidate): + continue + try: + payload = _read_json(candidate) + except Exception: + continue + if payload.get("$schema") != "https://specdev.local/schema/16_impl_context.schema.json": + continue + milestone_id = payload.get("id") + if roadmap_active_ids and isinstance(milestone_id, str) and milestone_id not in roadmap_active_ids: + continue + milestone_payloads.append(payload) + if not milestone_payloads: + return { + "active_contexts": 0, + "merged_seed_refs": 0, + "union_scope_in_count": 0, + "union_scope_out_count": 0, + "union_target_patterns_count": 0, + "union_docs_touched_count": 0, + "union_test_commands_count": 0, + "checklist_items_count": 0, + "checklist_conflicts_count": 0, + "checklist_conflict_ids": [], + } + + primary = None + for payload in milestone_payloads: + if payload.get("id") == self.step_id: + primary = payload + break + if primary is None: + primary = milestone_payloads[0] + + merged = copy.deepcopy(primary) + merged["id"] = "step-impl-anchor" + merged["created_at"] = _utc_now() + + seed_ids: List[str] = [] + union_scope_in: Set[str] = set() + union_scope_out: Set[str] = set() + union_targets: Set[str] = set() + union_docs_touched: Set[str] = set() + union_test_commands: List[Any] = [] + checklist_by_id: Dict[str, dict] = {} + checklist_conflict_ids: Set[str] = set() + + for payload in milestone_payloads: + for seed_ref in payload.get("seed_refs", []): + if isinstance(seed_ref, dict): + seed_id = seed_ref.get("seed_id") + if isinstance(seed_id, str) and seed_id and seed_id not in seed_ids: + seed_ids.append(seed_id) + plan = payload.get("plan", {}) + if not isinstance(plan, dict): + continue + summary = plan.get("summary", {}) + if isinstance(summary, dict): + scope_in = summary.get("scope_in", []) + if isinstance(scope_in, list): + for item in scope_in: + if isinstance(item, str) and item: + union_scope_in.add(item) + scope_out = summary.get("scope_out", []) + if isinstance(scope_out, list): + for item in scope_out: + if isinstance(item, str) and item: + union_scope_out.add(item) + targets = summary.get("target_file_patterns", []) + if isinstance(targets, list): + for item in targets: + if isinstance(item, str) and item: + union_targets.add(item) + docs_impact = plan.get("docs_impact", {}) + if isinstance(docs_impact, dict): + docs_touched = docs_impact.get("docs_touched", []) + if isinstance(docs_touched, list): + for item in docs_touched: + if isinstance(item, str) and item: + union_docs_touched.add(item) + review_requirements = plan.get("review_requirements", {}) + if isinstance(review_requirements, dict): + test_commands = review_requirements.get("test_commands", []) + if isinstance(test_commands, list): + for command in test_commands: + if command not in union_test_commands: + union_test_commands.append(command) + checklist = plan.get("spec_alignment", {}).get("checklist", []) if isinstance(plan.get("spec_alignment"), dict) else [] + if isinstance(checklist, list): + for item in checklist: + if not isinstance(item, dict): + continue + item_id = item.get("id") + if isinstance(item_id, str) and item_id: + candidate = copy.deepcopy(item) + existing = checklist_by_id.get(item_id) + if isinstance(existing, dict): + existing_canon = json.dumps(existing, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + candidate_canon = json.dumps(candidate, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + if existing_canon != candidate_canon: + checklist_conflict_ids.add(item_id) + checklist_by_id[item_id] = candidate + + if seed_ids: + merged["seed_refs"] = [{"seed_id": seed_id} for seed_id in seed_ids] + plan = merged.get("plan", {}) if isinstance(merged.get("plan"), dict) else {} + summary = plan.get("summary", {}) if isinstance(plan.get("summary"), dict) else {} + summary["functional_summary"] = summary.get("functional_summary") or ( + f"Anchor union of active milestone implementation contexts for run step '{self.step_id}'." + ) + summary["scope_in"] = sorted(union_scope_in) if union_scope_in else summary.get("scope_in", []) + summary["scope_out"] = sorted(union_scope_out) if union_scope_out else summary.get("scope_out", []) + summary["target_file_patterns"] = sorted(union_targets) if union_targets else summary.get("target_file_patterns", []) + plan["summary"] = summary + + docs_impact = plan.get("docs_impact", {}) if isinstance(plan.get("docs_impact"), dict) else {} + docs_impact["status"] = "required" + docs_impact["rationale"] = docs_impact.get("rationale") or "Union anchor reflects active milestone context updates." + docs_impact["docs_touched"] = sorted(union_docs_touched) if union_docs_touched else docs_impact.get("docs_touched", ["README.md"]) + if not docs_impact.get("docs_touched"): + docs_impact["docs_touched"] = ["README.md"] + plan["docs_impact"] = docs_impact + + spec_alignment = plan.get("spec_alignment", {}) if isinstance(plan.get("spec_alignment"), dict) else {} + checklist_union = [checklist_by_id[key] for key in sorted(checklist_by_id.keys())] + if checklist_union: + spec_alignment["checklist"] = checklist_union + plan["spec_alignment"] = spec_alignment + + review_requirements = plan.get("review_requirements", {}) if isinstance(plan.get("review_requirements"), dict) else {} + if union_test_commands: + review_requirements["test_commands"] = union_test_commands + review_requirements["guidelines"] = review_requirements.get("guidelines") or "Union of active milestone review requirements." + plan["review_requirements"] = review_requirements + + checklist_for_coverage = spec_alignment.get("checklist", []) + if isinstance(checklist_for_coverage, list): + total = len(checklist_for_coverage) + verified = 0 + deferred = 0 + for item in checklist_for_coverage: + if not isinstance(item, dict): + continue + impl = item.get("implementation", {}) + if isinstance(impl, dict) and impl.get("status") == "verified": + verified += 1 + elif item.get("checklist_status") == "deferred" or (isinstance(impl, dict) and impl.get("status") == "deferred"): + deferred += 1 + pending = max(0, total - verified - deferred) + plan["coverage_status"] = { + "total": total, + "verified": verified, + "deferred": deferred, + "pending": pending, + } + + merged["plan"] = plan + union_metrics = { + "active_contexts": len(milestone_payloads), + "merged_seed_refs": len(seed_ids), + "union_scope_in_count": len(union_scope_in), + "union_scope_out_count": len(union_scope_out), + "union_target_patterns_count": len(union_targets), + "union_docs_touched_count": len(union_docs_touched), + "union_test_commands_count": len(union_test_commands), + "checklist_items_count": len(checklist_by_id), + "checklist_conflicts_count": len(checklist_conflict_ids), + "checklist_conflict_ids": sorted(checklist_conflict_ids), + } + if checklist_conflict_ids and not self.config.allow_anchor_conflicts: + conflict_list = ", ".join(sorted(checklist_conflict_ids)) + raise RuntimeError( + "Anchor union detected conflicting checklist definitions: " + + conflict_list + + ". Set runtime.allow_anchor_conflicts=true to override." + ) + merged_extensions = merged.get("extensions", {}) if isinstance(merged.get("extensions"), dict) else {} + if existing_extensions: + merged_extensions.update(existing_extensions) + if merged_extensions: + merged["extensions"] = merged_extensions + _write_json_atomic(anchor_path, merged) + errs = validate_file(self.repo_root, anchor_path) + if errs: + raise RuntimeError("; ".join(errs)) + if logger is not None: + logger.append( + "VALIDATION", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=( + f"anchor union merged {union_metrics['active_contexts']} context(s), " + f"{union_metrics['checklist_items_count']} checklist item(s), " + f"{union_metrics['checklist_conflicts_count']} conflict(s)" + ), + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + content_extra=self._phase_validation_content( + passed=True, + schema_status="pass", + deep_status="pass", + governance_status="n/a", + ), + metadata_extra={"anchor_union_metrics": union_metrics}, + ) + return union_metrics + + def _update_roadmap_status(self, roadmap_path: str, status: str) -> None: + roadmap = _read_json(roadmap_path) + milestones = roadmap.get("milestones", []) + if not isinstance(milestones, list): + return + for milestone in milestones: + if isinstance(milestone, dict) and milestone.get("milestone_id") == self.step_id: + milestone["status"] = status + _write_json_atomic(roadmap_path, roadmap) + + def _write_session_state( + self, + *, + session_state_path: str, + active_phase: str, + status: str, + pending_child_id: Optional[str], + pending_spawn_ref: Optional[str], + pending_questions: Optional[List[str]] = None, + session_log_ref: Optional[str] = None, + spawn_log_ref: Optional[str], + scratchpad_ref: Optional[str], + retry_counters: dict, + ) -> None: + if session_log_ref is None and os.path.exists(session_state_path): + try: + existing = _read_json(session_state_path) + existing_ref = existing.get("session_log_ref") + if isinstance(existing_ref, str) and existing_ref: + session_log_ref = existing_ref + except Exception: + session_log_ref = None + payload = { + "protocol_version": PROTO_VER, + "run_id": self.run_id, + "parent_id": self.root_agent_id, + "active_phase": active_phase, + "step_id": self.step_id, + "status": status, + "pending_child_id": pending_child_id, + "pending_spawn_ref": pending_spawn_ref, + "pending_questions": pending_questions, + "session_log_ref": session_log_ref, + "spawn_log_ref": spawn_log_ref, + "scratchpad_ref": scratchpad_ref, + "last_event_id": None, + "retry_counters": retry_counters, + "updated_at": _utc_now(), + } + _write_json_atomic(session_state_path, payload) + errs = validate_runtime_file(self.repo_root, session_state_path, "session_state") + if errs: + raise RuntimeError("; ".join(errs)) + + def _update_spawn_log( + self, + *, + spawn_log_path: str, + child_id: str, + role: str, + phase: str, + attempt: int, + checklist_scope: List[str], + task_input_ref: Optional[str], + task_result_ref: Optional[str], + task_result: dict, + ) -> None: + if os.path.exists(spawn_log_path): + payload = _read_json(spawn_log_path) + else: + payload = {"protocol_version": PROTO_VER, "run_id": self.run_id, "entries": []} + if payload.get("run_id") != self.run_id: + payload = {"protocol_version": PROTO_VER, "run_id": self.run_id, "entries": []} + entries = payload.get("entries", []) + if not isinstance(entries, list): + entries = [] + now = _utc_now() + entries.append( + { + "spawn_id": f"spawn-{uuid.uuid4().hex[:12]}", + "parent_id": self.root_agent_id, + "child_id": child_id, + "purpose": f"{role} {phase}", + "phase": phase, + "step_id": self.step_id, + "attempt": max(1, int(attempt)), + "checklist_scope": self._normalize_checklist_scope(checklist_scope), + "status": ( + "completed" + if task_result.get("status") == "success" + else "blocked" + if task_result.get("status") in {"blocked", "questions"} + else "failed" + ), + "task_input_ref": task_input_ref, + "task_result_ref": task_result_ref, + "created_at": now, + "updated_at": now, + } + ) + payload["entries"] = entries + _write_json_atomic(spawn_log_path, payload) + errs = validate_runtime_file(self.repo_root, spawn_log_path, "spawn_log") + if errs: + raise RuntimeError("; ".join(errs)) + + def _write_scratchpad( + self, + scratchpad_path: str, + *, + phase: str, + next_action_ref: str, + state_summary: str, + checklist_scope: List[str], + validation_gate: Optional[dict] = None, + ) -> None: + now_iso = datetime.now(timezone.utc).isoformat() + gate = validation_gate if isinstance(validation_gate, dict) else {} + payload = { + "phase": phase, + "checklist_scope": checklist_scope, + "last_validation_gate": { + "schema": gate.get("schema") if gate.get("schema") in {"pass", "fail", "n/a"} else "n/a", + "deep_validator": ( + gate.get("deep_validator") if gate.get("deep_validator") in {"pass", "fail", "n/a"} else "n/a" + ), + "governance": gate.get("governance") if gate.get("governance") in {"pass", "fail", "n/a"} else "n/a", + }, + "next_action_ref": next_action_ref, + "state_summary": state_summary, + "variables": {"step_id": self.step_id}, + "milestone_step_id": self.step_id, + "created_at": now_iso, + "updated_at": now_iso, + } + _write_json_atomic(scratchpad_path, payload) + errs = validate_runtime_file(self.repo_root, scratchpad_path, "scratchpad_state") + if errs: + raise RuntimeError("; ".join(errs)) + + def _planner_handler(self, milestone_path: str): + def _handler(task_input: dict, context_pack: dict, child_id: str) -> dict: + try: + milestone = self._ensure_milestone_artifact(milestone_path, context_pack=context_pack) + except Exception as e: # noqa: BLE001 + return self._task_result( + child_id=child_id, + role="Planner", + phase="16a", + status="blocked", + summary="Planner blocked: missing or invalid milestone artifact", + artifacts=[], + findings=[ + self._finding( + "planner-missing-milestone-artifact", + "gap", + "blocking", + str(e), + ) + ], + ) + plan = milestone.get("plan", {}) + if not isinstance(plan, dict): + return self._task_result( + child_id=child_id, + role="Planner", + phase="16a", + status="blocked", + summary="Planner blocked: missing plan section", + artifacts=[], + findings=[self._finding("planner-missing-plan", "gap", "blocking", "Plan section missing in milestone context.")], + ) + checklist = plan.get("spec_alignment", {}).get("checklist", []) + if not isinstance(checklist, list) or not checklist: + return self._task_result( + child_id=child_id, + role="Planner", + phase="16a", + status="blocked", + summary="Planner blocked: checklist is missing", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=[ + self._finding( + "planner-missing-checklist", + "gap", + "blocking", + "plan.spec_alignment.checklist must be present and non-empty; planner will not invent checklist entries.", + ) + ], + ) + _write_json_atomic(milestone_path, milestone) + errs = validate_file(self.repo_root, milestone_path) + if errs: + return self._task_result( + child_id=child_id, + role="Planner", + phase="16a", + status="failed", + summary="Planner failed: milestone validation errors", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=[self._finding("planner-validation-failed", "policy", "blocking", "; ".join(errs))], + ) + return self._task_result( + child_id=child_id, + role="Planner", + phase="16a", + status="success", + summary="Planner produced valid milestone plan artifact", + artifacts=[_rel(self.repo_root, milestone_path)], + ) + + return _handler + + def _builder_handler(self, milestone_path: str, logger: SessionLogger): + def _linked_expectation_commands(item: dict) -> Set[str]: + linked = item.get("linked_test_expectation") + commands: Set[str] = set() + if isinstance(linked, str) and linked.strip(): + commands.add(linked.strip()) + elif isinstance(linked, list): + for entry in linked: + if isinstance(entry, str) and entry.strip(): + commands.add(entry.strip()) + return commands + + def _slug(text: object) -> str: + return re.sub(r"[^a-z0-9]+", "-", str(text).lower()).strip("-") or "item" + + def _handler(task_input: dict, context_pack: dict, child_id: str) -> dict: + milestone = _read_json(milestone_path) + plan = milestone.get("plan", {}) + if not isinstance(plan, dict): + return self._task_result( + child_id=child_id, + role="Builder", + phase="16b", + status="blocked", + summary="Builder blocked: missing plan", + artifacts=[], + findings=[self._finding("builder-missing-plan", "gap", "blocking", "Plan section missing.")], + ) + + checklist = plan.get("spec_alignment", {}).get("checklist", []) + if not isinstance(checklist, list): + return self._task_result( + child_id=child_id, + role="Builder", + phase="16b", + status="blocked", + summary="Builder blocked: missing checklist", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=[ + self._finding( + "builder-missing-checklist", + "gap", + "blocking", + "plan.spec_alignment.checklist must be a list.", + ) + ], + ) + + tools = ToolExecutor( + self.repo_root, + logger, + self.run_id, + agent_id=self.root_agent_id, + phase="16b", + step_id=self.step_id, + allowed_read_paths=context_pack.get("allowed_read_paths", []), + allowed_write_paths=context_pack.get("allowed_write_paths", []), + target_file_patterns=context_pack.get("target_file_patterns", []), + docs_policy=context_pack.get("docs_policy", {}), + protected_write_paths=( + ([context_pack.get("seed_manifest_path")] if isinstance(context_pack.get("seed_manifest_path"), str) else []) + + ( + context_pack.get("seed_files_ordered", []) + if isinstance(context_pack.get("seed_files_ordered"), list) + else [] + ) + ), + enable_checkpoints=self.config.checkpoint_commits, + ) + + execution_results: List[dict] = [] + command_order: List[str] = [] + command_cache: Dict[str, dict] = {} + passed_commands: Set[str] = set() + builder_findings: List[dict] = [] + files_touched: Set[str] = set() + satisfied_checklist_ids: List[str] = [] + + def _run_command(command: str) -> dict: + if command in command_cache: + return command_cache[command] + tool_result = tools.call( + "exec_cmd", + {"command": command, "mode": "standard", "timeout_seconds": 300}, + role="Builder", + parent_id=self.root_agent_id, + loop_id="l3", + ) + status = tool_result.get("status") + exit_code = tool_result.get("exit_code") + stdout_excerpt = tool_result.get("stdout_excerpt") or "" + stderr_excerpt = tool_result.get("stderr_excerpt") or "" + evidence, has_success_marker = _extract_command_excerpt( + stdout_excerpt, + stderr_excerpt, + exit_code if isinstance(exit_code, int) else 1, + ) + combined_output = ((stdout_excerpt or "") + ("\n" + stderr_excerpt if stderr_excerpt else "")).strip() + passed = ( + status == "success" + and isinstance(exit_code, int) + and exit_code == 0 + and has_success_marker + ) + if passed and len(evidence) < 20: + if len(combined_output) >= 20: + evidence = combined_output[:400] + else: + passed = False + evidence = "Blocked: command exited 0 but lacked sufficient verbatim success evidence." + if len(evidence) < 20: + if len(combined_output) >= 20: + evidence = combined_output[:400] + else: + evidence = (evidence + " :: evidence length below contract minimum").ljust(20, ".")[:400] + binding_sha = _sha256_text(evidence) + execution_results.append( + { + "status": ( + "passed" + if passed + else "blocked" + if status == "blocked" + or (status == "success" and isinstance(exit_code, int) and exit_code == 0 and not has_success_marker) + else "partial" + if status == "timeout" + else "failed" + ), + "outcome_description": ( + "Command passed" + if passed + else "Command exited without required success marker" + if (status == "success" and isinstance(exit_code, int) and exit_code == 0 and not has_success_marker) + else "Command did not pass" + ), + "reasoning": "Deterministic command execution in Trinity builder phase.", + "command": command, + "evidence": evidence, + "evidence_ref": f"sha256:{binding_sha}", + "evidence_binding": { + "timestamp": _utc_now(), + "sha256": binding_sha, + "exit_code": int(exit_code) if isinstance(exit_code, int) else 1, + "command": command, + }, + } + ) + if command not in command_order: + command_order.append(command) + if passed: + passed_commands.add(command) + payload = {"passed": passed, "evidence": evidence, "evidence_ref": f"sha256:{binding_sha}"} + command_cache[command] = payload + return payload + + review_req = plan.get("review_requirements", {}) if isinstance(plan.get("review_requirements"), dict) else {} + test_commands_raw = review_req.get("test_commands", []) if isinstance(review_req.get("test_commands"), list) else [] + normalized_cmds = [cmd for cmd in (_normalize_test_command(x) for x in test_commands_raw) if cmd] + if not normalized_cmds: + return self._task_result( + child_id=child_id, + role="Builder", + phase="16b", + status="blocked", + summary="Builder blocked: review_requirements.test_commands missing", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=[ + self._finding( + "builder-missing-test-contract", + "policy", + "blocking", + "plan.review_requirements.test_commands must be provided; runtime will not inject synthetic default commands.", + ) + ], + ) + if isinstance(checklist, list): + for idx, item in enumerate(checklist): + if not isinstance(item, dict): + continue + if item.get("checklist_status", "active") == "deferred": + continue + item_id = item.get("id", f"checklist-{idx + 1}") + impl = item.get("implementation") + if not isinstance(impl, dict): + impl = {"status": "deferred", "actions": []} + item["implementation"] = impl + builder_findings.append( + self._finding( + f"builder-missing-implementation-{_slug(item_id)}", + "gap", + "major", + f"Checklist item '{item_id}' is active but missing implementation actions.", + ) + ) + continue + actions = impl.get("actions") + if not isinstance(actions, list): + actions = [] + if not actions: + impl["status"] = "deferred" + impl["actions"] = [] + builder_findings.append( + self._finding( + f"builder-empty-actions-{_slug(item_id)}", + "gap", + "major", + f"Checklist item '{item_id}' has no implementation.actions to execute.", + ) + ) + continue + + impl["status"] = "in_progress" + item_files_touched: Set[str] = set() + existing_touched = impl.get("files_touched", []) + if isinstance(existing_touched, list): + for touched in existing_touched: + if isinstance(touched, str) and touched: + item_files_touched.add(touched) + item_verified = True + + for action_idx, action in enumerate(actions): + if not isinstance(action, dict): + item_verified = False + continue + action_type = action.get("type") + target = action.get("target") + + if action_type == "run_command": + command = action.get("command") + if not isinstance(command, str) or not command.strip(): + item_verified = False + action["evidence"] = { + "type": "snippet", + "content": "Blocked: run_command action missing command.", + } + builder_findings.append( + self._finding( + f"builder-missing-command-{_slug(item_id)}-{action_idx + 1}", + "gap", + "major", + f"Checklist item '{item_id}' has run_command action without a command.", + ) + ) + continue + result = _run_command(command.strip()) + action["evidence"] = { + "type": "snippet", + "content": result["evidence"], + "evidence_ref": result["evidence_ref"], + } + if not result["passed"]: + item_verified = False + continue + + if action_type == "manual_verification": + linked = _linked_expectation_commands(item) + selected: Optional[dict] = None + for command in linked: + if command in command_cache and command_cache[command]["passed"]: + selected = command_cache[command] + break + if selected is None: + for command in command_order: + maybe = command_cache.get(command) + if maybe and maybe["passed"]: + selected = maybe + break + if selected is None: + item_verified = False + action["evidence"] = { + "type": "snippet", + "content": "Blocked: manual_verification has no passing command evidence.", + } + else: + action["evidence"] = { + "type": "snippet", + "content": selected["evidence"], + "evidence_ref": selected["evidence_ref"], + } + continue + + if action_type == "file_create": + if not isinstance(target, str) or not target.strip(): + item_verified = False + action["evidence"] = { + "type": "snippet", + "content": "Blocked: file_create action missing target.", + } + builder_findings.append( + self._finding( + f"builder-missing-target-{_slug(item_id)}-{action_idx + 1}", + "gap", + "major", + f"Checklist item '{item_id}' has file_create action without target.", + ) + ) + continue + rel_target = target.strip() + abs_target = os.path.join(self.repo_root, rel_target) + if os.path.exists(abs_target): + check = tools.call( + "read_file", + {"path": rel_target, "start_line": 1, "end_line": 1}, + role="Builder", + parent_id=self.root_agent_id, + loop_id="l3", + ) + ok = check.get("status") == "success" + evidence = "file_create target already exists and is readable." + evidence_ref = f"sha256:{_sha256_text(evidence)}" + else: + write = tools.call( + "write_file", + {"path": rel_target, "content": "", "mode": "create_new", "create_parents": True}, + role="Builder", + parent_id=self.root_agent_id, + loop_id="l3", + ) + ok = write.get("status") == "success" + evidence = "Created empty file for deterministic file_create action." + evidence_ref = f"sha256:{_sha256_text(evidence)}" + if ok: + item_files_touched.add(rel_target) + action["evidence"] = {"type": "snippet", "content": evidence, "evidence_ref": evidence_ref} + if not ok: + item_verified = False + continue + + if action_type == "file_edit": + if not isinstance(target, str) or not target.strip(): + item_verified = False + action["evidence"] = { + "type": "snippet", + "content": "Blocked: file_edit action missing target.", + } + builder_findings.append( + self._finding( + f"builder-edit-missing-target-{_slug(item_id)}-{action_idx + 1}", + "gap", + "major", + f"Checklist item '{item_id}' has file_edit action without target.", + ) + ) + continue + command = action.get("command") + if not isinstance(command, str) or not command.strip(): + item_verified = False + action["evidence"] = { + "type": "snippet", + "content": "Blocked: file_edit action missing deterministic command.", + } + builder_findings.append( + self._finding( + f"builder-edit-missing-command-{_slug(item_id)}-{action_idx + 1}", + "gap", + "major", + ( + f"Checklist item '{item_id}' has file_edit action for '{target}' without a command; " + "runtime does not invent edit instructions." + ), + ) + ) + continue + rel_target = target.strip() + abs_target = os.path.join(self.repo_root, rel_target) + before_hash = _sha256_file(abs_target) if os.path.exists(abs_target) else None + result = _run_command(command.strip()) + action["evidence"] = { + "type": "snippet", + "content": result["evidence"], + "evidence_ref": result["evidence_ref"], + } + if result["passed"]: + after_hash = _sha256_file(abs_target) if os.path.exists(abs_target) else None + if before_hash == after_hash: + item_verified = False + builder_findings.append( + self._finding( + f"builder-edit-no-mutation-{_slug(item_id)}-{action_idx + 1}", + "gap", + "major", + ( + f"Checklist item '{item_id}' file_edit action for '{rel_target}' " + "reported a passing command but produced no target file mutation." + ), + ) + ) + else: + item_files_touched.add(rel_target) + else: + item_verified = False + continue + + item_verified = False + action["evidence"] = { + "type": "snippet", + "content": f"Blocked: unsupported action type '{action_type}'.", + } + builder_findings.append( + self._finding( + f"builder-unsupported-action-{_slug(item_id)}-{action_idx + 1}", + "policy", + "major", + f"Checklist item '{item_id}' uses unsupported action type '{action_type}'.", + ) + ) + + expected_commands = _linked_expectation_commands(item) + missing_expected = sorted(cmd for cmd in expected_commands if cmd not in passed_commands) + if missing_expected: + item_verified = False + builder_findings.append( + self._finding( + f"builder-missing-linked-command-{_slug(item_id)}", + "tests", + "major", + ( + f"Checklist item '{item_id}' linked_test_expectation commands were not observed as passing: " + + ", ".join(missing_expected) + ), + ) + ) + + impl["status"] = "verified" if item_verified else "deferred" + impl["actions"] = actions + impl["files_touched"] = sorted(item_files_touched) + files_touched.update(item_files_touched) + if item_verified and isinstance(item_id, str): + satisfied_checklist_ids.append(item_id) + + for command in normalized_cmds: + _run_command(command) + + execution = milestone.get("execution", {}) + if not isinstance(execution, dict): + execution = {} + existing_files_touched = execution.get("files_touched", []) + if isinstance(existing_files_touched, list): + for touched in existing_files_touched: + if isinstance(touched, str) and touched: + files_touched.add(touched) + execution["files_touched"] = sorted(files_touched) + execution["execution_results"] = execution_results + execution["critical_evidence"] = { + "satisfied_checklist_ids": sorted(dict.fromkeys(satisfied_checklist_ids)), + "passed_test_commands": [cmd for cmd in command_order if cmd in passed_commands], + } + milestone["execution"] = execution + _write_json_atomic(milestone_path, milestone) + errs = validate_file(self.repo_root, milestone_path) + if errs: + return self._task_result( + child_id=child_id, + role="Builder", + phase="16b", + status="failed", + summary="Builder failed: milestone execution artifact invalid", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=[self._finding("builder-validation-failed", "policy", "blocking", "; ".join(errs))], + ) + return self._task_result( + child_id=child_id, + role="Builder", + phase="16b", + status="success", + summary="Builder executed checklist actions and updated execution evidence", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=builder_findings if builder_findings else None, + ) + + return _handler + + def _verifier_handler(self, milestone_path: str): + def _linked_expectation_commands(item: dict) -> Set[str]: + linked = item.get("linked_test_expectation") + commands: Set[str] = set() + if isinstance(linked, str) and linked.strip(): + commands.add(linked.strip()) + elif isinstance(linked, list): + for entry in linked: + if isinstance(entry, str) and entry.strip(): + commands.add(entry.strip()) + return commands + + def _valid_spec_ref(ref: object) -> bool: + return ( + isinstance(ref, dict) + and isinstance(ref.get("type"), str) + and isinstance(ref.get("id"), str) + and isinstance(ref.get("line_range"), str) + and isinstance(ref.get("commit_hash"), str) + ) + + def _remediation(idx: int, summary: str, checklist_id: Optional[str]) -> dict: + return { + "task_id": f"rev-{self.step_id}-{idx}", + "summary": summary, + "files_to_touch": [], + "checklist_ids": [checklist_id] if isinstance(checklist_id, str) else [], + } + + def _handler(task_input: dict, context_pack: dict, child_id: str) -> dict: + milestone = _read_json(milestone_path) + plan = milestone.get("plan", {}) if isinstance(milestone.get("plan"), dict) else {} + checklist = plan.get("spec_alignment", {}).get("checklist", []) if isinstance(plan.get("spec_alignment"), dict) else [] + execution = milestone.get("execution", {}) if isinstance(milestone.get("execution"), dict) else {} + results = execution.get("execution_results", []) if isinstance(execution.get("execution_results"), list) else [] + passed_cmds = execution.get("critical_evidence", {}).get("passed_test_commands", []) + passed_commands = {cmd for cmd in passed_cmds if isinstance(cmd, str) and cmd} + has_failures = any(isinstance(r, dict) and r.get("status") in {"failed", "blocked", "partial"} for r in results) + findings: List[dict] = [] + active_items: List[dict] = [] + + default_spec_ref = None + if isinstance(checklist, list): + for item in checklist: + if not isinstance(item, dict): + continue + if item.get("checklist_status", "active") != "deferred": + active_items.append(item) + candidate = item.get("spec_ref") + if _valid_spec_ref(candidate): + default_spec_ref = candidate + break + + if not active_items: + findings.append( + { + "id": "finding-no-active-checklist-001", + "type": "gap", + "severity": "blocking", + "spec_ref": default_spec_ref + if _valid_spec_ref(default_spec_ref) + else { + "type": "api", + "id": "interface-contracts", + "line_range": "L1-L1", + "commit_hash": _git_head(self.repo_root) or "0" * 40, + }, + "description": "Milestone contains no active checklist items; verified closure is not allowed.", + "metadata": {"source": "Verifier", "impact": "No implementable contract executed"}, + "remediation_task": _remediation(0, "Planner must provide at least one active checklist item.", None), + } + ) + + if has_failures: + findings.append( + { + "id": "finding-builder-failure-001", + "type": "tests", + "severity": "blocking", + "spec_ref": default_spec_ref if _valid_spec_ref(default_spec_ref) else { + "type": "api", + "id": "interface-contracts", + "line_range": "L1-L1", + "commit_hash": _git_head(self.repo_root) or "0" * 40, + }, + "description": "Execution contains failed command results.", + "metadata": {"source": "Verifier", "impact": "Milestone cannot be verified"}, + "remediation_task": _remediation(1, "Fix failing builder command and rerun planner-first loop.", None), + } + ) + + fixture_results: List[dict] = [] + if isinstance(checklist, list): + for idx, item in enumerate(checklist): + if not isinstance(item, dict): + continue + if item.get("checklist_status", "active") == "deferred": + continue + checklist_id = item.get("id") if isinstance(item.get("id"), str) else f"item-{idx + 1}" + spec_ref = item.get("spec_ref") if _valid_spec_ref(item.get("spec_ref")) else default_spec_ref + if not _valid_spec_ref(spec_ref): + continue + impl = item.get("implementation") + verified = isinstance(impl, dict) and impl.get("status") == "verified" + if not verified: + findings.append( + { + "id": f"finding-checklist-not-verified-{idx + 1}", + "type": "gap", + "severity": "major", + "spec_ref": spec_ref, + "description": f"Checklist item '{checklist_id}' is not verified in implementation output.", + "metadata": {"source": "Verifier", "impact": "Spec contract remains incomplete"}, + "remediation_task": _remediation( + idx + 2, + f"Complete checklist item '{checklist_id}' and provide evidence.", + checklist_id, + ), + } + ) + else: + actions = impl.get("actions", []) + missing_evidence = False + if not isinstance(actions, list) or not actions: + missing_evidence = True + else: + for action in actions: + if not (isinstance(action, dict) and isinstance(action.get("evidence"), dict)): + missing_evidence = True + break + if missing_evidence: + findings.append( + { + "id": f"finding-missing-evidence-{idx + 1}", + "type": "tests", + "severity": "major", + "spec_ref": spec_ref, + "description": f"Checklist item '{checklist_id}' is verified but action evidence is incomplete.", + "metadata": {"source": "Verifier", "impact": "Evidence-binding contract violated"}, + "remediation_task": _remediation( + idx + 20, + f"Attach action evidence for checklist item '{checklist_id}'.", + checklist_id, + ), + } + ) + expected_commands = _linked_expectation_commands(item) + missing_commands = sorted(cmd for cmd in expected_commands if cmd not in passed_commands) + if missing_commands: + findings.append( + { + "id": f"finding-missing-linked-tests-{idx + 1}", + "type": "tests", + "severity": "major", + "spec_ref": spec_ref, + "description": ( + f"Checklist item '{checklist_id}' missing passed linked_test_expectation commands: " + + ", ".join(missing_commands) + ), + "metadata": {"source": "Verifier", "impact": "Cannot prove requirement closure"}, + "remediation_task": _remediation( + idx + 40, + f"Run and pass linked test expectations for '{checklist_id}'.", + checklist_id, + ), + } + ) + + fixture_ref = item.get("fixture_ref") + if isinstance(fixture_ref, str) and fixture_ref: + fixture_results.append( + {"fixture_ref": fixture_ref, "status": "pass" if verified else "fail"} + ) + + has_blocking = any(isinstance(f, dict) and f.get("severity") == "blocking" for f in findings) + has_major = any(isinstance(f, dict) and f.get("severity") == "major" for f in findings) + if has_blocking: + verdict = "rejected" + elif has_major: + verdict = "deferred" + else: + verdict = "verified" + + if not fixture_results: + fixture_results = [] + + findings_count = len(findings) + ratings = { + "spec_completeness": 5 if findings_count == 0 else 2 if has_blocking else 3, + "code_quality": 4 if findings_count == 0 else 2, + "tests_completeness": 5 if verdict == "verified" else 2, + "docs_completeness": 4 if verdict == "verified" else 3, + "metadata_usage": 5 if findings_count == 0 else 4, + } + + review = milestone.get("review", {}) + if not isinstance(review, dict): + review = {} + review.update( + { + "findings": findings, + "ratings": ratings, + "verdict": verdict, + "next_actions": ( + "Planner-first remediation required." + if verdict != "verified" + else "Milestone verified." + ), + "fixture_status": { + "implemented_endpoints": [], + "test_results": fixture_results, + "ci_status": "green" if verdict == "verified" else "red", + }, + } + ) + milestone["review"] = review + _write_json_atomic(milestone_path, milestone) + errs = validate_file(self.repo_root, milestone_path) + if errs: + return self._task_result( + child_id=child_id, + role="Verifier", + phase="16c", + status="failed", + summary="Verifier failed: review artifact invalid", + artifacts=[_rel(self.repo_root, milestone_path)], + findings=[self._finding("verifier-validation-failed", "policy", "blocking", "; ".join(errs))], + ) + return self._task_result( + child_id=child_id, + role="Verifier", + phase="16c", + status="success", + summary=f"Verifier completed with verdict={verdict}", + artifacts=[_rel(self.repo_root, milestone_path)], + ) + + return _handler + + def _verified_closure_issues(self, milestone: dict) -> List[str]: + issues: List[str] = [] + review = milestone.get("review", {}) if isinstance(milestone.get("review"), dict) else {} + verdict = review.get("verdict") + if verdict != "verified": + return issues + + findings = review.get("findings", []) + if not isinstance(findings, list): + issues.append("review.findings must be a list when review.verdict is 'verified'") + return issues + blocking_or_major = [ + f for f in findings + if isinstance(f, dict) and f.get("severity") in {"blocking", "major"} + ] + if blocking_or_major: + issues.append("review.verdict is 'verified' but review.findings contains blocking/major findings") + + execution = milestone.get("execution", {}) if isinstance(milestone.get("execution"), dict) else {} + execution_results = execution.get("execution_results", []) + if not isinstance(execution_results, list) or not execution_results: + issues.append("review.verdict is 'verified' but execution.execution_results is missing or empty") + else: + bad_results = [ + r for r in execution_results + if isinstance(r, dict) and r.get("status") in {"failed", "blocked", "partial"} + ] + if bad_results: + issues.append("review.verdict is 'verified' but execution results include failed/blocked/partial statuses") + + plan = milestone.get("plan", {}) if isinstance(milestone.get("plan"), dict) else {} + spec_alignment = plan.get("spec_alignment", {}) if isinstance(plan.get("spec_alignment"), dict) else {} + checklist = spec_alignment.get("checklist", []) + if not isinstance(checklist, list) or not checklist: + issues.append("review.verdict is 'verified' but plan.spec_alignment.checklist is missing or empty") + else: + for item in checklist: + if not isinstance(item, dict): + continue + if item.get("checklist_status", "active") == "deferred": + continue + impl = item.get("implementation") if isinstance(item.get("implementation"), dict) else {} + if impl.get("status") != "verified": + cid = item.get("id", "") + issues.append(f"review.verdict is 'verified' but checklist item '{cid}' implementation.status is not 'verified'") + break + return issues + + def _task_result( + self, + *, + child_id: str, + role: str, + phase: str, + status: str, + summary: str, + artifacts: List[str], + findings: Optional[List[dict]] = None, + questions: Optional[List[str]] = None, + ) -> dict: + payload = { + "protocol_version": PROTO_VER, + "child_id": child_id, + "role": role, + "phase": phase, + "step_id": self.step_id, + "status": status, + "summary": summary, + "artifacts": artifacts, + } + if findings is not None: + payload["findings"] = findings + if questions is not None: + payload["questions"] = questions + return payload + + def _finding(self, fid: str, ftype: str, severity: str, description: str) -> dict: + return { + "id": fid, + "type": ftype, + "severity": severity, + "description": description, + "source": "TrinityRuntime", + "impact": "Blocking contract", + } + + def _run_governance_gates(self, commit_message: str, milestone_path: str, anchor_path: str) -> dict: + spec_dir = os.path.join(self.repo_root, "spec") + schema_errors: List[str] = [] + seed_errors: List[str] = [] + docs_errors: List[str] = [] + commit_errors: List[str] = [] + + if os.path.exists(milestone_path): + schema_errors.extend(validate_file(self.repo_root, milestone_path)) + else: + schema_errors.append(f"Missing milestone artifact for governance: {_rel(self.repo_root, milestone_path)}") + if os.path.exists(anchor_path): + schema_errors.extend(validate_file(self.repo_root, anchor_path)) + else: + schema_errors.append(f"Missing anchor artifact for governance: {_rel(self.repo_root, anchor_path)}") + + seed_errors.extend(lint_seeds(self.repo_root, spec_dir)) + docs_errors.extend(lint_docs(spec_dir)) + commit_errors.extend(check_commit_message(spec_dir, commit_message)) + + errors = schema_errors + seed_errors + docs_errors + commit_errors + return { + "errors": errors, + "schema_status": "pass" if not schema_errors else "fail", + "deep_status": "pass" if not schema_errors else "fail", + "seed_lint_status": "pass" if not seed_errors else "fail", + "docs_lint_status": "pass" if not docs_errors else "fail", + "governance_status": "pass" if not errors else "fail", + "schema_errors": schema_errors, + "seed_errors": seed_errors, + "docs_errors": docs_errors, + "commit_errors": commit_errors, + } + + def _record_governance_validation( + self, + *, + logger: SessionLogger, + phase: str, + commit_message: str, + milestone_path: str, + anchor_path: str, + gate_result: dict, + ) -> None: + summary = ( + f"{phase} governance validation {gate_result.get('governance_status', 'fail')} " + f"({len(gate_result.get('errors', []))} issues)" + ) + artifact_path = anchor_path if os.path.exists(anchor_path) else milestone_path + artifact_ref = _rel(self.repo_root, artifact_path) if os.path.exists(artifact_path) else None + artifact_sha = _sha256_file(artifact_path) if artifact_ref else None + logger.append( + "VALIDATION", + role="Orchestrator", + phase_id=phase, + loop_id="l1", + agent_id=self.root_agent_id, + parent_id=None, + summary=summary, + prompt_template_id=f"prompt_{phase}", + step_id=self.step_id, + artifact_ref=artifact_ref, + artifact_sha256=artifact_sha, + content_extra=self._phase_validation_content( + passed=gate_result.get("governance_status") == "pass", + schema_status=gate_result.get("schema_status"), + deep_status=gate_result.get("deep_status"), + governance_status=gate_result.get("governance_status", "fail"), + seed_lint_status=gate_result.get("seed_lint_status", "fail"), + docs_lint_status=gate_result.get("docs_lint_status", "fail"), + ), + ) + + def _finalize_terminal_result(self, logger: SessionLogger, payload: dict) -> dict: + session_errors = validate_runtime_file(self.repo_root, logger.path, "session_event") + if session_errors: + return { + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "session_log", + "errors": session_errors, + "session_log": _rel(self.repo_root, logger.path), + } + finalized = dict(payload) if isinstance(payload, dict) else {} + if "step_id" not in finalized and isinstance(self.step_id, str): + finalized["step_id"] = self.step_id + if "run_id" not in finalized: + finalized["run_id"] = self.run_id + if "session_log" not in finalized: + finalized["session_log"] = _rel(self.repo_root, logger.path) + return finalized + + def run(self) -> dict: + roadmap = self._load_roadmap() + resume_state = self._load_resume_state() + if isinstance(resume_state, dict): + resume_run_id = resume_state.get("run_id") + resume_parent_id = resume_state.get("parent_id") + if isinstance(resume_run_id, str) and resume_run_id: + self.run_id = resume_run_id + if isinstance(resume_parent_id, str) and resume_parent_id: + self.root_agent_id = resume_parent_id + self.step_id = self._pick_step_id(roadmap) + if not self.step_id: + raise RuntimeError("Unable to resolve step_id") + + if not self.config.allow_dirty and not self.resume and _is_dirty_worktree(self.repo_root): + raise RuntimeError("Working tree must be clean before running Trinity. Set runtime.allow_dirty=true to override.") + + trinity_root = os.path.join(self.repo_root, ".trinity") + runtime_dir = os.path.join(trinity_root, "runtime") + os.makedirs(runtime_dir, exist_ok=True) + milestone_path = os.path.join(self.repo_root, "spec", "impl_context", f"{self.step_id}.json") + anchor_path = os.path.join(self.repo_root, "spec", "16_impl_context.json") + session_state_path = os.path.join(runtime_dir, f"session_state_{self.root_agent_id}.json") + spawn_log_path = os.path.join(runtime_dir, "spawn_log.json") + scratchpad_path = os.path.join(runtime_dir, "scratchpads", f"scratchpad_{self.step_id}.json") + resume_session_log_ref = resume_state.get("session_log_ref") if isinstance(resume_state, dict) else None + if not isinstance(resume_session_log_ref, str): + resume_session_log_ref = None + + resolver = ContextResolver( + self.repo_root, + self.step_id, + milestone_path, + anchor_path, + allow_authority_fallback=self.config.allow_bootstrap_authority_fallback, + ) + logger = SessionLogger( + self.repo_root, + self.run_id, + self.root_agent_id, + self.step_id, + self.config.llm_model, + decoding_temperature=self.config.llm_temperature, + decoding_top_p=self.config.llm_top_p, + decoding_max_tokens=self.config.llm_max_tokens, + log_path=resume_session_log_ref, + ) + def _terminal(payload: dict) -> dict: + return self._finalize_terminal_result(logger, payload) + + tools = ToolExecutor( + self.repo_root, + logger, + self.run_id, + agent_id=self.root_agent_id, + phase="16a", + step_id=self.step_id, + allowed_read_paths=["."], + allowed_write_paths=["."], + target_file_patterns=[], + docs_policy={}, + protected_write_paths=[], + enable_checkpoints=self.config.checkpoint_commits, + ) + + milestone_attempt = 0 + planner_retries = 0 + builder_retries = 0 + verifier_retries = 0 + if isinstance(resume_state, dict): + retry_counters = resume_state.get("retry_counters", {}) + if isinstance(retry_counters, dict): + planner_retries = int(retry_counters.get("planner", 0) or 0) + builder_retries = int(retry_counters.get("builder", 0) or 0) + verifier_retries = int(retry_counters.get("verifier", 0) or 0) + milestone_attempt = int(retry_counters.get("milestone", 0) or 0) + session_status = "resuming" if isinstance(resume_state, dict) else "idle" + pending_child_id = resume_state.get("pending_child_id") if isinstance(resume_state, dict) else None + pending_spawn_ref = resume_state.get("pending_spawn_ref") if isinstance(resume_state, dict) else None + pending_questions = resume_state.get("pending_questions") if isinstance(resume_state, dict) else None + if not isinstance(pending_child_id, str): + pending_child_id = None + if not isinstance(pending_spawn_ref, str): + pending_spawn_ref = None + if not isinstance(pending_questions, list): + pending_questions = None + self._write_session_state( + session_state_path=session_state_path, + active_phase="16a", + status=session_status, + pending_child_id=pending_child_id, + pending_spawn_ref=pending_spawn_ref, + pending_questions=pending_questions, + session_log_ref=_rel(self.repo_root, logger.path), + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + if not self.resume or not os.path.exists(spawn_log_path): + _write_json_atomic( + spawn_log_path, + {"protocol_version": PROTO_VER, "run_id": self.run_id, "entries": []}, + ) + spawn_log_errors = validate_runtime_file(self.repo_root, spawn_log_path, "spawn_log") + if spawn_log_errors: + raise RuntimeError("; ".join(spawn_log_errors)) + + if self.config.conformance_mode and not self.config.checkpoint_commits: + raise RuntimeError( + "runtime.checkpoint_commits=false is only allowed when runtime.conformance_mode=false" + ) + + if self.config.checkpoint_commits: + self._ensure_branch(tools) + + replayed_phase_results: Dict[str, Tuple[dict, List[str]]] = {} + if isinstance(resume_state, dict): + resumed = self._resume_pending_spawn( + logger=logger, + session_state_path=session_state_path, + spawn_log_path=spawn_log_path, + scratchpad_path=scratchpad_path, + milestone_path=milestone_path, + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + resume_state=resume_state, + ) + if isinstance(resumed, dict): + if resumed.get("status") == "questions": + return _terminal({ + "status": "questions", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": resumed.get("phase"), + "questions": resumed.get("questions", []), + "pending_spawn_ref": resumed.get("pending_spawn_ref"), + }) + if resumed.get("status") == "blocked": + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": resumed.get("phase"), + "errors": resumed.get("errors", []), + }) + if resumed.get("status") == "replayed": + phase_key = resumed.get("phase") + if isinstance(phase_key, str): + replayed_phase_results[phase_key] = ( + resumed.get("task_result", {}), + resumed.get("errors", []), + ) + + while milestone_attempt < self.retry_caps["milestone"]: + milestone_attempt += 1 + if "16a" in replayed_phase_results: + planner_result, planner_errors = replayed_phase_results.pop("16a") + else: + planner_result, planner_errors = self._spawn_phase( + logger=logger, + resolver=resolver, + phase="16a", + role="Planner", + phase_label="Planner", + milestone_path=milestone_path, + anchor_path=anchor_path, + session_state_path=session_state_path, + spawn_log_path=spawn_log_path, + scratchpad_path=scratchpad_path, + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + if planner_result.get("status") == "questions": + state_snapshot = _read_json(session_state_path) if os.path.exists(session_state_path) else {} + return _terminal({ + "status": "questions", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16a", + "questions": planner_result.get("questions", []), + "pending_spawn_ref": state_snapshot.get("pending_spawn_ref"), + }) + if planner_errors or planner_result.get("status") != "success": + planner_retries += 1 + if planner_retries >= self.retry_caps["planner"]: + self._write_session_state( + session_state_path=session_state_path, + active_phase="16a", + status="blocked", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16a", + "errors": planner_errors or planner_result.get("errors", []), + }) + continue + + try: + self._regenerate_anchor(milestone_path, anchor_path, logger=logger, phase="16a") + except Exception as e: # noqa: BLE001 + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16a-anchor", + "errors": [str(e)], + }) + if self.config.checkpoint_commits: + tools.phase = "16a" + commit_message = f"trinity({self.step_id}): checkpoint after 16a" + gate_result = self._run_governance_gates(commit_message, milestone_path, anchor_path) + self._record_governance_validation( + logger=logger, + phase="16a", + commit_message=commit_message, + milestone_path=milestone_path, + anchor_path=anchor_path, + gate_result=gate_result, + ) + self._write_scratchpad( + scratchpad_path, + phase="16a", + next_action_ref="phase:16a:checkpoint", + state_summary="16a governance gate evaluated", + checklist_scope=self._checklist_ids(milestone_path), + validation_gate={ + "schema": gate_result.get("schema_status"), + "deep_validator": gate_result.get("deep_status"), + "governance": gate_result.get("governance_status"), + }, + ) + gate_errs = gate_result.get("errors", []) + if gate_errs: + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16a-governance", + "errors": gate_errs, + }) + tools.call("checkpoint_commit", {"message": commit_message}, role="Orchestrator", loop_id="l1") + + if "16b" in replayed_phase_results: + builder_result, builder_errors = replayed_phase_results.pop("16b") + else: + builder_result, builder_errors = self._spawn_phase( + logger=logger, + resolver=resolver, + phase="16b", + role="Builder", + phase_label="Builder", + milestone_path=milestone_path, + anchor_path=anchor_path, + session_state_path=session_state_path, + spawn_log_path=spawn_log_path, + scratchpad_path=scratchpad_path, + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + if builder_result.get("status") == "questions": + state_snapshot = _read_json(session_state_path) if os.path.exists(session_state_path) else {} + return _terminal({ + "status": "questions", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16b", + "questions": builder_result.get("questions", []), + "pending_spawn_ref": state_snapshot.get("pending_spawn_ref"), + }) + if builder_errors or builder_result.get("status") != "success": + builder_retries += 1 + if builder_retries >= self.retry_caps["builder"]: + self._write_session_state( + session_state_path=session_state_path, + active_phase="16b", + status="blocked", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16b", + "errors": builder_errors or builder_result.get("errors", []), + }) + continue + + try: + self._regenerate_anchor(milestone_path, anchor_path, logger=logger, phase="16b") + except Exception as e: # noqa: BLE001 + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16b-anchor", + "errors": [str(e)], + }) + if self.config.checkpoint_commits: + tools.phase = "16b" + commit_message = f"trinity({self.step_id}): checkpoint after 16b" + gate_result = self._run_governance_gates(commit_message, milestone_path, anchor_path) + self._record_governance_validation( + logger=logger, + phase="16b", + commit_message=commit_message, + milestone_path=milestone_path, + anchor_path=anchor_path, + gate_result=gate_result, + ) + self._write_scratchpad( + scratchpad_path, + phase="16b", + next_action_ref="phase:16b:checkpoint", + state_summary="16b governance gate evaluated", + checklist_scope=self._checklist_ids(milestone_path), + validation_gate={ + "schema": gate_result.get("schema_status"), + "deep_validator": gate_result.get("deep_status"), + "governance": gate_result.get("governance_status"), + }, + ) + gate_errs = gate_result.get("errors", []) + if gate_errs: + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16b-governance", + "errors": gate_errs, + }) + tools.call("checkpoint_commit", {"message": commit_message}, role="Orchestrator", loop_id="l1") + + if "16c" in replayed_phase_results: + verifier_result, verifier_errors = replayed_phase_results.pop("16c") + else: + verifier_result, verifier_errors = self._spawn_phase( + logger=logger, + resolver=resolver, + phase="16c", + role="Verifier", + phase_label="Verifier", + milestone_path=milestone_path, + anchor_path=anchor_path, + session_state_path=session_state_path, + spawn_log_path=spawn_log_path, + scratchpad_path=scratchpad_path, + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + if verifier_result.get("status") == "questions": + state_snapshot = _read_json(session_state_path) if os.path.exists(session_state_path) else {} + return _terminal({ + "status": "questions", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16c", + "questions": verifier_result.get("questions", []), + "pending_spawn_ref": state_snapshot.get("pending_spawn_ref"), + }) + if verifier_errors or verifier_result.get("status") != "success": + verifier_retries += 1 + if verifier_retries >= self.retry_caps["verifier"]: + verifier_cap_errors = verifier_errors or verifier_result.get("errors", []) + if not verifier_cap_errors: + verifier_cap_errors = [ + f"verifier retry cap exceeded after {verifier_retries} attempts without explicit error payload" + ] + self._write_session_state( + session_state_path=session_state_path, + active_phase="16c", + status="blocked", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16c", + "errors": verifier_cap_errors, + }) + continue + milestone = _read_json(milestone_path) + verdict = ( + milestone.get("review", {}).get("verdict") + if isinstance(milestone.get("review"), dict) + else None + ) + try: + self._regenerate_anchor(milestone_path, anchor_path, logger=logger, phase="16c") + except Exception as e: # noqa: BLE001 + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16c-anchor", + "errors": [str(e)], + }) + if verdict == "verified": + closure_issues = self._verified_closure_issues(milestone) + if closure_issues: + verifier_retries += 1 + if verifier_retries >= self.retry_caps["verifier"]: + self._write_session_state( + session_state_path=session_state_path, + active_phase="16c", + status="blocked", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16c-closure", + "errors": closure_issues, + }) + continue + self._update_roadmap_status(self._roadmap_path(), "done") + if self.config.checkpoint_commits: + tools.phase = "16c" + commit_message = f"trinity({self.step_id}): final closure after 16c" + gate_result = self._run_governance_gates(commit_message, milestone_path, anchor_path) + self._record_governance_validation( + logger=logger, + phase="16c", + commit_message=commit_message, + milestone_path=milestone_path, + anchor_path=anchor_path, + gate_result=gate_result, + ) + self._write_scratchpad( + scratchpad_path, + phase="16c", + next_action_ref="phase:16c:checkpoint", + state_summary="16c governance gate evaluated", + checklist_scope=self._checklist_ids(milestone_path), + validation_gate={ + "schema": gate_result.get("schema_status"), + "deep_validator": gate_result.get("deep_status"), + "governance": gate_result.get("governance_status"), + }, + ) + gate_errs = gate_result.get("errors", []) + if gate_errs: + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "phase": "16c-governance", + "errors": gate_errs, + }) + tools.call("checkpoint_commit", {"message": commit_message}, role="Orchestrator", loop_id="l1") + self._write_session_state( + session_state_path=session_state_path, + active_phase="16c", + status="done", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + return _terminal({ + "status": "completed", + "step_id": self.step_id, + "run_id": self.run_id, + "execution_mode": self.execution_mode, + "milestone_artifact": _rel(self.repo_root, milestone_path), + "anchor_artifact": _rel(self.repo_root, anchor_path), + "session_log": _rel(self.repo_root, logger.path), + "verdict": verdict, + }) + # planner-first remediation + continue + + self._write_session_state( + session_state_path=session_state_path, + active_phase="16a", + status="blocked", + pending_child_id=None, + pending_spawn_ref=None, + pending_questions=None, + spawn_log_ref=".trinity/runtime/spawn_log.json", + scratchpad_ref=_rel(self.repo_root, scratchpad_path), + retry_counters={ + "planner": planner_retries, + "builder": builder_retries, + "verifier": verifier_retries, + "milestone": milestone_attempt, + }, + ) + return _terminal({ + "status": "blocked", + "step_id": self.step_id, + "run_id": self.run_id, + "errors": ["global milestone retry cap exceeded"], + }) + + +def run_trinity_child( + *, + repo_root: str, + step_id: str, + phase: str, + role: str, + child_id: str, + milestone_path: str, + task_input_path: str, + context_pack_path: str, + task_result_path: str, + session_log_path: str, + run_id: str, + parent_id: str, + mode_override: Optional[str] = None, +) -> dict: + config = TrinityConfig.load(repo_root) + if isinstance(mode_override, str) and mode_override.strip(): + config.execution_mode = mode_override.strip().lower() + runtime = TrinityRuntime(repo_root, config, step_id=step_id, resume=False, answers=None, resume_run_id=None) + runtime.run_id = run_id + runtime.root_agent_id = parent_id + logger = SessionLogger( + repo_root, + run_id, + child_id, + step_id, + config.llm_model, + decoding_temperature=config.llm_temperature, + decoding_top_p=config.llm_top_p, + decoding_max_tokens=config.llm_max_tokens, + log_path=session_log_path, + ) + milestone_abs = milestone_path if os.path.isabs(milestone_path) else os.path.join(repo_root, milestone_path) + task_input_abs = task_input_path if os.path.isabs(task_input_path) else os.path.join(repo_root, task_input_path) + context_pack_abs = context_pack_path if os.path.isabs(context_pack_path) else os.path.join(repo_root, context_pack_path) + task_result_abs = task_result_path if os.path.isabs(task_result_path) else os.path.join(repo_root, task_result_path) + task_input = _read_json(task_input_abs) + context_pack = _read_json(context_pack_abs) + if phase == "16a": + handler = ( + runtime._llm_phase_handler(milestone_abs, logger, phase="16a", role=role) + if runtime.execution_mode == "llm" + else runtime._planner_handler(milestone_abs) + ) + elif phase == "16b": + handler = ( + runtime._llm_phase_handler(milestone_abs, logger, phase="16b", role=role) + if runtime.execution_mode == "llm" + else runtime._builder_handler(milestone_abs, logger) + ) + elif phase == "16c": + handler = ( + runtime._llm_phase_handler(milestone_abs, logger, phase="16c", role=role) + if runtime.execution_mode == "llm" + else runtime._verifier_handler(milestone_abs) + ) + elif phase == "utility": + if runtime.execution_mode != "llm": + result = runtime._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: utility phase requires llm execution mode", + artifacts=[], + findings=[ + runtime._finding( + "utility-deterministic-unsupported", + "policy", + "blocking", + "Utility phase is only supported in llm execution mode.", + ) + ], + ) + _write_json_atomic(task_result_abs, result) + return result + handler = runtime._llm_phase_handler(milestone_abs, logger, phase="utility", role=role) + else: + result = runtime._task_result( + child_id=child_id, + role=role, + phase=phase, + status="blocked", + summary=f"{role} blocked: unsupported phase for child runner", + artifacts=[], + findings=[runtime._finding(f"child-unsupported-phase-{phase}", "policy", "blocking", f"Unsupported child phase '{phase}'")], + ) + _write_json_atomic(task_result_abs, result) + return result + result = handler(task_input, context_pack, child_id) + _write_json_atomic(task_result_abs, result) + errors = validate_runtime_file(repo_root, task_result_abs, "task_result") + if errors: + raise RuntimeError("; ".join(errors)) + return result + + +def run_trinity( + repo_root: str, + step_id: Optional[str], + resume: bool = False, + mode_override: Optional[str] = None, + answers: Optional[List[str]] = None, + resume_run_id: Optional[str] = None, +) -> dict: + config = TrinityConfig.load(repo_root) + if isinstance(mode_override, str) and mode_override.strip(): + config.execution_mode = mode_override.strip().lower() + runtime = TrinityRuntime( + repo_root, + config, + step_id=step_id, + resume=resume, + answers=answers, + resume_run_id=resume_run_id, + ) + return runtime.run() diff --git a/tools/specdev_tools/trinity_runtime_validate.py b/tools/specdev_tools/trinity_runtime_validate.py new file mode 100644 index 00000000..b852b840 --- /dev/null +++ b/tools/specdev_tools/trinity_runtime_validate.py @@ -0,0 +1,1485 @@ +from __future__ import annotations + +import json +import os +import re +import subprocess +import hashlib +from typing import Optional, Dict +import fnmatch + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +from .registry import SchemaRegistry + + +RUNTIME_SCHEMA_BY_TYPE: Dict[str, str] = { + "task_input": "https://specdev.local/schema/trinity/task_input.schema.json", + "context_pack": "https://specdev.local/schema/trinity/context_pack.schema.json", + "task_result": "https://specdev.local/schema/trinity/task_result.schema.json", + "tool_call_request": "https://specdev.local/schema/trinity/tool_call_request.schema.json", + "tool_call_result": "https://specdev.local/schema/trinity/tool_call_result.schema.json", + "utility_call": "https://specdev.local/schema/trinity/utility_call.schema.json", + "utility_result": "https://specdev.local/schema/trinity/utility_result.schema.json", + "session_event": "https://specdev.local/schema/trinity/session_event.schema.json", + "log_capture_policy": "https://specdev.local/schema/trinity/log_capture_policy.schema.json", + "eval_export_row": "https://specdev.local/schema/trinity/eval_export_row.schema.json", + "scratchpad_state": "https://specdev.local/schema/trinity/scratchpad_state.schema.json", + "session_state": "https://specdev.local/schema/trinity/session_state.schema.json", + "spawn_log": "https://specdev.local/schema/trinity/spawn_log.schema.json", +} + +_SPEC_REF_TYPE_BY_BASENAME: Dict[str, str] = { + "04_fr_list.json": "fr", + "05_interface_contracts.json": "api", + "06_invariants.json": "inv", + "07_nfrs.json": "nfr", + "08_fixtures.json": "fixture", +} + +_SENSITIVE_PATTERNS = { + "private_key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |)?PRIVATE KEY-----"), + "aws_access_key": re.compile(r"\bAKIA[0-9A-Z]{16}\b"), + "aws_secret_access_key": re.compile(r"\baws[_-]?secret[_-]?access[_-]?key\b[^\n]{0,40}[A-Za-z0-9/+=]{40}"), + "openai_key": re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), + "github_pat": re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), + "gitlab_pat": re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b"), + "google_api_key": re.compile(r"\bAIza[0-9A-Za-z\-_]{35}\b"), + "npm_token": re.compile(r"\bnpm_[A-Za-z0-9]{36}\b"), + "slack_token": re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), + "bearer_token": re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._\-]{20,}\b"), + "jwt_token": re.compile(r"\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b"), + "generic_secret_assignment": re.compile( + r"(?i)\b(api[_-]?key|secret|token|password)\b\s*[:=]\s*['\"]?[A-Za-z0-9_\-./+=]{16,}" + ), +} + + +def _registry_for(registry: SchemaRegistry) -> Registry: + store = {uri: Resource.from_contents(schema) for uri, schema in registry.store.items()} + return Registry().with_resources(store.items()) + + +def detect_runtime_artifact_type(path: str) -> Optional[str]: + norm = path.replace("\\", "/") + name = os.path.basename(norm) + + if name == "task_input.json": + return "task_input" + if name == "context_pack.json": + return "context_pack" + if name == "task_result.json": + return "task_result" + if name == "tool_call_request.json": + return "tool_call_request" + if name == "tool_call_result.json": + return "tool_call_result" + if name == "utility_call.json": + return "utility_call" + if name == "utility_result.json": + return "utility_result" + if name.startswith("scratchpad_") and name.endswith(".json"): + return "scratchpad_state" + if name.startswith("session_state_") and name.endswith(".json"): + return "session_state" + if name == "spawn_log.json": + return "spawn_log" + if name == "log_capture_policy.json": + return "log_capture_policy" + if name == "eval_export_row.json": + return "eval_export_row" + if name.endswith(".jsonl") and "/.trinity/sessions/" in norm: + return "session_event" + return None + + +def maybe_validate_runtime_artifact(repo_root: str, path: str) -> Optional[list[str]]: + artifact_type = detect_runtime_artifact_type(path) + if not artifact_type: + return None + return validate_runtime_file(repo_root, path, artifact_type) + + +def _validate_payload( + repo_root: str, + path: str, + payload: dict, + schema_uri: str, + line_no: Optional[int] = None, +) -> list[str]: + registry = SchemaRegistry(repo_root) + schema = registry.load(schema_uri) + reg = _registry_for(registry) + validator = Draft202012Validator( + schema, + registry=reg, + format_checker=Draft202012Validator.FORMAT_CHECKER, + ) + errors = sorted(validator.iter_errors(payload), key=lambda e: list(e.path)) + prefix = f"{path}:{line_no}" if line_no is not None else path + return [f"{prefix}:{'/'.join(map(str, e.path))}: {e.message}" for e in errors] + + +def _normalize_path(path: str) -> str: + raw = path.replace("\\", "/").strip() + if raw.startswith("./"): + raw = raw[2:] + normalized = os.path.normpath(raw or ".").replace("\\", "/") + if normalized in {"", "."}: + return "." + return normalized + + +def _is_escape_path(path: str) -> bool: + normalized = _normalize_path(path) + return normalized == ".." or normalized.startswith("../") + + +def _find_git_root(repo_root: str) -> Optional[str]: + cur = os.path.abspath(repo_root) + while True: + if os.path.isdir(os.path.join(cur, ".git")): + return cur + parent = os.path.dirname(cur) + if parent == cur: + break + cur = parent + return None + + +def _parse_line_range(value: str) -> Optional[tuple[int, int]]: + match = re.match(r"^L(\d+)-L(\d+)$", value or "") + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def _git_commit_exists(git_root: str, commit_hash: str, cache: dict[str, bool]) -> bool: + if commit_hash in cache: + return cache[commit_hash] + result = subprocess.run( + ["git", "cat-file", "-e", f"{commit_hash}^{{commit}}"], + cwd=git_root, + capture_output=True, + text=True, + check=False, + ) + cache[commit_hash] = result.returncode == 0 + return cache[commit_hash] + + +def _git_file_lines( + git_root: str, + commit_hash: str, + rel_path: str, + cache: dict[tuple[str, str], Optional[list[str]]], +) -> Optional[list[str]]: + key = (commit_hash, rel_path) + if key in cache: + return cache[key] + result = subprocess.run( + ["git", "show", f"{commit_hash}:{rel_path}"], + cwd=git_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + cache[key] = None + return None + cache[key] = result.stdout.splitlines() + return cache[key] + + +def _line_range_contains_reference_id(lines: list[str], start: int, end: int, ref_id: str) -> bool: + excerpt = "\n".join(lines[start - 1 : end]) + id_field_pattern = re.compile(rf'"id"\s*:\s*"{re.escape(ref_id)}"') + quoted_id_pattern = re.compile(rf'"{re.escape(ref_id)}"') + return bool(id_field_pattern.search(excerpt) or quoted_id_pattern.search(excerpt)) + + +def _resolve_ref(base_file: str, ref: str, repo_root: str) -> str: + if os.path.isabs(ref): + return ref + candidate_local = os.path.abspath(os.path.join(os.path.dirname(base_file), ref)) + if os.path.exists(candidate_local): + return candidate_local + return os.path.abspath(os.path.join(repo_root, ref)) + + +def _detect_sensitive_classes(text: Optional[str]) -> list[str]: + if not isinstance(text, str) or not text: + return [] + hits: list[str] = [] + for cls, pattern in _SENSITIVE_PATTERNS.items(): + if pattern.search(text): + hits.append(cls) + return hits + + +def _artifact_sensitive_classes(base_file: str, ref: Optional[str], repo_root: str) -> list[str]: + if not isinstance(ref, str) or not ref: + return [] + resolved = _resolve_ref(base_file, ref, repo_root) + if not os.path.exists(resolved): + return [] + try: + with open(resolved, "r", encoding="utf-8", errors="ignore") as f: + sample = f.read(256 * 1024) + except Exception: + return [] + return _detect_sensitive_classes(sample) + + +def _is_pattern_covered_by_allowed(pattern: str, allowed_paths: list[str]) -> bool: + normalized_pattern = _normalize_path(pattern) + if _is_escape_path(normalized_pattern): + return False + root = normalized_pattern.split("*", 1)[0] + root = root.rstrip("/") + if not root: + return False + + for allowed in allowed_paths: + allowed_norm = _normalize_path(allowed).rstrip("/") + if not allowed_norm: + continue + if _is_escape_path(allowed_norm): + continue + if allowed_norm == ".": + return True + if root == allowed_norm or root.startswith(allowed_norm + "/"): + return True + if fnmatch.fnmatch(root, allowed_norm): + return True + return False + + +def _is_file_covered_by_pattern(path: str, patterns: list[str]) -> bool: + normalized_path = _normalize_path(path) + if _is_escape_path(normalized_path): + return False + for pattern in patterns: + if not isinstance(pattern, str): + continue + normalized_pattern = _normalize_path(pattern) + if _is_escape_path(normalized_pattern): + continue + if fnmatch.fnmatch(normalized_path, normalized_pattern): + return True + return False + + +def _is_file_covered_by_allowed(path: str, allowed_paths: list[str]) -> bool: + normalized_path = _normalize_path(path) + if _is_escape_path(normalized_path): + return False + for allowed in allowed_paths: + allowed_norm = _normalize_path(allowed).rstrip("/") + if not allowed_norm: + continue + if _is_escape_path(allowed_norm): + continue + if allowed_norm == ".": + return True + if normalized_path == allowed_norm or normalized_path.startswith(allowed_norm + "/"): + return True + if fnmatch.fnmatch(normalized_path, allowed_norm): + return True + return False + + +def _validate_required_spec_refs_grounding(repo_root: str, path: str, payload: dict) -> list[str]: + errors: list[str] = [] + refs = payload.get("required_spec_refs", []) + if not isinstance(refs, list) or not refs: + return errors + + git_root = _find_git_root(repo_root) + if not git_root: + return [f"{path}: git root not found for required_spec_refs grounding"] + + commit_cache: dict[str, bool] = {} + lines_cache: dict[tuple[str, str], Optional[list[str]]] = {} + for idx, ref in enumerate(refs, start=1): + if not isinstance(ref, dict): + continue + + ref_type = ref.get("type") + ref_id = ref.get("id") + ref_path = ref.get("path") + line_range = ref.get("line_range") + commit_hash = ref.get("commit_hash") + + if not all(isinstance(v, str) and v for v in (ref_type, ref_id, ref_path, line_range, commit_hash)): + continue + + if not _git_commit_exists(git_root, commit_hash, commit_cache): + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) commit_hash '{commit_hash}' not found in git" + ) + continue + + rel_path = _normalize_path(ref_path) + if os.path.isabs(ref_path): + rel_path = _normalize_path(os.path.relpath(ref_path, git_root)) + if _is_escape_path(rel_path): + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) path '{ref_path}' is outside git root" + ) + continue + + expected_type = _SPEC_REF_TYPE_BY_BASENAME.get(os.path.basename(rel_path)) + if expected_type and ref_type != expected_type: + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) path '{rel_path}' implies type '{expected_type}'" + ) + + parsed = _parse_line_range(line_range) + if not parsed: + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) invalid line_range '{line_range}'" + ) + continue + start, end = parsed + if start < 1 or end < start: + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) invalid line_range bounds '{line_range}'" + ) + continue + + lines = _git_file_lines(git_root, commit_hash, rel_path, lines_cache) + if lines is None: + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) path '{rel_path}' not present at commit '{commit_hash}'" + ) + continue + if end > len(lines): + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) line_range '{line_range}' exceeds file length at commit" + ) + continue + + if not _line_range_contains_reference_id(lines, start, end, ref_id): + errors.append( + f"{path}: required_spec_refs[{idx}] ({ref_type}:{ref_id}) line_range '{line_range}' does not contain referenced id at commit" + ) + + return errors + + +def _validate_context_pack_deep(repo_root: str, path: str, payload: dict) -> list[str]: + errors: list[str] = [] + + phase = payload.get("phase") + required_spec_refs = payload.get("required_spec_refs", []) + allowed_write_paths = payload.get("allowed_write_paths", []) + target_file_patterns = payload.get("target_file_patterns", []) + if phase in {"16a", "16b", "16c"}: + if not isinstance(required_spec_refs, list) or len(required_spec_refs) == 0: + errors.append( + f"{path}: required_spec_refs must include at least one grounded spec reference for phase '{phase}'" + ) + if phase in {"16a", "16b", "16c"}: + for pattern in target_file_patterns: + if isinstance(pattern, str) and pattern and not _is_pattern_covered_by_allowed(pattern, allowed_write_paths): + errors.append( + f"{path}: target_file_patterns entry '{pattern}' is outside allowed_write_paths" + ) + + manifest_ref = payload.get("seed_manifest_path") + if isinstance(manifest_ref, str) and manifest_ref: + manifest_path = _resolve_ref(path, manifest_ref, repo_root) + if not os.path.exists(manifest_path): + errors.append(f"{path}: seed_manifest_path not found: {manifest_ref}") + return errors + try: + with open(manifest_path, "r", encoding="utf-8") as f: + manifest = json.load(f) + except Exception as e: + errors.append(f"{path}: unable to read seed manifest for deep validation ({e})") + return errors + + manifest_seed_paths = set() + seed_path_by_id: dict[str, str] = {} + for seed in manifest.get("seeds", []): + if isinstance(seed, dict): + sid = seed.get("seed_id") + spath = seed.get("path") + if isinstance(spath, str): + manifest_seed_paths.add(spath) + if isinstance(sid, str) and isinstance(spath, str): + seed_path_by_id[sid] = spath + + provided_seed_files = payload.get("seed_files_ordered", []) + for seed_file in provided_seed_files: + if isinstance(seed_file, str) and seed_file not in manifest_seed_paths: + errors.append( + f"{path}: seed_files_ordered entry '{seed_file}' is not present in seed_manifest.seeds" + ) + + global_order = manifest.get("global_seed_order", []) + expected_global_paths = [seed_path_by_id[sid] for sid in global_order if sid in seed_path_by_id] + missing_global_paths = [p for p in expected_global_paths if p not in provided_seed_files] + if missing_global_paths: + errors.append( + f"{path}: seed_files_ordered missing global seed paths: {', '.join(missing_global_paths)}" + ) + else: + idxs = [provided_seed_files.index(p) for p in expected_global_paths] + if idxs != sorted(idxs): + errors.append( + f"{path}: seed_files_ordered violates global_seed_order sequence from seed_manifest" + ) + + # Fail-fast phase contract: context pack must include all seeds required + # by step_requirements for the active phase. + if phase in {"16a", "16b", "16c"}: + step_requirements = manifest.get("step_requirements", {}) + required_seed_ids = step_requirements.get(phase) + if not isinstance(required_seed_ids, list): + errors.append( + f"{path}: seed_manifest.step_requirements['{phase}'] is missing or invalid" + ) + else: + unknown_required_ids = [ + sid for sid in required_seed_ids if isinstance(sid, str) and sid not in seed_path_by_id + ] + if unknown_required_ids: + errors.append( + f"{path}: seed_manifest.step_requirements['{phase}'] references unknown seed ids: " + + ", ".join(unknown_required_ids) + ) + expected_phase_paths = [ + seed_path_by_id[sid] + for sid in required_seed_ids + if isinstance(sid, str) and sid in seed_path_by_id + ] + missing_phase_paths = [p for p in expected_phase_paths if p not in provided_seed_files] + if missing_phase_paths: + errors.append( + f"{path}: seed_files_ordered missing step_requirements['{phase}'] seed paths: " + + ", ".join(missing_phase_paths) + ) + + seen_refs: set[tuple[str, str]] = set() + for ref in required_spec_refs if isinstance(required_spec_refs, list) else []: + if not isinstance(ref, dict): + continue + key = (str(ref.get("type", "")), str(ref.get("id", ""))) + if key in seen_refs: + errors.append(f"{path}: duplicate required_spec_refs entry for type/id {key[0]}:{key[1]}") + seen_refs.add(key) + + bootstrap_trace = payload.get("bootstrap_ref_trace") + if phase == "16a" and isinstance(bootstrap_trace, list) and bootstrap_trace: + grounded_keys = { + (str(ref.get("type", "")), str(ref.get("id", ""))) + for ref in required_spec_refs + if isinstance(ref, dict) + } + for idx, entry in enumerate(bootstrap_trace): + if not isinstance(entry, dict): + continue + key = (str(entry.get("spec_type", "")), str(entry.get("id", ""))) + if key not in grounded_keys: + errors.append( + f"{path}: bootstrap_ref_trace[{idx}] ({key[0]}:{key[1]}) must map to required_spec_refs" + ) + source = entry.get("selected_from") + mode = entry.get("selection_mode") + if isinstance(source, str) and source.startswith("roadmap.") and mode == "authority_fallback": + errors.append( + f"{path}: bootstrap_ref_trace[{idx}] cannot use authority_fallback mode with roadmap source '{source}'" + ) + + errors.extend(_validate_required_spec_refs_grounding(repo_root, path, payload)) + + return errors + + +def _validate_task_input_deep(repo_root: str, path: str, payload: dict) -> list[str]: + errors: list[str] = [] + + context_pack_ref = payload.get("context_pack_ref") + if not isinstance(context_pack_ref, str) or not context_pack_ref: + return errors + + context_pack_path = _resolve_ref(path, context_pack_ref, repo_root) + if not os.path.exists(context_pack_path): + return [f"{path}: context_pack_ref not found: {context_pack_ref}"] + + try: + with open(context_pack_path, "r", encoding="utf-8") as f: + context_pack = json.load(f) + except Exception as e: + return [f"{path}: failed reading context_pack_ref '{context_pack_ref}' ({e})"] + + context_pack_schema_errors = _validate_payload( + repo_root, + context_pack_path, + context_pack, + RUNTIME_SCHEMA_BY_TYPE["context_pack"], + ) + if context_pack_schema_errors: + errors.extend( + [f"{path}: referenced context_pack_ref failed schema validation: {msg}" for msg in context_pack_schema_errors] + ) + return errors + + task_phase = payload.get("phase") + context_phase = context_pack.get("phase") + if isinstance(task_phase, str) and isinstance(context_phase, str) and task_phase != context_phase: + errors.append( + f"{path}: task_input phase '{task_phase}' does not match context_pack phase '{context_phase}'" + ) + + task_step_id = payload.get("step_id") + context_step_id = context_pack.get("step_id") + if isinstance(task_step_id, str) and isinstance(context_step_id, str) and task_step_id != context_step_id: + errors.append( + f"{path}: task_input step_id '{task_step_id}' does not match context_pack step_id '{context_step_id}'" + ) + + target_files = payload.get("target_files", []) + target_file_patterns = context_pack.get("target_file_patterns", []) + allowed_write_paths = context_pack.get("allowed_write_paths", []) + + if isinstance(target_files, list): + for target in target_files: + if not isinstance(target, str) or not target: + continue + if isinstance(target_file_patterns, list) and target_file_patterns: + if not _is_file_covered_by_pattern(target, target_file_patterns): + errors.append( + f"{path}: target_files entry '{target}' is not covered by context_pack.target_file_patterns" + ) + if isinstance(allowed_write_paths, list) and allowed_write_paths: + if not _is_file_covered_by_allowed(target, allowed_write_paths): + errors.append( + f"{path}: target_files entry '{target}' is outside context_pack.allowed_write_paths" + ) + + task_spec_refs = payload.get("spec_refs", []) + context_required_refs = context_pack.get("required_spec_refs", []) + context_ref_keys = set() + if isinstance(context_required_refs, list): + for ref in context_required_refs: + if isinstance(ref, dict): + ref_type = ref.get("type") + ref_id = ref.get("id") + if isinstance(ref_type, str) and isinstance(ref_id, str): + context_ref_keys.add((ref_type, ref_id)) + + if task_phase in {"16a", "16b", "16c"} and not context_ref_keys: + errors.append( + f"{path}: context_pack.required_spec_refs must be non-empty for phase '{task_phase}'" + ) + if isinstance(task_spec_refs, list): + for idx, ref in enumerate(task_spec_refs, start=1): + if not isinstance(ref, dict): + continue + ref_type = ref.get("type") + ref_id = ref.get("id") + if isinstance(ref_type, str) and isinstance(ref_id, str): + if (ref_type, ref_id) not in context_ref_keys: + errors.append( + f"{path}: spec_refs[{idx}] ({ref_type}:{ref_id}) is missing from context_pack.required_spec_refs" + ) + + return errors + + +def _validate_task_result_deep(repo_root: str, path: str, payload: dict) -> list[str]: + errors: list[str] = [] + phase = payload.get("phase") + status = payload.get("status") + artifacts = payload.get("artifacts", []) + + if status != "success" or phase not in {"16a", "16b", "16c"}: + return errors + if not isinstance(artifacts, list) or not artifacts: + return errors + + step16_artifacts: list[tuple[str, dict, str]] = [] + for ref in artifacts: + if not isinstance(ref, str) or not ref or not ref.endswith(".json"): + continue + resolved = _resolve_ref(path, ref, repo_root) + if not os.path.exists(resolved): + continue + try: + with open(resolved, "r", encoding="utf-8") as f: + payload_json = json.load(f) + except Exception: + continue + if payload_json.get("$schema") == "https://specdev.local/schema/16_impl_context.schema.json": + step16_artifacts.append((ref, payload_json, resolved)) + + if not step16_artifacts: + errors.append( + f"{path}: success task_result for phase '{phase}' must include at least one 16_impl_context artifact reference" + ) + return errors + + for artifact_ref, artifact, resolved_path in step16_artifacts: + # A success task_result must never point at a malformed Step 16 artifact. + # Validate referenced artifacts fully (schema + deep checks) before phase gates. + from .validate import validate_file + + step16_validation_errors = validate_file(repo_root, resolved_path) + if step16_validation_errors: + errors.append( + f"{path}: phase {phase} success artifact '{artifact_ref}' failed step16 validation: " + + "; ".join(step16_validation_errors) + ) + continue + + if phase == "16a": + plan = artifact.get("plan") + checklist = ( + plan.get("spec_alignment", {}).get("checklist", []) + if isinstance(plan, dict) + else [] + ) + if not isinstance(plan, dict) or not isinstance(checklist, list) or not checklist: + errors.append( + f"{path}: phase 16a success artifact '{artifact_ref}' must include a non-empty plan.spec_alignment.checklist" + ) + + if phase == "16b": + execution = artifact.get("execution") + execution_results = execution.get("execution_results", []) if isinstance(execution, dict) else [] + if not isinstance(execution, dict): + errors.append( + f"{path}: phase 16b success artifact '{artifact_ref}' must include execution section" + ) + elif not isinstance(execution_results, list) or not execution_results: + errors.append( + f"{path}: phase 16b success artifact '{artifact_ref}' must include non-empty execution.execution_results" + ) + + if phase == "16c": + review = artifact.get("review") + verdict = review.get("verdict") if isinstance(review, dict) else None + if not isinstance(review, dict): + errors.append( + f"{path}: phase 16c success artifact '{artifact_ref}' must include review section" + ) + elif verdict not in {"verified", "deferred", "rejected"}: + errors.append( + f"{path}: phase 16c success artifact '{artifact_ref}' must include review.verdict" + ) + elif verdict == "verified": + findings = review.get("findings", []) + if not isinstance(findings, list): + errors.append( + f"{path}: phase 16c success artifact '{artifact_ref}' has verdict=verified but review.findings is not a list" + ) + else: + major_or_blocking = [ + f + for f in findings + if isinstance(f, dict) and f.get("severity") in {"blocking", "major"} + ] + if major_or_blocking: + errors.append( + f"{path}: phase 16c success artifact '{artifact_ref}' has verdict=verified but includes blocking/major findings" + ) + execution = artifact.get("execution") + execution_results = execution.get("execution_results", []) if isinstance(execution, dict) else [] + if not isinstance(execution_results, list) or not execution_results: + errors.append( + f"{path}: phase 16c success artifact '{artifact_ref}' has verdict=verified but execution.execution_results is missing or empty" + ) + else: + non_passed = [ + r + for r in execution_results + if isinstance(r, dict) and r.get("status") in {"failed", "blocked", "partial"} + ] + if non_passed: + errors.append( + f"{path}: phase 16c success artifact '{artifact_ref}' has verdict=verified but execution results include failed/blocked/partial status" + ) + + return errors + + +def _extract_child_id_from_spawn_ref(spawn_ref: object) -> Optional[str]: + if not isinstance(spawn_ref, str) or not spawn_ref: + return None + m = re.search(r"/spawns/([^/]+)/task_input\.json$", spawn_ref.replace("\\", "/")) + if not m: + return None + return m.group(1) + + +def _extract_child_id_from_result_ref(result_ref: object) -> Optional[str]: + if not isinstance(result_ref, str) or not result_ref: + return None + m = re.search(r"/spawns/([^/]+)/task_result\.json$", result_ref.replace("\\", "/")) + if not m: + return None + return m.group(1) + + +def _validate_session_state_deep(repo_root: str, path: str, payload: dict) -> list[str]: + errors: list[str] = [] + + status = payload.get("status") + pending_child_id = payload.get("pending_child_id") + pending_spawn_ref = payload.get("pending_spawn_ref") + pending_questions = payload.get("pending_questions") + retry_counters = payload.get("retry_counters") + + if status == "waiting_child": + if not isinstance(pending_child_id, str) or not pending_child_id: + errors.append(f"{path}: status waiting_child requires non-null pending_child_id") + if not isinstance(pending_spawn_ref, str) or not pending_spawn_ref: + errors.append(f"{path}: status waiting_child requires non-null pending_spawn_ref") + if pending_questions is not None and pending_questions != []: + errors.append(f"{path}: status waiting_child requires pending_questions to be null/empty") + elif status == "awaiting_input": + if not isinstance(pending_child_id, str) or not pending_child_id: + errors.append(f"{path}: status awaiting_input requires non-null pending_child_id") + if not isinstance(pending_spawn_ref, str) or not pending_spawn_ref: + errors.append(f"{path}: status awaiting_input requires non-null pending_spawn_ref") + if not ( + isinstance(pending_questions, list) + and any(isinstance(q, str) and q.strip() for q in pending_questions) + ): + errors.append(f"{path}: status awaiting_input requires non-empty pending_questions") + elif status in {"idle", "done", "blocked"}: + if pending_child_id is not None: + errors.append(f"{path}: status {status} requires pending_child_id=null") + if pending_spawn_ref is not None: + errors.append(f"{path}: status {status} requires pending_spawn_ref=null") + if pending_questions is not None and pending_questions != []: + errors.append(f"{path}: status {status} requires pending_questions to be null/empty") + + if isinstance(pending_spawn_ref, str) and pending_spawn_ref: + if _is_escape_path(pending_spawn_ref): + errors.append(f"{path}: pending_spawn_ref must stay within repo root") + child_from_ref = _extract_child_id_from_spawn_ref(pending_spawn_ref) + if child_from_ref is None: + errors.append( + f"{path}: pending_spawn_ref must use canonical '/spawns//task_input.json' path" + ) + elif isinstance(pending_child_id, str) and pending_child_id and child_from_ref != pending_child_id: + errors.append( + f"{path}: pending_spawn_ref child_id '{child_from_ref}' does not match pending_child_id '{pending_child_id}'" + ) + + for ref_name in ("session_log_ref", "spawn_log_ref", "scratchpad_ref"): + ref = payload.get(ref_name) + if isinstance(ref, str) and ref and _is_escape_path(ref): + errors.append(f"{path}: {ref_name} '{ref}' escapes repo root") + + if not isinstance(retry_counters, dict): + errors.append(f"{path}: retry_counters must be an object") + else: + for key in ("planner", "builder", "verifier", "milestone"): + value = retry_counters.get(key) + if not isinstance(value, int) or value < 0: + errors.append(f"{path}: retry_counters.{key} must be integer >= 0") + + return errors + + +def _validate_spawn_log_deep(_repo_root: str, path: str, payload: dict) -> list[str]: + errors: list[str] = [] + entries = payload.get("entries", []) + + if not isinstance(entries, list): + return errors + + attempt_key_last: dict[tuple[str, str, str, tuple[str, ...]], int] = {} + seen_spawn_ids: set[str] = set() + for idx, entry in enumerate(entries): + if not isinstance(entry, dict): + continue + spawn_id = entry.get("spawn_id") + child_id = entry.get("child_id") + phase = entry.get("phase") + purpose = entry.get("purpose") + attempt = entry.get("attempt") + checklist_scope = entry.get("checklist_scope") + status = entry.get("status") + task_input_ref = entry.get("task_input_ref") + task_result_ref = entry.get("task_result_ref") + + if isinstance(spawn_id, str): + if spawn_id in seen_spawn_ids: + errors.append(f"{path}: entries[{idx}] duplicate spawn_id '{spawn_id}'") + seen_spawn_ids.add(spawn_id) + + if isinstance(task_input_ref, str): + child_from_input = _extract_child_id_from_spawn_ref(task_input_ref) + if child_from_input is None: + errors.append( + f"{path}: entries[{idx}] task_input_ref must use canonical '/spawns//task_input.json' path" + ) + elif isinstance(child_id, str) and child_id and child_from_input != child_id: + errors.append( + f"{path}: entries[{idx}] task_input_ref child '{child_from_input}' does not match child_id '{child_id}'" + ) + if isinstance(task_result_ref, str): + child_from_result = _extract_child_id_from_result_ref(task_result_ref) + if child_from_result is None: + errors.append( + f"{path}: entries[{idx}] task_result_ref must use canonical '/spawns//task_result.json' path" + ) + elif isinstance(child_id, str) and child_id and child_from_result != child_id: + errors.append( + f"{path}: entries[{idx}] task_result_ref child '{child_from_result}' does not match child_id '{child_id}'" + ) + + if status == "completed" and not (isinstance(task_result_ref, str) and task_result_ref): + errors.append(f"{path}: entries[{idx}] completed spawn must include task_result_ref") + if status == "spawned" and task_result_ref not in {None, ""}: + errors.append(f"{path}: entries[{idx}] spawned status must not include task_result_ref") + + scope_tuple: tuple[str, ...] = () + if isinstance(checklist_scope, list): + scope_tuple = tuple(sorted([x for x in checklist_scope if isinstance(x, str) and x])) + if all(isinstance(v, str) and v for v in (child_id, phase, purpose)) and isinstance(attempt, int): + key = (child_id, phase, purpose, scope_tuple) + last_attempt = attempt_key_last.get(key, 0) + if attempt > 0 and attempt < last_attempt: + errors.append( + f"{path}: entries[{idx}] attempt regressed for child '{child_id}' phase '{phase}' " + f"(found {attempt} after {last_attempt})" + ) + attempt_key_last[key] = max(last_attempt, attempt) + + return errors + + +def _compute_event_sha256(event: dict) -> str: + hash_payload = dict(event) + hash_payload["event_sha256"] = None + canonical = json.dumps(hash_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _stable_unit_interval(key: str) -> float: + digest = hashlib.sha256(key.encode("utf-8")).hexdigest() + return int(digest[:16], 16) / float(0xFFFFFFFFFFFFFFFF) + + +def _canonical_json_sha256(payload: dict) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _runtime_schema_sha256( + schema_registry: SchemaRegistry, + schema_uri: str, + cache: dict[str, Optional[str]], +) -> Optional[str]: + if schema_uri in cache: + return cache[schema_uri] + try: + payload = schema_registry.load(schema_uri) + except Exception: + cache[schema_uri] = None + return None + sha = _canonical_json_sha256(payload) + cache[schema_uri] = sha + return sha + + +def _as_non_negative_int(value: object) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if value >= 0 else None + return None + + +def _capture_policy_completeness_warnings(policy: object) -> list[str]: + warnings: list[str] = [] + if not isinstance(policy, dict): + return ["capture policy payload was not an object; using builtin defaults"] + + profile = policy.get("operating_profile") + if not isinstance(profile, dict): + warnings.append("capture policy operating_profile missing; default profile applied") + else: + for key in ("profile", "tier", "budget_tier"): + value = profile.get(key) + if not isinstance(value, str) or not value: + warnings.append(f"capture policy operating_profile.{key} missing; default applied") + + retention = policy.get("retention") + if not isinstance(retention, dict): + warnings.append("capture policy retention missing; default retention applied") + else: + for key in ("session_log_days", "capture_artifact_days", "eval_export_days"): + value = retention.get(key) + if not isinstance(value, int) or value < 1: + warnings.append(f"capture policy retention.{key} missing or invalid; default applied") + + budgets = policy.get("budgets") + budget_requirements = { + "context_window_token_target": 1024, + "full_capture_token_budget_per_run": 0, + "max_full_prompt_tokens_per_event": 0, + "max_full_completion_tokens_per_event": 0, + } + for key, minimum in budget_requirements.items(): + chosen = budgets.get(key) if isinstance(budgets, dict) else None + if not (isinstance(chosen, int) and chosen >= minimum): + chosen = policy.get(key) + if not (isinstance(chosen, int) and chosen >= minimum): + warnings.append(f"capture policy budget '{key}' missing or invalid; default applied") + return warnings + + +def _load_capture_policy( + repo_root: str, + session_log_path: str, + policy_ref: str, + cache: dict[str, tuple[Optional[dict], list[str]]], +) -> tuple[Optional[dict], list[str]]: + if policy_ref in cache: + return cache[policy_ref] + + policy_path = _resolve_ref(session_log_path, policy_ref, repo_root) + if not os.path.exists(policy_path): + result = (None, [f"capture_policy_ref not found: {policy_ref}"]) + cache[policy_ref] = result + return result + + try: + with open(policy_path, "r", encoding="utf-8") as f: + payload = json.load(f) + except Exception as e: + result = (None, [f"unable to read capture policy '{policy_ref}' ({e})"]) + cache[policy_ref] = result + return result + + schema_errors = _validate_payload( + repo_root, + policy_path, + payload, + RUNTIME_SCHEMA_BY_TYPE["log_capture_policy"], + ) + if schema_errors: + result = (None, [f"invalid capture policy '{policy_ref}'"] + schema_errors) + cache[policy_ref] = result + return result + + result = (payload, []) + cache[policy_ref] = result + return result + + +def _validate_session_event_log_deep(repo_root: str, path: str, events: list[tuple[int, dict]]) -> list[str]: + errors: list[str] = [] + expected_next_sequence = 1 + previous_hash: Optional[str] = None + capture_policy_cache: dict[str, tuple[Optional[dict], list[str]]] = {} + sampled_full_counts: dict[str, int] = {} + full_capture_tokens: dict[str, int] = {} + seen_tool_calls: dict[str, int] = {} + seen_result_ids: dict[str, int] = {} + spawn_counts: dict[str, int] = {} + terminate_counts: dict[str, int] = {} + spawn_sequences: dict[str, list[int]] = {} + terminate_sequences: dict[str, list[int]] = {} + validation_input_pass_sequences: dict[str, list[int]] = {} + validation_result_pass_sequences: dict[str, list[int]] = {} + schema_sha_cache: dict[str, Optional[str]] = {} + schema_registry = SchemaRegistry(repo_root) + + for line_no, event in events: + sequence = event.get("event_sequence") + event_type = event.get("event_type") + prev_hash = event.get("prev_event_sha256") + event_hash = event.get("event_sha256") + + if sequence != expected_next_sequence: + errors.append( + f"{path}:{line_no}: event_sequence must be contiguous; expected {expected_next_sequence}, found {sequence}" + ) + + if expected_next_sequence == 1: + if prev_hash is not None: + errors.append(f"{path}:{line_no}: first event must set prev_event_sha256 to null") + else: + if prev_hash != previous_hash: + errors.append( + f"{path}:{line_no}: prev_event_sha256 does not match previous event hash" + ) + + expected_hash = _compute_event_sha256(event) + if event_hash != expected_hash: + errors.append( + f"{path}:{line_no}: event_sha256 does not match canonical event payload hash" + ) + + content = event.get("content", {}) if isinstance(event.get("content"), dict) else {} + metadata = event.get("metadata", {}) if isinstance(event.get("metadata"), dict) else {} + step_id = event.get("step_id") + role = event.get("role") + parent_id = event.get("parent_id") + tool_call_id = event.get("tool_call_id") + result_id = event.get("result_id") + capture_level = content.get("capture_level") + capture_reason = content.get("capture_decision_reason") + prompt_ref = content.get("prompt_artifact_ref") + prompt_sha = content.get("prompt_sha256") + response_ref = content.get("response_artifact_ref") + response_sha = content.get("response_sha256") + tool_call = content.get("tool_call", {}) if isinstance(content.get("tool_call"), dict) else {} + token_usage = metadata.get("token_usage", {}) if isinstance(metadata.get("token_usage"), dict) else {} + prompt_tokens = _as_non_negative_int(token_usage.get("prompt")) or 0 + completion_tokens = _as_non_negative_int(token_usage.get("completion")) or 0 + total_tokens = _as_non_negative_int(token_usage.get("total")) or 0 + + if role in {"Orchestrator", "Planner", "Builder", "Verifier", "Worker"} and not ( + isinstance(step_id, str) and step_id + ): + errors.append( + f"{path}:{line_no}: role '{role}' requires non-null step_id for deterministic milestone lineage" + ) + + if expected_next_sequence > 1 and role != "Orchestrator" and parent_id is None: + errors.append( + f"{path}:{line_no}: non-root role '{role}' must include parent_id" + ) + + if event_type == "TOOL_CALL" and isinstance(tool_call_id, str): + if tool_call_id in seen_tool_calls: + errors.append( + f"{path}:{line_no}: duplicate TOOL_CALL tool_call_id '{tool_call_id}'" + ) + else: + seen_tool_calls[tool_call_id] = line_no + + if event_type == "TOOL_RESULT": + if isinstance(tool_call_id, str): + if tool_call_id not in seen_tool_calls: + errors.append( + f"{path}:{line_no}: TOOL_RESULT references unknown tool_call_id '{tool_call_id}'" + ) + if isinstance(result_id, str): + if result_id in seen_result_ids: + errors.append( + f"{path}:{line_no}: duplicate TOOL_RESULT result_id '{result_id}'" + ) + else: + seen_result_ids[result_id] = line_no + + if event_type in {"TOOL_CALL", "TOOL_RESULT"}: + tool_schema_context = ( + metadata.get("tool_schema_context") + if isinstance(metadata.get("tool_schema_context"), dict) + else None + ) + if tool_schema_context is None: + errors.append( + f"{path}:{line_no}: {event_type} must include metadata.tool_schema_context" + ) + else: + mode = tool_schema_context.get("mode") + expanded_tools = tool_schema_context.get("expanded_tool_names") + if mode == "catalog_plus_on_demand": + if event_type == "TOOL_CALL": + tool_name = tool_call.get("name") if isinstance(tool_call, dict) else None + if isinstance(tool_name, str): + if not isinstance(expanded_tools, list) or tool_name not in expanded_tools: + errors.append( + f"{path}:{line_no}: TOOL_CALL tool '{tool_name}' must appear in tool_schema_context.expanded_tool_names for mode 'catalog_plus_on_demand'" + ) + elif not isinstance(expanded_tools, list) or len(expanded_tools) == 0: + errors.append( + f"{path}:{line_no}: TOOL_RESULT with mode 'catalog_plus_on_demand' must include non-empty expanded_tool_names" + ) + + request_schema_uri = tool_schema_context.get("request_schema_uri") + request_schema_sha = tool_schema_context.get("request_schema_sha256") + result_schema_uri = tool_schema_context.get("result_schema_uri") + result_schema_sha = tool_schema_context.get("result_schema_sha256") + if isinstance(request_schema_uri, str) and isinstance(request_schema_sha, str): + expected_request_sha = _runtime_schema_sha256( + schema_registry, + request_schema_uri, + schema_sha_cache, + ) + if expected_request_sha and request_schema_sha != expected_request_sha: + errors.append( + f"{path}:{line_no}: tool_schema_context.request_schema_sha256 does not match canonical schema hash" + ) + if isinstance(result_schema_uri, str) and isinstance(result_schema_sha, str): + expected_result_sha = _runtime_schema_sha256( + schema_registry, + result_schema_uri, + schema_sha_cache, + ) + if expected_result_sha and result_schema_sha != expected_result_sha: + errors.append( + f"{path}:{line_no}: tool_schema_context.result_schema_sha256 does not match canonical schema hash" + ) + + spawn_ref = content.get("task_input_artifact_ref") + if event_type == "SPAWN" and isinstance(spawn_ref, str): + m = re.search(r"/spawns/([^/]+)/task_input\.json$", spawn_ref.replace("\\", "/")) + if m: + child_id = m.group(1) + spawn_counts[child_id] = spawn_counts.get(child_id, 0) + 1 + spawn_sequences.setdefault(child_id, []).append(sequence) + else: + errors.append( + f"{path}:{line_no}: SPAWN task_input_artifact_ref must use canonical '/spawns//task_input.json' path" + ) + + terminate_ref = content.get("task_result_artifact_ref") + if event_type == "TERMINATE" and isinstance(terminate_ref, str): + m = re.search(r"/spawns/([^/]+)/task_result\.json$", terminate_ref.replace("\\", "/")) + if m: + child_id = m.group(1) + terminate_counts[child_id] = terminate_counts.get(child_id, 0) + 1 + terminate_sequences.setdefault(child_id, []).append(sequence) + if terminate_counts[child_id] > spawn_counts.get(child_id, 0): + errors.append( + f"{path}:{line_no}: TERMINATE for child '{child_id}' appears without matching prior SPAWN" + ) + else: + errors.append( + f"{path}:{line_no}: TERMINATE task_result_artifact_ref must use canonical '/spawns//task_result.json' path" + ) + + if event_type == "VALIDATION": + validation = content.get("validation", {}) if isinstance(content.get("validation"), dict) else {} + schema_status = validation.get("schema") + deep_status = validation.get("deep_validator") + if schema_status == "pass" and deep_status == "pass": + validation_task_input_ref = content.get("task_input_artifact_ref") + if isinstance(validation_task_input_ref, str): + m = re.search( + r"/spawns/([^/]+)/task_input\.json$", + validation_task_input_ref.replace("\\", "/"), + ) + if m: + child_id = m.group(1) + validation_input_pass_sequences.setdefault(child_id, []).append(sequence) + else: + errors.append( + f"{path}:{line_no}: VALIDATION task_input_artifact_ref must use canonical '/spawns//task_input.json' path" + ) + + validation_task_result_ref = content.get("task_result_artifact_ref") + if isinstance(validation_task_result_ref, str): + m = re.search( + r"/spawns/([^/]+)/task_result\.json$", + validation_task_result_ref.replace("\\", "/"), + ) + if m: + child_id = m.group(1) + validation_result_pass_sequences.setdefault(child_id, []).append(sequence) + else: + errors.append( + f"{path}:{line_no}: VALIDATION task_result_artifact_ref must use canonical '/spawns//task_result.json' path" + ) + + if capture_level == "full": + if not all(isinstance(v, str) and v for v in (prompt_ref, prompt_sha, response_ref, response_sha)): + errors.append( + f"{path}:{line_no}: capture_level 'full' requires non-null prompt/response artifact refs and hashes" + ) + elif capture_level == "none": + if any(v is not None for v in (prompt_ref, prompt_sha, response_ref, response_sha)): + errors.append( + f"{path}:{line_no}: capture_level 'none' requires null prompt/response artifact refs and hashes" + ) + + if total_tokens != prompt_tokens + completion_tokens: + errors.append( + f"{path}:{line_no}: token_usage.total must equal token_usage.prompt + token_usage.completion" + ) + + secret_hits: list[str] = [] + secret_hits.extend(_detect_sensitive_classes(content.get("summary"))) + tool_result = content.get("tool_result", {}) if isinstance(content.get("tool_result"), dict) else {} + secret_hits.extend(_detect_sensitive_classes(tool_result.get("stdout_excerpt"))) + secret_hits.extend(_detect_sensitive_classes(tool_result.get("stderr_excerpt"))) + tool_call = content.get("tool_call", {}) if isinstance(content.get("tool_call"), dict) else {} + if tool_call: + try: + args_json = json.dumps(tool_call.get("args", {}), ensure_ascii=False) + except Exception: + args_json = "" + secret_hits.extend(_detect_sensitive_classes(args_json)) + if capture_level == "full": + secret_hits.extend(_artifact_sensitive_classes(path, prompt_ref, repo_root)) + secret_hits.extend(_artifact_sensitive_classes(path, response_ref, repo_root)) + if secret_hits: + classes = ", ".join(sorted(set(secret_hits))) + errors.append( + f"{path}:{line_no}: sensitive content detected in persisted session artifacts ({classes})" + ) + + redaction_applied = metadata.get("redaction_applied") + redaction_stats = metadata.get("redaction_stats", {}) if isinstance(metadata.get("redaction_stats"), dict) else {} + total_replacements = redaction_stats.get("total_replacements") + by_class = redaction_stats.get("by_class", {}) if isinstance(redaction_stats.get("by_class"), dict) else {} + classes_detected = redaction_stats.get("classes_detected", []) + by_class_sum = sum(v for v in by_class.values() if isinstance(v, int)) + + if isinstance(total_replacements, int): + if by_class_sum > total_replacements: + errors.append( + f"{path}:{line_no}: redaction_stats.by_class total exceeds redaction_stats.total_replacements" + ) + if redaction_applied is False and total_replacements > 0: + errors.append( + f"{path}:{line_no}: redaction_applied is false but redaction_stats.total_replacements is greater than 0" + ) + if isinstance(classes_detected, list): + missing_class_keys = [c for c in classes_detected if isinstance(c, str) and c not in by_class] + if missing_class_keys: + errors.append( + f"{path}:{line_no}: redaction_stats.classes_detected includes classes not present in by_class: {', '.join(missing_class_keys)}" + ) + + policy_ref = metadata.get("capture_policy_ref") + policy_sha = metadata.get("capture_policy_sha256") + if isinstance(policy_ref, str) and policy_ref: + policy, policy_errors = _load_capture_policy(repo_root, path, policy_ref, capture_policy_cache) + if policy_errors: + errors.extend([f"{path}:{line_no}: {msg}" for msg in policy_errors]) + elif policy is not None: + if isinstance(policy_sha, str): + expected_policy_sha = _canonical_json_sha256(policy) + if policy_sha != expected_policy_sha: + errors.append( + f"{path}:{line_no}: capture_policy_sha256 does not match canonical policy payload hash" + ) + + policy_id = policy.get("policy_id", "policy") + run_id = event.get("run_id", "run") + policy_run_key = f"{policy_id}|{run_id}" + event_type = event.get("event_type") + role = event.get("role") + default_capture_level = policy.get("default_capture_level", "summary") + always_full_events = set(policy.get("always_full_on_event_types", [])) + allowlist_roles = policy.get("full_capture_allowlist_roles", []) + role_allowed_for_full = (not allowlist_roles) or (role in allowlist_roles) + sampling_salt = policy.get("sampling_salt", "default") + + expected_capture_level = default_capture_level + expected_reason_prefix = "policy:default" + is_always_full = role_allowed_for_full and event_type in always_full_events + if is_always_full: + expected_capture_level = "full" + expected_reason_prefix = "policy:always_full" + else: + sample_rates = policy.get("sample_rate_by_event_type", {}) + sample_rate = sample_rates.get(event_type, 0.0) if isinstance(sample_rates, dict) else 0.0 + sample_rate = sample_rate if isinstance(sample_rate, (int, float)) else 0.0 + sampled_for_full = False + if role_allowed_for_full and sample_rate > 0: + sample_key = ( + f"{policy_id}|{sampling_salt}|{event.get('run_id')}|" + f"{event.get('event_id')}|{event.get('event_sequence')}" + ) + sampled_for_full = _stable_unit_interval(sample_key) < float(sample_rate) + if sampled_for_full: + max_full_events = policy.get("max_full_events_per_run", 0) + max_full_events = max_full_events if isinstance(max_full_events, int) else 0 + used = sampled_full_counts.get(policy_run_key, 0) + if used < max_full_events: + expected_capture_level = "full" + expected_reason_prefix = "policy:sampled" + sampled_full_counts[policy_run_key] = used + 1 + else: + expected_capture_level = policy.get("oversize_fallback", "summary") + expected_reason_prefix = "policy:capped" + + # Apply token guards after the initial decision to support 60k-80k context budgets. + if expected_capture_level == "full": + max_prompt_tokens = _as_non_negative_int(policy.get("max_full_prompt_tokens_per_event")) + max_completion_tokens = _as_non_negative_int(policy.get("max_full_completion_tokens_per_event")) + explicit_budget = _as_non_negative_int(policy.get("full_capture_token_budget_per_run")) + window_target = _as_non_negative_int(policy.get("context_window_token_target")) + window_fraction_raw = policy.get("max_full_capture_context_fraction") + derived_budget: Optional[int] = None + if ( + isinstance(window_fraction_raw, (int, float)) + and window_fraction_raw > 0 + and window_fraction_raw <= 1 + and isinstance(window_target, int) + ): + derived_budget = int(window_target * float(window_fraction_raw)) + + effective_budget: Optional[int] = None + for candidate in (explicit_budget, derived_budget): + if candidate is None: + continue + effective_budget = candidate if effective_budget is None else min(effective_budget, candidate) + + if isinstance(max_prompt_tokens, int) and prompt_tokens > max_prompt_tokens: + expected_capture_level = policy.get("oversize_fallback", "summary") + expected_reason_prefix = "policy:token_guard_prompt" + elif isinstance(max_completion_tokens, int) and completion_tokens > max_completion_tokens: + expected_capture_level = policy.get("oversize_fallback", "summary") + expected_reason_prefix = "policy:token_guard_completion" + else: + used_tokens = full_capture_tokens.get(policy_run_key, 0) + if isinstance(effective_budget, int) and (used_tokens + total_tokens > effective_budget): + expected_capture_level = policy.get("oversize_fallback", "summary") + expected_reason_prefix = "policy:token_budget" + else: + full_capture_tokens[policy_run_key] = used_tokens + total_tokens + + if capture_level != expected_capture_level: + errors.append( + f"{path}:{line_no}: capture_level '{capture_level}' does not match policy-expected level '{expected_capture_level}'" + ) + if not isinstance(capture_reason, str) or not capture_reason.startswith(expected_reason_prefix): + errors.append( + f"{path}:{line_no}: capture_decision_reason must start with '{expected_reason_prefix}'" + ) + if ( + policy.get("require_redaction_before_full") is True + and expected_capture_level == "full" + and redaction_applied is not True + ): + errors.append( + f"{path}:{line_no}: policy requires redaction_applied=true before full capture" + ) + + fallback_applied = metadata.get("capture_policy_fallback_applied") + fallback_reasons = metadata.get("capture_policy_fallback_reasons") + profile_meta = metadata.get("capture_policy_profile") + expected_fallback_warnings = _capture_policy_completeness_warnings(policy) + if isinstance(fallback_applied, bool): + if fallback_applied and not ( + isinstance(fallback_reasons, list) + and any(isinstance(reason, str) and reason for reason in fallback_reasons) + ): + errors.append( + f"{path}:{line_no}: capture_policy_fallback_applied=true requires non-empty capture_policy_fallback_reasons" + ) + if (not fallback_applied) and isinstance(fallback_reasons, list) and any( + isinstance(reason, str) and reason for reason in fallback_reasons + ): + errors.append( + f"{path}:{line_no}: capture_policy_fallback_applied=false must not include capture_policy_fallback_reasons entries" + ) + expected_fallback_applied = bool(expected_fallback_warnings) + if fallback_applied != expected_fallback_applied: + errors.append( + f"{path}:{line_no}: capture_policy_fallback_applied does not match policy completeness expectations" + ) + if isinstance(fallback_reasons, list) and expected_fallback_warnings: + missing_expected = sorted( + set(expected_fallback_warnings) - {r for r in fallback_reasons if isinstance(r, str)} + ) + if missing_expected: + errors.append( + f"{path}:{line_no}: capture_policy_fallback_reasons missing expected entries: {', '.join(missing_expected[:3])}" + ) + if isinstance(profile_meta, dict): + for pkey in ("profile", "tier", "budget_tier"): + if not isinstance(profile_meta.get(pkey), str) or not profile_meta.get(pkey): + errors.append( + f"{path}:{line_no}: capture_policy_profile.{pkey} must be a non-empty string when capture_policy_profile is present" + ) + + expected_next_sequence += 1 + previous_hash = event_hash if isinstance(event_hash, str) else None + + # Every child spawn that is tracked by runtime artifact refs must close. + for child_id, spawn_total in spawn_counts.items(): + terminate_total = terminate_counts.get(child_id, 0) + if terminate_total < spawn_total: + errors.append( + f"{path}: child '{child_id}' has {spawn_total} SPAWN event(s) but only {terminate_total} TERMINATE event(s)" + ) + + # Enforce transaction boundaries around parent-child handoff: + # spawn must be followed by validated task_input, and terminate must close + # only after validated task_result for the same child span. + for child_id, spans in spawn_sequences.items(): + terminations = terminate_sequences.get(child_id, []) + input_passes = validation_input_pass_sequences.get(child_id, []) + result_passes = validation_result_pass_sequences.get(child_id, []) + for idx, spawn_seq in enumerate(spans): + terminate_seq = terminations[idx] if idx < len(terminations) else None + upper_bound = terminate_seq if isinstance(terminate_seq, int) else 10**18 + has_input_validation = any( + isinstance(vs, int) and vs > spawn_seq and vs <= upper_bound for vs in input_passes + ) + if not has_input_validation: + errors.append( + f"{path}: child '{child_id}' spawn sequence {spawn_seq} is missing pass VALIDATION for task_input_artifact_ref before termination" + ) + + if isinstance(terminate_seq, int): + has_result_validation = any( + isinstance(vs, int) and vs >= spawn_seq and vs <= terminate_seq for vs in result_passes + ) + if not has_result_validation: + errors.append( + f"{path}: child '{child_id}' terminate sequence {terminate_seq} is missing pass VALIDATION for task_result_artifact_ref in same transaction span" + ) + + return errors + + +def validate_runtime_file(repo_root: str, path: str, artifact_type: Optional[str] = None) -> list[str]: + resolved_type = artifact_type or detect_runtime_artifact_type(path) + if not resolved_type: + return [f"{path}: unable to infer trinity runtime artifact type"] + if resolved_type not in RUNTIME_SCHEMA_BY_TYPE: + return [f"{path}: unsupported trinity runtime artifact type '{resolved_type}'"] + + schema_uri = RUNTIME_SCHEMA_BY_TYPE[resolved_type] + + try: + if resolved_type == "session_event": + errors: list[str] = [] + valid_events: list[tuple[int, dict]] = [] + with open(path, "r", encoding="utf-8") as f: + for idx, raw_line in enumerate(f, start=1): + stripped = raw_line.strip() + if not stripped: + continue + try: + payload = json.loads(stripped) + except json.JSONDecodeError as e: + errors.append(f"{path}:{idx}: invalid json line ({e})") + continue + line_errors = _validate_payload(repo_root, path, payload, schema_uri, line_no=idx) + if line_errors: + errors.extend(line_errors) + continue + valid_events.append((idx, payload)) + if errors: + return errors + errors.extend(_validate_session_event_log_deep(repo_root, path, valid_events)) + return errors + + with open(path, "r", encoding="utf-8") as f: + payload = json.load(f) + errors = _validate_payload(repo_root, path, payload, schema_uri) + if errors: + return errors + if resolved_type == "task_input": + errors.extend(_validate_task_input_deep(repo_root, path, payload)) + if resolved_type == "task_result": + errors.extend(_validate_task_result_deep(repo_root, path, payload)) + if resolved_type == "context_pack": + errors.extend(_validate_context_pack_deep(repo_root, path, payload)) + if resolved_type == "session_state": + errors.extend(_validate_session_state_deep(repo_root, path, payload)) + if resolved_type == "spawn_log": + errors.extend(_validate_spawn_log_deep(repo_root, path, payload)) + return errors + except (OSError, json.JSONDecodeError) as e: + return [f"{path}: error during runtime validation - {e}"] diff --git a/tools/specdev_tools/validate.py b/tools/specdev_tools/validate.py index a5dc16a9..d0ac45db 100644 --- a/tools/specdev_tools/validate.py +++ b/tools/specdev_tools/validate.py @@ -38,6 +38,29 @@ def _get_prompt_path(path: str) -> str: return f"prompts/prompt_{step}*.md" return "prompts/*.md" + +def _get_step_from_schema(schema_uri: str | None) -> str: + """Extract step number from known schema URIs.""" + if not isinstance(schema_uri, str) or not schema_uri: + return "unknown" + + if schema_uri.endswith("/16_impl_context.schema.json"): + return "16" + if schema_uri.endswith("/15_scaffold.schema.json"): + return "15" + if schema_uri.endswith("/10_governance.schema.json"): + return "10" + if schema_uri.endswith("/04_fr_list.schema.json"): + return "04" + if schema_uri.endswith("/03_glossary.schema.json"): + return "03" + if schema_uri.endswith("/02_system_sketch.schema.json"): + return "02" + if schema_uri.endswith("/01_capabilities.schema.json"): + return "01" + + return "unknown" + def validate_file(repo_root: str, path: str) -> list[str]: registry = SchemaRegistry(repo_root) @@ -47,6 +70,14 @@ def validate_file(repo_root: str, path: str) -> list[str]: schema_uri = data.get("$schema") if not schema_uri: + # Trinity runtime protocol artifacts intentionally omit "$schema". + # When a known runtime artifact path is detected, validate against + # the dedicated trinity runtime schemas instead of failing early. + from .trinity_runtime_validate import maybe_validate_runtime_artifact + + runtime_errors = maybe_validate_runtime_artifact(repo_root, path) + if runtime_errors is not None: + return runtime_errors return [f"{path}: missing $schema. Please add schema reference to the top of the file"] schema = registry.load(schema_uri) @@ -64,13 +95,18 @@ def validate_file(repo_root: str, path: str) -> list[str]: ) errors = sorted(v.iter_errors(data_for_validation), key=lambda e: e.path) + # Prefer schema-driven step detection to avoid path-based bypasses for + # runtime milestone artifacts such as spec/impl_context/{step_id}.json. + step = _get_step_from_schema(schema_uri) + if step == "unknown": + step = _get_step_from_path(path) + # Enhance error messages with context enhanced_errors = [] for e in errors: error_msg = f"{path}:{'/'.join(map(str, e.path))}: {e.message}" # Add context about what to do next - step = _get_step_from_path(path) if step != "unknown": prompt_path = _get_prompt_path(path) error_msg += f"\n See: {prompt_path} for guidance on requirements" @@ -79,8 +115,6 @@ def validate_file(repo_root: str, path: str) -> list[str]: # Run deep logic checks for all steps # Deep checks are important for validating optional fields that are required by business logic - step = _get_step_from_path(path) - deep_errors = [] try: if step == "01": @@ -128,55 +162,6 @@ def validate_file(repo_root: str, path: str) -> list[str]: if deep_errors: enhanced_errors.extend([f"{path}: {e}" for e in deep_errors]) - step = _get_step_from_path(path) - - deep_errors = [] - try: - if step == "01": - # For Step 01, we need component IDs from step 02 if available - # For now, we pass None and let the validator handle it or just rely on schema - # In a real CLI run, we might want to load dependencies. - # However, the validator signature is (instance, toolkit_root, component_ids) - # We can try to load 02 if it exists relative to repo_root - # Try to find sketch relative to the file being validated (User Project) - sketch_path = os.path.join(os.path.dirname(path), "02_system_sketch.json") - if not os.path.exists(sketch_path): - # Fallback to toolkit root (Internal Testing) - sketch_path = os.path.join(repo_root, "spec", "02_system_sketch.json") - component_ids = None - if os.path.exists(sketch_path): - try: - with open(sketch_path) as f: - cid_data = json.load(f) - component_ids = {c.get("component_id") for c in cid_data.get("components", []) if c.get("component_id")} - except: - pass - deep_errors = step_01.validate_step_01(data, repo_root, component_ids) - - elif step == "02": - deep_errors = step_02.validate_step_02(data, repo_root) - - elif step == "03": - # Step 03 might need NFRs or Monitoring - deep_errors = step_03.validate_step_03(data, repo_root) - - elif step == "04": - deep_errors = step_04.validate_step_04(data, repo_root) - - elif step == "10": - deep_errors = step_10.validate_step_10(data, repo_root) - - elif step == "15": - deep_errors = step_15.validate_step_15(data, repo_root) - - elif step == "16": - deep_errors = step_16.validate_step_16(data, repo_root, path) - - except Exception as e: - deep_errors = [f"Deep Validation Critical Error: {str(e)}"] - - if deep_errors: - enhanced_errors.extend([f"{path}: {e}" for e in deep_errors]) return enhanced_errors except (OSError, json.JSONDecodeError, ValueError, KeyError, AttributeError, TypeError) as e: diff --git a/tools/specdev_tools/validators/step_16.py b/tools/specdev_tools/validators/step_16.py index 71a76bb3..089cffa4 100644 --- a/tools/specdev_tools/validators/step_16.py +++ b/tools/specdev_tools/validators/step_16.py @@ -1,6 +1,55 @@ -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Set, Tuple +import fnmatch +import hashlib import json import os +import re +import subprocess + + +_CORE_AUTHORITY_FILES = [ + "spec/04_fr_list.json", + "spec/05_interface_contracts.json", + "spec/06_invariants.json", + "spec/07_nfrs.json", + "spec/08_fixtures.json", + "spec/09_impl_plan.json", + "spec/10_governance.json", + "spec/11_redteam.json", + "spec/12_ci_gates.json", + "spec/13_extension_manifest.json", + "spec/13a_completeness_assessment.json", + "spec/14_roadmap.json", + "spec/15_scaffold.json", +] + +_SPEC_REF_TYPE_BY_BASENAME = { + "04_fr_list.json": "fr", + "05_interface_contracts.json": "api", + "06_invariants.json": "inv", + "07_nfrs.json": "nfr", + "08_fixtures.json": "fixture", +} + +_KNOWN_TYPED_SPEC_REFS = {"fr", "api", "nfr", "inv", "fixture"} + +_SENSITIVE_PATTERNS = { + "private_key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |)?PRIVATE KEY-----"), + "aws_access_key": re.compile(r"\bAKIA[0-9A-Z]{16}\b"), + "aws_secret_access_key": re.compile(r"\baws[_-]?secret[_-]?access[_-]?key\b[^\n]{0,40}[A-Za-z0-9/+=]{40}"), + "openai_key": re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), + "github_pat": re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), + "gitlab_pat": re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b"), + "google_api_key": re.compile(r"\bAIza[0-9A-Za-z\-_]{35}\b"), + "npm_token": re.compile(r"\bnpm_[A-Za-z0-9]{36}\b"), + "slack_token": re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b"), + "bearer_token": re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._\-]{20,}\b"), + "jwt_token": re.compile(r"\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b"), + "generic_secret_assignment": re.compile( + r"(?i)\b(api[_-]?key|secret|token|password)\b\s*[:=]\s*['\"]?[A-Za-z0-9_\-./+=]{16,}" + ), +} + def _find_seed_manifest(spec_path: Optional[str], toolkit_root: str) -> Optional[str]: if spec_path: @@ -20,97 +69,518 @@ def _find_seed_manifest(spec_path: Optional[str], toolkit_root: str) -> Optional return None +def _is_fixture_validation(spec_path: Optional[str]) -> bool: + if not spec_path: + return False + normalized = spec_path.replace("\\", "/") + return "/tests/fixtures/" in normalized + + +def _project_root_from_manifest(manifest_path: str) -> str: + # seed_manifest.json is expected at /spec/common/seed_manifest.json + return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(manifest_path)))) + + +def _find_git_root(spec_path: Optional[str], toolkit_root: str) -> Optional[str]: + def walk_up(start: str) -> Optional[str]: + cur = os.path.abspath(start) + while True: + if os.path.isdir(os.path.join(cur, ".git")): + return cur + parent = os.path.dirname(cur) + if parent == cur: + break + cur = parent + return None + + if spec_path: + root = walk_up(os.path.dirname(spec_path)) + if root: + return root + return walk_up(toolkit_root) + + +def _collect_spec_refs(node: Any, refs: List[Dict[str, Any]]) -> None: + if isinstance(node, dict): + if {"type", "id", "line_range", "commit_hash"}.issubset(node.keys()): + if isinstance(node.get("id"), str): + refs.append(node) + for value in node.values(): + _collect_spec_refs(value, refs) + elif isinstance(node, list): + for item in node: + _collect_spec_refs(item, refs) + + +def _collect_ids(node: Any, source_rel: str, ids: Set[str], id_to_paths: Dict[str, Set[str]]) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key == "id" and isinstance(value, str) and value: + ids.add(value) + id_to_paths.setdefault(value, set()).add(source_rel) + _collect_ids(value, source_rel, ids, id_to_paths) + elif isinstance(node, list): + for item in node: + _collect_ids(item, source_rel, ids, id_to_paths) + + +def _build_authority_index( + manifest_path: str, +) -> Tuple[ + Set[str], + Dict[str, Set[str]], + Dict[str, Set[str]], + Dict[Tuple[str, str], Set[str]], +]: + all_ids: Set[str] = set() + all_id_to_paths: Dict[str, Set[str]] = {} + typed_ids: Dict[str, Set[str]] = {} + typed_id_to_paths: Dict[Tuple[str, str], Set[str]] = {} + + project_root = _project_root_from_manifest(manifest_path) + with open(manifest_path, "r", encoding="utf-8") as f: + manifest: Dict[str, Any] = json.load(f) + + authority_paths: Set[str] = set() + + seeds_by_id: Dict[str, str] = {} + for seed in manifest.get("seeds", []): + if isinstance(seed, dict): + sid = seed.get("seed_id") + path = seed.get("path") + if isinstance(sid, str) and isinstance(path, str): + seeds_by_id[sid] = path + + step_requirements = manifest.get("step_requirements", {}) + required_seed_ids: Set[str] = set() + for step_key in ("16", "16a", "16b", "16c"): + seed_ids = step_requirements.get(step_key, []) + if isinstance(seed_ids, list): + for seed_id in seed_ids: + if isinstance(seed_id, str): + required_seed_ids.add(seed_id) + + for seed_id in required_seed_ids: + rel_path = seeds_by_id.get(seed_id) + if isinstance(rel_path, str) and rel_path.endswith(".json"): + authority_paths.add(os.path.join(project_root, rel_path)) + + for rel in _CORE_AUTHORITY_FILES: + authority_paths.add(os.path.join(project_root, rel)) + + for abs_path in sorted(authority_paths): + if not os.path.exists(abs_path): + continue + try: + with open(abs_path, "r", encoding="utf-8") as f: + payload = json.load(f) + except Exception: + continue + + rel_path = os.path.relpath(abs_path, project_root).replace("\\", "/") + local_ids: Set[str] = set() + local_id_to_paths: Dict[str, Set[str]] = {} + _collect_ids(payload, rel_path, local_ids, local_id_to_paths) + + for item_id in local_ids: + all_ids.add(item_id) + all_id_to_paths.setdefault(item_id, set()).add(rel_path) + + spec_ref_type = _SPEC_REF_TYPE_BY_BASENAME.get(os.path.basename(rel_path)) + if spec_ref_type: + typed_ids.setdefault(spec_ref_type, set()).update(local_ids) + for item_id in local_ids: + typed_id_to_paths.setdefault((spec_ref_type, item_id), set()).add(rel_path) + + return all_ids, all_id_to_paths, typed_ids, typed_id_to_paths + + +def _git_commit_exists(git_root: str, commit_hash: str, cache: Dict[str, bool]) -> bool: + if commit_hash in cache: + return cache[commit_hash] + result = subprocess.run( + ["git", "cat-file", "-e", f"{commit_hash}^{{commit}}"], + cwd=git_root, + capture_output=True, + text=True, + check=False, + ) + cache[commit_hash] = result.returncode == 0 + return cache[commit_hash] + + +def _git_file_lines( + git_root: str, + commit_hash: str, + rel_path: str, + cache: Dict[Tuple[str, str], Optional[List[str]]], +) -> Optional[List[str]]: + key = (commit_hash, rel_path) + if key in cache: + return cache[key] + + result = subprocess.run( + ["git", "show", f"{commit_hash}:{rel_path}"], + cwd=git_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + cache[key] = None + return None + + cache[key] = result.stdout.splitlines() + return cache[key] + + +def _parse_line_range(value: str) -> Optional[Tuple[int, int]]: + match = re.match(r"^L(\d+)-L(\d+)$", value or "") + if not match: + return None + start = int(match.group(1)) + end = int(match.group(2)) + return start, end + + +def _line_range_contains_reference_id(lines: List[str], start: int, end: int, ref_id: str) -> bool: + excerpt = "\n".join(lines[start - 1 : end]) + id_field_pattern = re.compile(rf'"id"\s*:\s*"{re.escape(ref_id)}"') + quoted_id_pattern = re.compile(rf'"{re.escape(ref_id)}"') + return bool(id_field_pattern.search(excerpt) or quoted_id_pattern.search(excerpt)) + + +def _normalize_test_command(entry: Any) -> Optional[str]: + if isinstance(entry, str): + normalized = entry.strip() + return normalized or None + if isinstance(entry, dict): + command = entry.get("command") + if isinstance(command, str): + normalized = command.strip() + return normalized or None + return None + + +def _detect_sensitive_classes(text: Any) -> List[str]: + if not isinstance(text, str) or not text: + return [] + hits: List[str] = [] + for cls, pattern in _SENSITIVE_PATTERNS.items(): + if pattern.search(text): + hits.append(cls) + return hits + + +def _validate_spec_ref_grounding( + data: Dict[str, Any], + toolkit_root: str, + spec_path: Optional[str], + manifest_path: Optional[str], +) -> List[str]: + errors: List[str] = [] + spec_refs: List[Dict[str, Any]] = [] + _collect_spec_refs(data, spec_refs) + + if not spec_refs: + return errors + + if not manifest_path or not os.path.exists(manifest_path): + errors.append("spec_ref grounding check failed: seed_manifest.json not found.") + return errors + + try: + all_ids, all_id_to_paths, typed_ids, typed_id_to_paths = _build_authority_index(manifest_path) + except Exception as e: + return [f"spec_ref grounding check failed: unable to build authority index ({e})."] + + if not all_ids: + errors.append("spec_ref grounding check failed: no authority IDs were indexed from governed artifacts.") + return errors + + git_root = _find_git_root(spec_path, toolkit_root) + if not git_root: + errors.append("spec_ref grounding check failed: git root not found.") + return errors + + commit_cache: Dict[str, bool] = {} + lines_cache: Dict[Tuple[str, str], Optional[List[str]]] = {} + + for idx, ref in enumerate(spec_refs, start=1): + ref_type = ref.get("type", "unknown") + ref_id = ref.get("id", "") + line_range = ref.get("line_range", "") + commit_hash = ref.get("commit_hash", "") + + candidate_paths: List[str] = [] + + if ref_type in _KNOWN_TYPED_SPEC_REFS: + type_ids = typed_ids.get(ref_type, set()) + if not isinstance(ref_id, str) or ref_id not in type_ids: + errors.append( + f"spec_ref[{idx}] ({ref_type}:{ref_id}) is not grounded: id not found for type '{ref_type}' in authority artifacts." + ) + candidate_paths = sorted(typed_id_to_paths.get((ref_type, ref_id), set())) + else: + if not isinstance(ref_id, str) or ref_id not in all_ids: + errors.append( + f"spec_ref[{idx}] ({ref_type}:{ref_id}) is not grounded: id not found in authority artifacts." + ) + candidate_paths = sorted(all_id_to_paths.get(ref_id, set())) + + if not isinstance(commit_hash, str) or not _git_commit_exists(git_root, commit_hash, commit_cache): + errors.append( + f"spec_ref[{idx}] ({ref_type}:{ref_id}) is not grounded: commit_hash '{commit_hash}' not found in git." + ) + continue + + parsed = _parse_line_range(line_range if isinstance(line_range, str) else "") + if not parsed: + errors.append( + f"spec_ref[{idx}] ({ref_type}:{ref_id}) has invalid line_range '{line_range}'. Expected format Lx-Ly." + ) + continue + + start, end = parsed + if start < 1 or end < start: + errors.append( + f"spec_ref[{idx}] ({ref_type}:{ref_id}) has invalid line_range bounds '{line_range}'." + ) + continue + + if not candidate_paths: + continue + + plausible = False + for rel_path in candidate_paths: + lines = _git_file_lines(git_root, commit_hash, rel_path, lines_cache) + if not lines: + continue + if end > len(lines): + continue + if _line_range_contains_reference_id(lines, start, end, ref_id): + plausible = True + break + + if not plausible: + errors.append( + f"spec_ref[{idx}] ({ref_type}:{ref_id}) line_range '{line_range}' does not map to referenced authority object content at commit {commit_hash}." + ) + + return errors + + def validate_step_16(data: Dict[str, Any], toolkit_root: str, spec_path: Optional[str] = None) -> List[str]: """ Deep validation for Step 16 (Implementation Context). - + Args: data: The parsed JSON content of the step file. toolkit_root: The root directory of the toolkit (for resolving references). - + Returns: List of error messages. Empty list if valid. """ errors = [] - + plan = data.get("plan", {}) checklist = plan.get("spec_alignment", {}).get("checklist", []) docs_impact = plan.get("docs_impact") - + review_requirements = plan.get("review_requirements", {}) if isinstance(plan.get("review_requirements"), dict) else {} + review_test_commands_raw = review_requirements.get("test_commands", []) if isinstance(review_requirements.get("test_commands"), list) else [] + review_test_commands = [cmd for cmd in (_normalize_test_command(x) for x in review_test_commands_raw) if cmd] + review_test_command_set = set(review_test_commands) + active_checklist_expected_commands: Dict[str, Set[str]] = {} + for item in checklist: impl = item.get("implementation", {}) status = impl.get("status") item_id = item.get("id", "unknown") checklist_status = item.get("checklist_status", "active") - - # Logic Check: New checklist types and layers + linked_expectation = item.get("linked_test_expectation") + expected_commands: Set[str] = set() + if isinstance(linked_expectation, str): + normalized = linked_expectation.strip() + if normalized: + expected_commands.add(normalized) + elif isinstance(linked_expectation, list): + for entry in linked_expectation: + if isinstance(entry, str) and entry.strip(): + expected_commands.add(entry.strip()) + + if checklist_status != "deferred": + active_checklist_expected_commands[item_id] = expected_commands + if not expected_commands: + errors.append(f"Checklist item '{item_id}' is active but has no concrete linked_test_expectation command.") + else: + missing_from_review_plan = sorted(expected_commands - review_test_command_set) + if missing_from_review_plan: + errors.append( + f"Checklist item '{item_id}' has linked_test_expectation commands not present in plan.review_requirements.test_commands: " + + ", ".join(missing_from_review_plan) + ) + item_type = item.get("type", "") item_layer = item.get("layer", "") - + if item_type not in ["behavior", "constraint", "validation", "metadata", "perf", "logging", "docs", "security"]: - errors.append(f"Checklist item '{item_id}' has invalid type '{item_type}'. Must be one of: behavior, constraint, validation, metadata, perf, logging, docs, security") - + errors.append( + f"Checklist item '{item_id}' has invalid type '{item_type}'. Must be one of: behavior, constraint, validation, metadata, perf, logging, docs, security" + ) + if item_layer not in ["db", "model", "service", "api", "integration", "tests", "docs", "config", "security"]: - errors.append(f"Checklist item '{item_id}' has invalid layer '{item_layer}'. Must be one of: db, model, service, api, integration, tests, docs, config, security") - - # Logic Check: New checklist fields (nfr_refs, fixture_ref) required for non-deferred items + errors.append( + f"Checklist item '{item_id}' has invalid layer '{item_layer}'. Must be one of: db, model, service, api, integration, tests, docs, config, security" + ) + if checklist_status != "deferred": nfr_refs = item.get("nfr_refs", []) if not nfr_refs: errors.append(f"Checklist item '{item_id}' is not deferred but has no nfr_refs") - + fixture_ref = item.get("fixture_ref") if not fixture_ref: errors.append(f"Checklist item '{item_id}' is not deferred but has no fixture_ref") - - # Logic Check: Verified/In-Progress items must have actions + if status in ["verified", "in_progress"]: actions = impl.get("actions", []) if not actions and status == "verified": - # Strict check: Verified items must have actions documenting what was done - errors.append(f"Checklist item '{item_id}' is 'verified' but has no actions.") - - # Logic Check: Verified items must have evidence for at least one action if actions exist + errors.append(f"Checklist item '{item_id}' is 'verified' but has no actions.") + if status == "verified" and actions: - has_evidence = False - for action in actions: - if "evidence" in action: - has_evidence = True - break + has_evidence = any("evidence" in action for action in actions) if not has_evidence: errors.append(f"Checklist item '{item_id}' is 'verified' but contains no evidence in any action.") - # Logic Check: Ensure target_file_patterns cover touched files - summary_patterns = set(plan.get("summary", {}).get("target_file_patterns", [])) - - # Collect all files touched in actions - actually_touched = set() + summary_patterns = list(plan.get("summary", {}).get("target_file_patterns", [])) + + def is_path_covered_by_scope(path: str) -> bool: + return any(fnmatch.fnmatch(path, pattern) for pattern in summary_patterns) + + implementation_touched: Set[str] = set() for item in checklist: impl = item.get("implementation", {}) if impl.get("status") in ["in_progress", "verified"]: - for f in impl.get("files_touched", []): - actually_touched.add(f) - - # Logic: Warn if files are touched but not in target_file_patterns - # For now, we won't implement complex glob matching in this audit step because we don't want to add fnmatch overhead - # But we can check for direct matches if patterns are simple paths - # Or just skip this check if we want to be lenient. - # Let's implement a simple check: if the file is NOT in the patterns list EXACTLY, we flag it. - # This encourages specific file listing or explicit patterns. - # Note: This is a strict interpretation. If users rely on patterns like "*.py", this strict check will fail. - # So we probably should skip this strict check unless we import fnmatch. - - import fnmatch - - for f in actually_touched: - matched = False - for pattern in summary_patterns: - if fnmatch.fnmatch(f, pattern): - matched = True - break - - if not matched: - errors.append(f"File '{f}' is touched by implementation but not covered by target_file_patterns.") + for touched in impl.get("files_touched", []): + implementation_touched.add(touched) + + for touched in implementation_touched: + if not is_path_covered_by_scope(touched): + errors.append(f"File '{touched}' is touched by implementation but not covered by target_file_patterns.") + + execution = data.get("execution", {}) if isinstance(data.get("execution"), dict) else {} + execution_files_touched: Set[str] = set(execution.get("files_touched", [])) if isinstance(execution.get("files_touched"), list) else set() + for touched in execution_files_touched: + if not is_path_covered_by_scope(touched): + errors.append(f"File '{touched}' is touched by execution but not covered by target_file_patterns.") + + execution_results = execution.get("execution_results", []) if isinstance(execution.get("execution_results"), list) else [] + executed_command_set: Set[str] = set() + passed_command_set: Set[str] = set() + + for idx, result in enumerate(execution_results, start=1): + command = result.get("command", "") + normalized_command = command.strip() if isinstance(command, str) else "" + if normalized_command: + executed_command_set.add(normalized_command) + + status = result.get("status") + evidence = result.get("evidence") + evidence_ref = result.get("evidence_ref") + evidence_binding = result.get("evidence_binding", {}) if isinstance(result.get("evidence_binding"), dict) else {} + if isinstance(evidence, str) and evidence: + sensitive_hits = _detect_sensitive_classes(evidence) + if sensitive_hits: + errors.append( + f"execution.execution_results[{idx}] evidence contains sensitive content classes: {', '.join(sorted(set(sensitive_hits)))}." + ) + + if status == "passed": + if normalized_command: + passed_command_set.add(normalized_command) + + if isinstance(evidence, str) and evidence: + expected_sha = hashlib.sha256(evidence.encode("utf-8")).hexdigest() + actual_sha = evidence_binding.get("sha256") + if actual_sha != expected_sha: + errors.append( + f"execution.execution_results[{idx}] has invalid evidence_binding.sha256; expected hash of evidence content." + ) + expected_ref = f"sha256:{expected_sha}" + if evidence_ref != expected_ref: + errors.append( + f"execution.execution_results[{idx}] has invalid evidence_ref; expected '{expected_ref}'." + ) + else: + errors.append( + f"execution.execution_results[{idx}] is passed but evidence is missing or not a string." + ) + + binding_command = evidence_binding.get("command") + if isinstance(binding_command, str) and normalized_command and binding_command.strip() and binding_command.strip() != normalized_command: + errors.append( + f"execution.execution_results[{idx}] evidence_binding.command does not match command." + ) + + for item in checklist: + item_id = item.get("id", "unknown") + actions = ( + item.get("implementation", {}).get("actions", []) + if isinstance(item.get("implementation"), dict) + else [] + ) + if not isinstance(actions, list): + continue + for action_idx, action in enumerate(actions, start=1): + evidence_obj = action.get("evidence", {}) if isinstance(action, dict) else {} + if not isinstance(evidence_obj, dict): + continue + content = evidence_obj.get("content") + sensitive_hits = _detect_sensitive_classes(content) + if sensitive_hits: + errors.append( + f"Checklist item '{item_id}' action[{action_idx}] evidence.content contains sensitive content classes: " + + ", ".join(sorted(set(sensitive_hits))) + ) + + has_execution_context = bool(execution) + if has_execution_context and review_test_commands and not execution_results: + errors.append("execution.execution_results must be populated when execution context exists for active review test commands.") + if execution_results: + missing_executed = sorted(review_test_command_set - executed_command_set) + if missing_executed: + errors.append( + "execution.execution_results is missing required plan.review_requirements.test_commands: " + + ", ".join(missing_executed) + ) + + critical_evidence = execution.get("critical_evidence", {}) if isinstance(execution.get("critical_evidence"), dict) else {} + satisfied_checklist_ids = critical_evidence.get("satisfied_checklist_ids", []) if isinstance(critical_evidence.get("satisfied_checklist_ids"), list) else [] + checklist_ids = {item.get("id") for item in checklist if isinstance(item, dict) and isinstance(item.get("id"), str)} + unknown_satisfied_ids = sorted({cid for cid in satisfied_checklist_ids if isinstance(cid, str)} - checklist_ids) + if unknown_satisfied_ids: + errors.append( + "execution.critical_evidence.satisfied_checklist_ids contains unknown checklist IDs: " + + ", ".join(unknown_satisfied_ids) + ) + passed_test_commands = critical_evidence.get("passed_test_commands", []) if isinstance(critical_evidence.get("passed_test_commands"), list) else [] + normalized_passed_test_commands = { + cmd.strip() for cmd in passed_test_commands if isinstance(cmd, str) and cmd.strip() + } + if normalized_passed_test_commands: + missing_from_review_plan = sorted(normalized_passed_test_commands - review_test_command_set) + if missing_from_review_plan: + errors.append( + "execution.critical_evidence.passed_test_commands includes commands not present in plan.review_requirements.test_commands: " + + ", ".join(missing_from_review_plan) + ) + + for checklist_id, expected_commands in active_checklist_expected_commands.items(): + if execution_results and expected_commands: + if not (expected_commands & normalized_passed_test_commands): + errors.append( + f"Checklist item '{checklist_id}' has no matching passed command in execution.critical_evidence.passed_test_commands." + ) manifest_path = _find_seed_manifest(spec_path, toolkit_root) doc_patterns: List[str] = [] @@ -129,15 +599,14 @@ def validate_step_16(data: Dict[str, Any], toolkit_root: str, spec_path: Optiona errors.append("seed_manifest.json not found; cannot validate docs_impact doc paths.") def is_doc_path(path: str) -> bool: - if not path: - return False - if not doc_patterns_valid: + if not path or not doc_patterns_valid: return False norm = path.replace("\\", "/").lstrip("./") - for pattern in doc_patterns: - if fnmatch.fnmatch(norm, pattern): - return True - return False + return any(fnmatch.fnmatch(norm, pattern) for pattern in doc_patterns) + + planned_non_doc_targets = [ + path for path in summary_patterns if path and not is_doc_path(path) + ] code_change_targets = [] for item in checklist: @@ -148,19 +617,93 @@ def is_doc_path(path: str) -> bool: if target and not is_doc_path(target): code_change_targets.append(target) - if code_change_targets: + should_require_docs = bool(planned_non_doc_targets or code_change_targets) + + if should_require_docs: if not isinstance(docs_impact, dict): - errors.append("plan.docs_impact is required when code changes are present.") + errors.append("plan.docs_impact is required when non-doc implementation scope is present.") else: status = docs_impact.get("status") if status != "required": - errors.append("plan.docs_impact.status must be 'required' when code changes are present.") + errors.append("plan.docs_impact.status must be 'required' when non-doc implementation scope is present.") docs_touched = docs_impact.get("docs_touched", []) if not docs_touched: - errors.append("plan.docs_impact.docs_touched must be provided when code changes are present.") - elif doc_patterns_valid: + errors.append("plan.docs_impact.docs_touched must be provided when non-doc implementation scope is present.") + else: + out_of_scope_docs = [doc_path for doc_path in docs_touched if not is_path_covered_by_scope(doc_path)] + if out_of_scope_docs: + errors.append( + "plan.docs_impact.docs_touched includes paths outside plan.summary.target_file_patterns: " + + ", ".join(out_of_scope_docs) + ) + + if docs_touched and doc_patterns_valid: for doc_path in docs_touched: if not is_doc_path(doc_path): errors.append(f"plan.docs_impact.docs_touched contains non-doc path: {doc_path}") + if execution_files_touched: + missing_docs_in_execution = [ + doc_path for doc_path in docs_touched if doc_path not in execution_files_touched + ] + if missing_docs_in_execution: + errors.append( + "Execution touched code/spec scope but did not include documented updates in execution.files_touched: " + + ", ".join(missing_docs_in_execution) + ) + + delivery = plan.get("delivery") + if isinstance(delivery, dict) and delivery.get("status") == "planned": + review = data.get("review", {}) + delivery_status = review.get("delivery_status", {}) if isinstance(review, dict) else {} + deployments = delivery_status.get("deployments", []) if isinstance(delivery_status, dict) else [] + dashboards_verified = delivery_status.get("dashboards_verified", []) if isinstance(delivery_status, dict) else [] + alerts_verified = delivery_status.get("alerts_verified", []) if isinstance(delivery_status, dict) else [] + + if not (deployments or dashboards_verified or alerts_verified): + errors.append( + "plan.delivery.status is 'planned' but review.delivery_status has no verification entries " + "(expected deployments or dashboards_verified or alerts_verified)." + ) + + planned_dashboards = delivery.get("dashboards", []) if isinstance(delivery.get("dashboards"), list) else [] + if planned_dashboards: + planned_ids = { + item.get("dashboard_id") + for item in planned_dashboards + if isinstance(item, dict) and item.get("dashboard_id") + } + verified_ids = { + item.get("dashboard_id") + for item in dashboards_verified + if isinstance(item, dict) and item.get("dashboard_id") + } + missing = sorted(planned_ids - verified_ids) + if missing: + errors.append( + "plan.delivery.dashboards includes planned dashboards without matching review.delivery_status." + f"dashboards_verified entries: {', '.join(missing)}" + ) + + planned_alerts = delivery.get("alerts", []) if isinstance(delivery.get("alerts"), list) else [] + if planned_alerts: + planned_ids = { + item.get("alert_id") + for item in planned_alerts + if isinstance(item, dict) and item.get("alert_id") + } + verified_ids = { + item.get("alert_id") + for item in alerts_verified + if isinstance(item, dict) and item.get("alert_id") + } + missing = sorted(planned_ids - verified_ids) + if missing: + errors.append( + "plan.delivery.alerts includes planned alerts without matching review.delivery_status." + f"alerts_verified entries: {', '.join(missing)}" + ) + + if not _is_fixture_validation(spec_path): + errors.extend(_validate_spec_ref_grounding(data, toolkit_root, spec_path, manifest_path)) - return errors \ No newline at end of file + return errors From ae6b9d1c695b9e02b5b0145989f42378d776ffb1 Mon Sep 17 00:00:00 2001 From: Shantanu Agarwal Date: Sat, 14 Feb 2026 23:24:44 +0530 Subject: [PATCH 2/6] (spec): step 00 Charter --- .../00_charter/project_charter-trinity.json | 782 ++++++++++++++++++ toolkit_agent/spec/common/seed_manifest.json | 174 ++++ 2 files changed, 956 insertions(+) create mode 100644 toolkit_agent/spec/00_charter/project_charter-trinity.json create mode 100644 toolkit_agent/spec/common/seed_manifest.json diff --git a/toolkit_agent/spec/00_charter/project_charter-trinity.json b/toolkit_agent/spec/00_charter/project_charter-trinity.json new file mode 100644 index 00000000..504a5cca --- /dev/null +++ b/toolkit_agent/spec/00_charter/project_charter-trinity.json @@ -0,0 +1,782 @@ +{ + "id": "project_charter-trinity", + "owner": "engineering", + "created_at": "2026-02-14T00:00:00Z", + "title": "Trinity Automation System Specification", + "problem_statement": "Long-running agentic implementation loops suffer from context loss from window limits, spec drift from seeded constraints, hallucinated APIs/files/contracts, infinite repair loops without stop conditions, and unverified code changes without evidence-bound tests. These failures prevent reliable unattended AI-driven development execution across multi-milestone roadmaps. Local LLMs have shorter context token window and are not capable to run long sessions.", + "in_scope": [ + "Three-level fractal orchestration (L1 milestone, L2 persona, L3 atomic) with strict parent-child process boundaries", + "Disk-first two-phase artifact exchange contract (questions only → filesystem artifacts)", + "Checklist-first state machine with deterministic tool-call protocols and retry caps", + "Spec authority enforced through seed-manifest governance and SpecRefResolver", + "Evidence binding with verbatim excerpts, SHA-256 hashes, and evidence_ref format for all Step 16 execution results", + "Eval-grade structured session logging with deterministic replay metadata, validation gate lineage, and redaction profiles", + "Context pack budget enforcement with soft/hard token limits and truncation policies", + "Context management including SpecRefResolver lookups, context pack regeneration, and Spec drift detection", + "Three utility personas (Researcher, ToolUser, Summarizer, Auditor) with bounded context discovery and extraction-only evidence support", + "Checkpoint-based incremental commits with governance gates on every state transition and milestone closure", + "Terminal dashboard with three-panel live reporting for L1 orchestration monitoring", + "Session spawning and management with parent-child spawn protocols, spawn logging, and loop detection", + "Scratchpad-based state recovery and serialization for crash tolerance and token boundary handling", + "Session log rotation and compaction with configurable thresholds and archival preservation", + "Workspace artifact versioning (Draft→Audit→Refine loops) with versioned artifacts and archive cleanup", + "Secret safety enforcement including pre-persist scanning, redaction profiles, and denylist command patterns", + "Validation gate enforcement (schema, deep, governance, spec authority) on all state transitions and artifacts", + "Agent turn tracking and session event capture for replay and eval pipeline integration", + "Deterministic tool-call protocol with typed envelopes, schema validation, and tool argument safety", + "Spec drift detection and warnings for spec_ref references with commit hash and line_range grounding checks", + "Error recovery and blocker resolution with retry caps, blocked states, and human escalation paths", + "Agent failure detection and handling for timeouts, hallucinations, ambiguous conditions, and blocked states", + "Implementation loop completion tracking for L3 atomic units with retry caps and pass/fail verification", + "Spec baseline commit policy with controlled re-plan cycles for mid-run spec changes", + "SpecRefResolver error handling for missing or invalid spec_ref entries with explicit ambiguity findings", + "Workspace cleanup safety checks to prevent incorrect archival or deletion of debug artifacts", + "SpecRefResolver staleness detection for spec_ref references across git history and repository state" + ], + "out_of_scope": [ + "Real-time collaborative editing or multi-user concurrent session management", + "Machine learning model training or fine-tuning pipelines (eval-only dataset export)", + "External CI/CD integration beyond governance gate enforcement and checkpoint commits", + "Natural language query interface or conversational UI layer", + "Automated test generation or test suite optimization algorithms", + "Plugin ecosystem or third-party extension framework for custom tools", + "Automated deployment to cloud infrastructure or container orchestration", + "Real-time telemetry streaming to external monitoring platforms", + "Automated security vulnerability scanning beyond secret detection in logs", + "Automated performance profiling or optimization recommendations" + ], + "assumptions": [ + "Existing Git-based version control with commit hashes available for spec_ref grounding", + "OpenAI-compatible chat endpoints with configurable timeouts available for LLM strategy", + "Filesystem artifacts are the authoritative shared state for all state transitions", + "Seed-manifest governance is maintained and updated before each milestone run", + "Deterministic tool schemas are versioned and available at schema URI references", + "Parent agents consume child artifacts and never child chat transcripts", + "Context pack budget tokens are sufficient to cover required spec refs and seed files", + "SpecRefResolver can resolve git history, line ranges, and commit hashes for all spec_refs", + "Workspace artifact versioning supports Draft→Audit→Refine loops with rollback capability", + "Session logs can be rotated and archived without loss of replay metadata", + "Terminal dashboard does not require persistent storage or external dependencies", + "Secret scanning/redaction can be applied to persisted artifacts without blocking execution", + "Spec baseline commit policy allows controlled re-plan cycles when seed/spec changes are required mid-run", + "SpecRefResolver can handle missing or invalid spec_ref entries gracefully and surface explicit ambiguity findings", + "Git repository has sufficient history depth to resolve all required spec_ref commit hashes for the active milestone", + "SpecRefResolver line_range lookups are accurate and deterministic regardless of repository state (clean or with uncommitted changes)" + ], + "risks": [ + "Dependency readiness: OpenAI-compatible endpoint availability and latency may block unattended runs", + "Spec drift: Changes to seed-manifest or governed artifacts may cause context pack mismatches", + "Token budget exhaustion: Long-running LLM generations may exceed soft/hard token limits", + "Retry cap exhaustion: Infinite repair loops may exceed configured retry caps without human intervention", + "Secret leakage: Failed secret scanning/redaction may persist raw secrets to disk", + "Context loss: Long execution windows may lose critical state across resume boundaries", + "Schema drift: Runtime protocol schemas may diverge from prompt-side catalog contracts", + "Tool-call ambiguity: Compact tool catalog may be insufficient for complex deterministic planning", + "Evidence truncation: Long command outputs may lose critical pass/fail markers in evidence excerpts", + "Audit scope creep: L2 Collective Audit may miss cross-cutting quality issues across checklist items", + "SpecRefResolver failure: Missing or invalid spec_ref entries may block progress without clear remediation path", + "Spec baseline commit policy failure: Mid-run spec changes may require complex re-plan cycles that exceed retry caps", + "Agent hallucination: LLM may hallucinate APIs, file paths, or contracts that are not in governed spec artifacts", + "Workspace cleanup failure: Intermediate draft/audit versions may be incorrectly archived or deleted, losing evidence for debugging", + "SpecRefResolver stale content: Git history may contain stale spec_ref references that are not detected until runtime validation" + ], + "stakeholders": [ + { + "role": "Engineering Lead", + "needs": [ + "Milestone execution completes with verified code changes and evidence bindings", + "Unattended runs can resume from checkpoint without manual intervention", + "Schema validation gates prevent invalid artifacts from entering repository", + "Session logs provide deterministic replay for post-mortem analysis", + "Spec authority enforcement prevents unauthorized spec changes", + "Retry caps prevent infinite repair loops without human escalation", + "Error recovery success rate indicates system resilience", + "Blocker resolution rate ensures progress can be maintained", + "Implementation loop completion rate guarantees milestone progress" + ] + }, + { + "role": "DevOps Engineer", + "needs": [ + "Incremental commit checkpoints with governance gates prevent dirty working trees", + "Secret scanning/redaction prevents credential leakage in logs and artifacts", + "Token budget limits prevent runaway context inflation in long runs", + "Session log rotation and archival preserve replay metadata for long-term analysis", + "Workspace cleanup operations properly archive intermediate artifacts", + "Agent failure detection identifies runtime issues early", + "SpecRefResolver failures surface spec grounding problems" + ] + }, + { + "role": "Security Lead", + "needs": [ + "No raw secrets persist to disk in any artifact or log file", + "Redaction profiles can be tuned per environment (dev/staging/prod)", + "Secret detection confidence thresholds can be adjusted", + "Command denylist patterns prevent sensitive command execution", + "Pre-persist secret scanning blocks unsafe artifact writes", + "SpecRefResolver staleness detection prevents outdated spec references", + "Workspace cleanup safety prevents accidental secret leakage in archived artifacts" + ] + }, + { + "role": "QA Engineer", + "needs": [ + "Evidence excerpts contain verbatim pass/fail markers for test verification", + "Step 16 evidence fields are never paraphrased and include SHA-256 hashes", + "Session logs capture validation gate outcomes with full lineage metadata", + "Spec drift detection surfaces stale references before runtime execution", + "Session log completeness ensures all events are replayable and verifiable", + "Agent failure detection provides visibility into runtime issues", + "Error recovery success rate indicates system robustness" + ] + }, + { + "role": "Product Manager", + "needs": [ + "Milestone completion status is visible in terminal dashboard", + "Roadmap progress syncs automatically after verified milestones", + "Execution can be paused/resumed without losing state", + "Agent turn counts provide visibility into milestone effort and scope", + "Resume success rate indicates reliability of unattended operations", + "Implementation loop completion rate guarantees milestone progress", + "Error recovery success rate indicates system resilience" + ] + }, + { + "role": "ML Engineer / Eval Researcher", + "needs": [ + "Eval-grade structured session logs with deterministic replay metadata", + "OpenAI-style messages export for ML training dataset generation", + "Redaction profiles applied consistently across all persisted artifacts", + "Full lineage metadata (event_sequence, prev_event_sha256, artifact_sha256) for replay reconstruction", + "Session log rotation preserves archived segments for long-term eval pipelines", + "Agent failure detection provides clean data for eval dataset generation", + "SpecRefResolver errors surface spec grounding issues for eval improvement" + ] + }, + { + "role": "Platform Engineer", + "needs": [ + "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", + "Child spawn success rate indicates runtime stability", + "State transition success rate ensures reliable orchestration", + "Scratchpad recovery success rate enables crash-tolerant execution", + "Validation gate pass rate guarantees artifact quality before commit", + "Error recovery success rate indicates system robustness", + "Blocker resolution rate ensures progress can be maintained", + "Agent failure detection identifies runtime issues early" + ] + }, + { + "role": "Compliance Officer", + "needs": [ + "Spec authority enforcement prevents unauthorized spec modifications", + "Secret safety enforcement maintains auditability of sensitive data", + "Session logs capture complete execution history for regulatory review", + "Governance gate compliance ensures all commits meet policy requirements", + "Spec drift detection surfaces historical spec changes for compliance verification", + "Agent failure detection provides complete audit trail", + "Error recovery success rate indicates system reliability" + ] + }, + { + "role": "Site Reliability Engineer", + "needs": [ + "Error recovery success rate indicates system resilience", + "Blocker resolution rate ensures progress can be maintained", + "Agent failure detection identifies runtime issues early", + "SpecRefResolver failures surface spec grounding problems", + "Session log rotation prevents disk space exhaustion", + "Context budget utilization ensures optimal resource usage" + ] + } + ], + "user_segments": [ + { + "segment_id": "trinity-operator", + "description": "Platform engineers who invoke `specdev trinity` for unattended milestone execution", + "jobs_to_be_done": [ + "Execute one-milestone vertical slice from roadmap without manual intervention", + "Monitor execution progress via terminal dashboard", + "Resume interrupted runs from checkpoint", + "Review findings and verdicts after milestone completion", + "Debug blocked or deferred milestones via session logs", + "Configure context pack budgets and token limits for different milestone sizes", + "Review spec drift warnings and remediate stale references", + "Validate governance gate failures and override scope budgets when appropriate", + "Configure error recovery and retry cap parameters", + "Monitor implementation loop completion and agent failure detection", + "Review SpecRefResolver errors and address spec grounding issues", + "Handle spec baseline commit policy mid-run re-plan cycles" + ], + "pains": [ + "Context loss across long-running LLM generations", + "Infinite repair loops without clear stop conditions", + "Unverified code changes without evidence-backed tests", + "Hallucinated APIs or file paths breaking implementation", + "Spec drift causing unexpected behavior mid-run", + "Token budget exhaustion causing premature truncation", + "Retry cap exhaustion without human intervention", + "Secret leakage in persisted artifacts or logs", + "Agent failure detection failures masking runtime issues", + "Error recovery failures blocking progress", + "SpecRefResolver errors blocking execution without clear remediation" + ], + "gains": [ + "Lossless persistence allows resume from any checkpoint", + "Evidence bindings provide verifiable proof of test pass/fail", + "Spec authority prevents unauthorized changes to governed artifacts", + "Retry caps prevent infinite loops", + "Terminal dashboard provides real-time visibility into milestone state", + "Context pack budget enforcement prevents runaway token inflation", + "Spec drift detection surfaces stale references proactively", + "Secret safety enforcement prevents credential leakage", + "Agent failure detection identifies runtime issues early", + "Error recovery success rate indicates system resilience", + "Implementation loop completion rate guarantees milestone progress", + "SpecRefResolver errors surface spec grounding problems clearly" + ] + }, + { + "segment_id": "spec-author", + "description": "Domain experts who author and maintain governed specification artifacts", + "jobs_to_be_done": [ + "Update seed-manifest and governed spec files (FRs, NFRs, governance, etc.)", + "Maintain seed-manifest governance and authority set", + "Validate spec changes before committing as baseline", + "Review spec_ref grounding and line_range accuracy in implementation artifacts", + "Ensure spec drift detection identifies stale references", + "Add new spec artifacts to authority set with proper grounding", + "Update spec_ref line ranges when implementing changes", + "Review drift warnings after spec changes are committed", + "Handle SpecRefResolver staleness detection for historical references", + "Verify spec baseline commit policy allows controlled re-plan cycles" + ], + "pains": [ + "Spec changes breaking existing implementation traces", + "Spec_ref grounding failing due to missing or stale line ranges", + "Seed-manifest governance becoming outdated", + "Spec drift not detected until runtime failures occur", + "Manual verification of spec_ref commit hashes and line ranges", + "Reconciliation of spec changes across multiple milestones", + "SpecRefResolver staleness not detected until runtime", + "Spec baseline commit policy complexity for mid-run changes" + ], + "gains": [ + "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", + "Drift warnings surface stale references before runtime execution", + "Spec authority prevents unauthorized mutations to governed artifacts", + "Seed-manifest order ensures consistent context resolution across runs", + "Automated drift detection saves manual verification effort", + "Commit hash grounding ensures reproducible spec references", + "SpecRefResolver staleness detection surfaces historical changes", + "Spec baseline commit policy allows controlled re-plan cycles" + ] + }, + { + "segment_id": "eval-researcher", + "description": "Researchers and ML engineers who analyze execution traces for model training datasets", + "jobs_to_be_done": [ + "Export session logs to OpenAI-style messages format", + "Validate export rows against eval_export_row.schema.json", + "Apply deterministic redaction profiles to sensitive data", + "Create replay artifacts for dataset generation", + "Analyze validation gate outcomes and evidence bindings", + "Validate session log completeness and replay metadata", + "Create eval datasets from verified milestones", + "Analyze retry cap usage patterns and failure modes", + "Review agent failure detection events for eval improvement", + "Analyze error recovery success rates and failure patterns" + ], + "pains": [ + "Sensitive data (API keys, tokens) in session logs or prompt artifacts", + "Lack of structured export format for ML pipelines", + "Redaction applied inconsistently across different artifact types", + "Loss of replay metadata makes dataset reconstruction difficult", + "Incomplete session logs missing events or lineage", + "Redaction profiles not consistently applied across all artifacts", + "Agent failure detection events not captured in export", + "Error recovery failures not included in eval datasets" + ], + "gains": [ + "Eval strategy provides native event schema as source-of-truth", + "OpenAI-style messages export with validation", + "Deterministic redaction profiles applied to all persisted artifacts", + "Full lineage metadata (event_sequence, prev_event_sha256, artifact_sha256) for replay", + "Session log rotation preserves archival segments for long-term eval", + "Secret safety enforcement ensures export datasets are safe for external use", + "Agent failure detection events captured for eval improvement", + "Error recovery success rates tracked for eval dataset quality" + ] + }, + { + "segment_id": "platform-engineer", + "description": "Platform engineers responsible for Trinity runtime infrastructure and tooling", + "jobs_to_be_done": [ + "Implement and maintain SpecRefResolver with git grounding", + "Implement and maintain validation gate schemas and validators", + "Implement and maintain scratchpad serialization/deserialization", + "Implement and maintain session log rotation and archival", + "Implement and maintain secret scanning and redaction pipelines", + "Implement and maintain tool protocol schemas and validators", + "Monitor spawn success rates and state transition success rates", + "Debug validation gate failures and optimize performance", + "Implement error recovery and blocker resolution mechanisms", + "Implement agent failure detection and handling", + "Implement Spec baseline commit policy and re-plan cycles", + "Implement SpecRefResolver staleness detection" + ], + "pains": [ + "SpecRefResolver failures due to missing git history or invalid commit hashes", + "Validation gate performance bottlenecks in long-running sessions", + "Scratchpad corruption after crashes or token boundary handling", + "Session log rotation causing data loss or replay failures", + "Secret scanning false positives blocking legitimate artifacts", + "Tool protocol schema drift between prompt and runtime", + "Agent turn count explosion due to inefficient planning", + "Error recovery failures blocking progress", + "Blocker resolution failures preventing progress", + "Spec baseline commit policy complexity for re-plan cycles" + ], + "gains": [ + "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", + "Validation gates prevent invalid artifacts before commit", + "Scratchpad recovery ensures crash-tolerant execution", + "Session log rotation preserves archival segments without data loss", + "Secret scanning prevents leakage without false positives", + "Tool protocol schemas guarantee type safety and deterministic behavior", + "Spawn success and state transition success metrics indicate runtime health", + "Error recovery success rate indicates system resilience", + "Implementation loop completion rate guarantees milestone progress", + "Blocker resolution rate ensures progress can be maintained" + ] + }, + { + "segment_id": "compliance-officer", + "description": "Compliance officers who ensure Trinity execution adheres to organizational policies and regulations", + "jobs_to_be_done": [ + "Review session logs for compliance violations", + "Validate governance gate compliance on all commits", + "Verify spec authority enforcement prevents unauthorized changes", + "Ensure secret safety enforcement meets regulatory requirements", + "Audit spec drift detection for historical spec changes", + "Review session log completeness for audit trail requirements", + "Validate agent turn counts and execution effort for scope verification", + "Review error recovery success rates for system resilience", + "Verify agent failure detection captures all relevant events" + ], + "pains": [ + "Unable to trace execution history for audit requirements", + "Unauthorized spec changes slipping through governance gates", + "Credential leakage in logs or artifacts", + "Incomplete execution records missing events or lineage", + "Spec drift not detected until after compliance review", + "Governance gate failures not properly documented", + "Agent failure detection events not captured in audit trail", + "Error recovery failures not included in compliance review" + ], + "gains": [ + "Spec authority enforcement prevents unauthorized spec modifications", + "Secret safety enforcement maintains auditability of sensitive data", + "Session logs capture complete execution history for regulatory review", + "Governance gate compliance ensures all commits meet policy requirements", + "Spec drift detection surfaces historical spec changes for compliance verification", + "Session log completeness ensures replayable audit trails", + "Agent failure detection events captured for complete audit trail", + "Error recovery success rates tracked for compliance review" + ] + }, + { + "segment_id": "site-reliability-engineer", + "description": "Site Reliability Engineers responsible for system reliability, monitoring, and incident response", + "jobs_to_be_done": [ + "Monitor error recovery success rates and alert on failures", + "Track blocker resolution rates and alert on bottlenecks", + "Review agent failure detection events for incident response", + "Validate SpecRefResolver stability and error handling", + "Monitor session log rotation and archival for disk space management", + "Track context budget utilization for resource optimization", + "Review implementation loop completion for capacity planning", + "Validate governance gate compliance for system health" + ], + "pains": [ + "Error recovery failures blocking progress without visibility", + "Blocker resolution failures preventing milestone completion", + "Agent failure detection events not surfaced for incident response", + "SpecRefResolver instability causing runtime failures", + "Session log rotation causing disk space exhaustion", + "Context budget exhaustion causing premature truncation", + "Implementation loop completion failures indicating system issues" + ], + "gains": [ + "Error recovery success rate indicates system resilience", + "Blocker resolution rate ensures progress can be maintained", + "Agent failure detection identifies runtime issues early", + "SpecRefResolver errors surface spec grounding problems", + "Session log rotation prevents disk space exhaustion", + "Context budget utilization ensures optimal resource usage", + "Implementation loop completion rate guarantees milestone progress" + ] + } + ], + "success_metrics": [ + { + "metric_id": "trinity-m1-coverage", + "name": "Spec authority compliance", + "baseline": 95, + "target": 99, + "unit": "percent", + "measurement_method": "Validate that all tool calls and file writes reference governed seed-manifest artifacts; track violations in session logs and aggregate as percentage of total operations" + }, + { + "metric_id": "trinity-m2-evidence-binding", + "name": "Evidence binding coverage", + "baseline": 0, + "target": 100, + "unit": "percent of checklist items", + "measurement_method": "Count checklist items with valid evidence excerpts and SHA-256 hashes in execution.execution_results[]; divide by total checklist items for milestone" + }, + { + "metric_id": "trinity-m3-resume-success", + "name": "Resume success rate", + "baseline": 0, + "target": 95, + "unit": "percent of resumed runs", + "measurement_method": "Track resume attempts from checkpoint; record success if milestone completes within token budget and without context loss; measure success rate over 50 resume attempts" + }, + { + "metric_id": "trinity-m4-secret-detection", + "name": "Secret detection coverage", + "baseline": 0, + "target": 100, + "unit": "percent of persisted artifacts", + "measurement_method": "Scan all persisted prompt/response/session/artifact files using secret_scanner_v1; track detections and false negatives; calculate coverage as files scanned / files persisted" + }, + { + "metric_id": "trinity-m5-retry-cap-usage", + "name": "Retry cap utilization", + "baseline": 0, + "target": 20, + "unit": "percent of milestone runs", + "measurement_method": "Count milestone runs where retry caps are exceeded; divide by total milestone runs; track by state (16a/16b/16c) to identify failure modes" + }, + { + "metric_id": "trinity-m6-spawn-success-rate", + "name": "Child spawn success rate", + "baseline": 95, + "target": 99, + "unit": "percent of spawns", + "measurement_method": "Track all child spawns (16a/16b/16c/utility) and count successful spawns; divide by total spawns; exclude blocked/deferred states" + }, + { + "metric_id": "trinity-m7-state-transition-success-rate", + "name": "State transition success rate", + "baseline": 95, + "target": 99, + "unit": "percent of transitions", + "measurement_method": "Count valid state transitions (16a→16b→16c) that complete without validation failures; divide by total attempted transitions" + }, + { + "metric_id": "trinity-m8-validation-gate-pass-rate", + "name": "Validation gate pass rate", + "baseline": 90, + "target": 98, + "unit": "percent of gates", + "measurement_method": "Track all validation gates (schema, deep, governance, spec authority); count successful passes; divide by total gate evaluations" + }, + { + "metric_id": "trinity-m9-spec-drift-detection-coverage", + "name": "Spec drift detection coverage", + "baseline": 0, + "target": 100, + "unit": "percent of spec_ref references", + "measurement_method": "Count spec_ref records with successful drift detection (stale content or commit hash mismatch); divide by total spec_ref references in session logs" + }, + { + "metric_id": "trinity-m10-context-budget-utilization", + "name": "Context budget utilization", + "baseline": 0, + "target": 85, + "unit": "percent of hard token limit", + "measurement_method": "Calculate average context_pack token usage across all state transitions; divide by hard_token_limit; track by phase (16a/16b/16c)" + }, + { + "metric_id": "trinity-m11-context-truncation-events", + "name": "Context truncation events", + "baseline": 0, + "target": 5, + "unit": "events per milestone run", + "measurement_method": "Count context_budget_truncation events in session logs; track by cause (token overflow, priority item truncation)" + }, + { + "metric_id": "trinity-m12-specref-resolver-success-rate", + "name": "SpecRefResolver success rate", + "baseline": 95, + "target": 99, + "unit": "percent of lookups", + "measurement_method": "Count successful SpecRefResolver calls (path, line_range, commit_hash resolved) divided by total lookups; track failures by type" + }, + { + "metric_id": "trinity-m13-session-log-completeness", + "name": "Session log completeness", + "baseline": 90, + "target": 99, + "unit": "percent of events", + "measurement_method": "Validate session log events contain required fields (event_sequence, prev_event_sha256, event_sha256, artifact_ref, artifact_sha256); count complete events divided by total events" + }, + { + "metric_id": "trinity-m14-scratchpad-recovery-success", + "name": "Scratchpad recovery success rate", + "baseline": 0, + "target": 95, + "unit": "percent of recovery attempts", + "measurement_method": "Track scratchpad recovery attempts after crash or token boundary; count successful loads; measure over 50 recovery events" + }, + { + "metric_id": "trinity-m15-governance-gate-compliance", + "name": "Governance gate compliance", + "baseline": 95, + "target": 99, + "unit": "percent of commits", + "measurement_method": "Count commits passing governance checks (seed-manifest authority, spec_ref grounding, scope adherence) divided by total commits; track failures by gate type" + }, + { + "metric_id": "trinity-m16-agent-turn-count", + "name": "Agent turn count per milestone", + "baseline": 50, + "target": 100, + "unit": "turns", + "measurement_method": "Count total agent turns (messages sent/received) per milestone run; track distribution by state (16a/16b/16c/utility)" + }, + { + "metric_id": "trinity-m17-session-log-rotation-events", + "name": "Session log rotation events", + "baseline": 0, + "target": 3, + "unit": "events per milestone run", + "measurement_method": "Count session_log_compaction_threshold exceeded events; track by rotation cause (event count, time boundary)" + }, + { + "metric_id": "trinity-m18-workspace-cleanup-completeness", + "name": "Workspace cleanup completeness", + "baseline": 95, + "target": 100, + "unit": "percent of cleanup operations", + "measurement_method": "Track workspace cleanup operations after milestone closure; verify workspace artifacts and session logs are archived for future reference; count successful archive operations divided by total cleanup attempts; ensure logs are preserved, not deleted" + }, + { + "metric_id": "trinity-m19-spawn-efficiency-overhead", + "name": "Spawn efficiency and overhead", + "baseline": 5, + "target": 2, + "unit": "seconds per spawn", + "measurement_method": "Measure average time from spawn initiation to child task_input.json creation; track overhead reduction over time; exclude blocked/deferred spawns" + }, + { + "metric_id": "trinity-m20-spawn-utilization-rate", + "name": "Spawn utilization rate", + "baseline": 70, + "target": 90, + "unit": "percent of spawns", + "measurement_method": "Count effective spawns that contribute to milestone progress vs redundant or blocked spawns; divide effective spawns by total spawns; track by utility type (Researcher, ToolUser, Summarizer, Auditor)" + }, + { + "metric_id": "trinity-m21-spawn-loop-prevention-effectiveness", + "name": "Spawn loop prevention effectiveness", + "baseline": 0, + "target": 95, + "unit": "percent of duplicate spawns prevented", + "measurement_method": "Count duplicate spawn attempts detected by spawn_log loop detection; verify spawn_log prevents exceeding configured retry caps; measure prevention effectiveness over 100 milestone runs" + }, + { + "metric_id": "trinity-m22-error-recovery-success-rate", + "name": "Error recovery success rate", + "baseline": 0, + "target": 90, + "unit": "percent of error states", + "measurement_method": "Track error states (validation failures, schema violations, blocked conditions) and count successful recoveries; measure success rate over 100 error recovery attempts" + }, + { + "metric_id": "trinity-m23-blocker-resolution-rate", + "name": "Blocker resolution rate", + "baseline": 0, + "target": 85, + "unit": "percent of blockers", + "measurement_method": "Track blocker conditions (missing seed, out-of-scope writes, security concerns, retry cap exhaustion) and count successful resolutions before cap exhaustion; measure success rate over 100 blocker events" + }, + { + "metric_id": "trinity-m24-implementation-loop-completion-rate", + "name": "Implementation loop completion rate", + "baseline": 0, + "target": 95, + "unit": "percent of checklist units", + "measurement_method": "Count checklist units (atomic L3 loops) that complete successfully vs fail; track by state (16a/16b/16c); measure success rate over 100 units" + }, + { + "metric_id": "trinity-m25-agent-failure-detection-rate", + "name": "Agent failure detection rate", + "baseline": 95, + "target": 99, + "unit": "percent of agent failures", + "measurement_method": "Track agent failures (timeout, blocked, ambiguous, hallucination) and count successful detection in session logs; measure detection rate over 100 agent failure events" + } + ], + "links": [ + { + "source": "project_charter-trinity", + "target": "fr-16a-planner", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "fr-16b-builder", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "fr-16c-verifier", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-context-budget", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-evidence-integrity", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-secret-safety", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-spec-authority", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "seed-manifest-common", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "trinity_spec", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-session-management", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-validation-gates", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-spawn-protocol", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-agent-spawning", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-context-pack", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-secret-scanning", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-spawn-success-rate", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-state-transition-success", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-validation-gate-pass", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-spec-drift-detection", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-context-truncation", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-specref-resolver", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-session-log-completeness", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-scratchpad-recovery", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-governance-gate-compliance", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-agent-turn-count", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-session-rotation", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-workspace-cleanup", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-error-recovery-success", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-blocker-resolution-rate", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-implementation-loop-completion", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-agent-failure-detection", + "relation": "downstream" + } + ] +} \ No newline at end of file diff --git a/toolkit_agent/spec/common/seed_manifest.json b/toolkit_agent/spec/common/seed_manifest.json new file mode 100644 index 00000000..fe692fbd --- /dev/null +++ b/toolkit_agent/spec/common/seed_manifest.json @@ -0,0 +1,174 @@ +{ + "$schema": "https://specdev.local/schema/seed_manifest.schema.json", + "seed_manifest_id": "seed-manifest-common", + "version": "0.2.0", + "created_at": "2026-02-14T00:00:00Z", + "last_updated": "2026-02-14T17:30:00Z", + "global_seed_order": [ + "seed-charter", + "seed-capabilities", + "seed-system-sketch", + "seed-glossary", + "seed-fr-list", + "seed-docs", + "seed-tools", + "seed-tests" + ], + "nested_order": [ + { + "level_id": "docs", + "description": "Documentation and reference materials for developers and users.", + "seed_ids": [ + "seed-docs" + ] + }, + { + "level_id": "tools", + "description": "Tooling and utilities documentation.", + "seed_ids": [ + "seed-tools" + ] + }, + { + "level_id": "tests", + "description": "Test fixtures and integration test documentation.", + "seed_ids": [ + "seed-tests" + ] + } + ], + "seeds": [ + { + "seed_id": "seed-docs", + "path": "docs/README.md", + "description": "Main documentation index and entry point for developers.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-reference", + "path": "docs/developers/reference.md", + "description": "Developer reference and API documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-designs", + "path": "docs/designs/trinity_spec.md", + "description": "Trinity specification design document.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-migration", + "path": "docs/developers/workflows/workflow_migration.md", + "description": "Migration workflow documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-bootstrap", + "path": "docs/developers/workflows/workflow_bootstrap_legacy.md", + "description": "Legacy bootstrap workflow documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-tooling", + "path": "docs/developers/tools/schema_differ.md", + "description": "Schema differ tool documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-observability", + "path": "docs/developers/tools/trinity_observability.md", + "description": "Trinity observability documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-gap", + "path": "docs/developers/tooling/gap_hunter_checklist.md", + "description": "Gap hunter checklist for requirements coverage.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-coverage", + "path": "docs/developers/tooling/coverage_matrix.md", + "description": "Coverage matrix for specification testing.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-changelog", + "path": "docs/developers/tools/changelog_parser.md", + "description": "Changelog parser tool documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-align", + "path": "docs/developers/tools/align.md", + "description": "Alignment tool documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-docs-system", + "path": "docs/audit/review_report_03_docs.md", + "description": "System review report documentation.", + "required": true, + "source_type": "doc" + }, + { + "seed_id": "seed-tools", + "path": "tools/README.md", + "description": "CLI tools and utilities documentation.", + "required": true, + "source_type": "doc" + } + ], + "step_requirements": { + + }, + "docs_policy": { + "readme_required": true, + "root_readme_required": true, + "readme_depth_default": 0, + "readme_depth_by_scope": { + "toolkit_agent/": 1, + "spec/": 0, + "docs/": 1, + "tools/": 1, + "tests/": 0, + "schema/": 0 + }, + "scope": [ + "toolkit_agent/", + "spec/", + "docs/", + "tools/", + "schema/", + "seed_templates/", + "tests/" + ], + "exclusions": [ + "node_modules/", + ".git/", + ".venv/", + "__pycache__/", + "dist/", + "build/", + "coverage/", + ".pytest_cache/", + "*.egg-info/" + ], + "doc_paths": [ + "docs/**", + "README.md", + "CHANGELOG.md" + ] + } +} \ No newline at end of file From 33f15baebac8078f3917a2ffc17dd2a7602bf02d Mon Sep 17 00:00:00 2001 From: Shantanu Agarwal Date: Sun, 15 Feb 2026 00:04:20 +0530 Subject: [PATCH 3/6] version bump --- changelog/v0.3.0.md | 16 ++++++++++++++++ changelog/v0.3.0.yaml | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 changelog/v0.3.0.md create mode 100644 changelog/v0.3.0.yaml diff --git a/changelog/v0.3.0.md b/changelog/v0.3.0.md new file mode 100644 index 00000000..7380a925 --- /dev/null +++ b/changelog/v0.3.0.md @@ -0,0 +1,16 @@ +# v0.3.0 (2026-02-14) + +## Trinity Spec Optimization & Disk-Only Prompts + +This release focuses on optimizing the Trinity Specification for better token efficiency and refactoring all prompts to enforce "disk-only" artifact creation, removing older "fenced code block" output styles. + +### Optimized +- **Trinity Specification**: Refactored `trinity_spec.md` to reduce token count while maintaining all normative rules. Replaced inline JSON with schema references. +- **Project Charter**: Updated `project_charter-trinity.json` structure. + +### Changed +- **Prompts**: Refactored `00_project_charter.md` through `06_invariants.md` (and others) to strictly instruct Agents to write artifacts to disk using tools, rather than outputting markdown code blocks. +- **Workflow**: Agents now operate in a more autonomous "disk-first" mode. + +### Fixed +- **Schema Validation**: Various schema fixes in `spec` files to align with the new stricter validation rules. diff --git a/changelog/v0.3.0.yaml b/changelog/v0.3.0.yaml new file mode 100644 index 00000000..0f6e55dd --- /dev/null +++ b/changelog/v0.3.0.yaml @@ -0,0 +1,14 @@ +version: "0.3.0" +release_date: "2026-02-14" +changes: + - type: usage_update + step_id: "all" + description: "Prompts now enforce disk-only artifact creation. Agents should not expect or generate fenced code blocks for artifacts." + migration: + action: manual + details: "Ensure any custom agent workflows are updated to look for files on disk instead of parsing chat output." + - type: optimization + step_id: "trinity_spec" + description: "Trinity Spec refactored for token efficiency (approx 30% reduction)." + migration: + action: auto From 57d2437bea26a4b3009b4c491ac2303ac4ca6f84 Mon Sep 17 00:00:00 2001 From: Shantanu Agarwal Date: Sun, 15 Feb 2026 00:05:46 +0530 Subject: [PATCH 4/6] Version bump --- CHANGELOG.md | 1 + tools/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c232a26a..189f029b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All detailed version records are stored in the [`changelog/`](./changelog/) dire | Version | Release Date | Documentation | Migration Spec | Status | | :--- | :--- | :--- | :--- | :--- | +| **[0.3.0]** | 2026-02-14 | [v0.3.0.md](changelog/v0.3.0.md) | [v0.3.0.yaml](changelog/v0.3.0.yaml) | ⚠️ **Breaking** (Spec Optimization + Prompt Refactor) | | **[0.2.3]** | 2026-02-12 | [v0.2.3.md](changelog/v0.2.3.md) | [v0.2.3.yaml](changelog/v0.2.3.yaml) | ⚠️ **Breaking** | | **[0.2.2]** | 2026-02-12 | [v0.2.2.md](changelog/v0.2.2.md) | [v0.2.2.yaml](changelog/v0.2.2.yaml) | ✅ Patch | | **[0.2.1]** | 2026-02-07 | [v0.2.1.md](changelog/v0.2.1.md) | [v0.2.1.yaml](changelog/v0.2.1.yaml) | ⚠️ **Breaking** | diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 59c106d0..bfe561f4 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "specdev_tools" -version = "0.2.3" +version = "0.3.0" description = "Internal tooling for DevSpec Toolkit" authors = [{name = "Vichitra Collective"}] license = {text = "MIT"} From a56e0dd7c2f2c5003d0def813004dadf86e945e1 Mon Sep 17 00:00:00 2001 From: Shantanu Agarwal Date: Sun, 15 Feb 2026 00:06:03 +0530 Subject: [PATCH 5/6] correcting spec path --- .../00_charter/project_charter-trinity.json | 782 ------------------ 1 file changed, 782 deletions(-) delete mode 100644 toolkit_agent/spec/00_charter/project_charter-trinity.json diff --git a/toolkit_agent/spec/00_charter/project_charter-trinity.json b/toolkit_agent/spec/00_charter/project_charter-trinity.json deleted file mode 100644 index 504a5cca..00000000 --- a/toolkit_agent/spec/00_charter/project_charter-trinity.json +++ /dev/null @@ -1,782 +0,0 @@ -{ - "id": "project_charter-trinity", - "owner": "engineering", - "created_at": "2026-02-14T00:00:00Z", - "title": "Trinity Automation System Specification", - "problem_statement": "Long-running agentic implementation loops suffer from context loss from window limits, spec drift from seeded constraints, hallucinated APIs/files/contracts, infinite repair loops without stop conditions, and unverified code changes without evidence-bound tests. These failures prevent reliable unattended AI-driven development execution across multi-milestone roadmaps. Local LLMs have shorter context token window and are not capable to run long sessions.", - "in_scope": [ - "Three-level fractal orchestration (L1 milestone, L2 persona, L3 atomic) with strict parent-child process boundaries", - "Disk-first two-phase artifact exchange contract (questions only → filesystem artifacts)", - "Checklist-first state machine with deterministic tool-call protocols and retry caps", - "Spec authority enforced through seed-manifest governance and SpecRefResolver", - "Evidence binding with verbatim excerpts, SHA-256 hashes, and evidence_ref format for all Step 16 execution results", - "Eval-grade structured session logging with deterministic replay metadata, validation gate lineage, and redaction profiles", - "Context pack budget enforcement with soft/hard token limits and truncation policies", - "Context management including SpecRefResolver lookups, context pack regeneration, and Spec drift detection", - "Three utility personas (Researcher, ToolUser, Summarizer, Auditor) with bounded context discovery and extraction-only evidence support", - "Checkpoint-based incremental commits with governance gates on every state transition and milestone closure", - "Terminal dashboard with three-panel live reporting for L1 orchestration monitoring", - "Session spawning and management with parent-child spawn protocols, spawn logging, and loop detection", - "Scratchpad-based state recovery and serialization for crash tolerance and token boundary handling", - "Session log rotation and compaction with configurable thresholds and archival preservation", - "Workspace artifact versioning (Draft→Audit→Refine loops) with versioned artifacts and archive cleanup", - "Secret safety enforcement including pre-persist scanning, redaction profiles, and denylist command patterns", - "Validation gate enforcement (schema, deep, governance, spec authority) on all state transitions and artifacts", - "Agent turn tracking and session event capture for replay and eval pipeline integration", - "Deterministic tool-call protocol with typed envelopes, schema validation, and tool argument safety", - "Spec drift detection and warnings for spec_ref references with commit hash and line_range grounding checks", - "Error recovery and blocker resolution with retry caps, blocked states, and human escalation paths", - "Agent failure detection and handling for timeouts, hallucinations, ambiguous conditions, and blocked states", - "Implementation loop completion tracking for L3 atomic units with retry caps and pass/fail verification", - "Spec baseline commit policy with controlled re-plan cycles for mid-run spec changes", - "SpecRefResolver error handling for missing or invalid spec_ref entries with explicit ambiguity findings", - "Workspace cleanup safety checks to prevent incorrect archival or deletion of debug artifacts", - "SpecRefResolver staleness detection for spec_ref references across git history and repository state" - ], - "out_of_scope": [ - "Real-time collaborative editing or multi-user concurrent session management", - "Machine learning model training or fine-tuning pipelines (eval-only dataset export)", - "External CI/CD integration beyond governance gate enforcement and checkpoint commits", - "Natural language query interface or conversational UI layer", - "Automated test generation or test suite optimization algorithms", - "Plugin ecosystem or third-party extension framework for custom tools", - "Automated deployment to cloud infrastructure or container orchestration", - "Real-time telemetry streaming to external monitoring platforms", - "Automated security vulnerability scanning beyond secret detection in logs", - "Automated performance profiling or optimization recommendations" - ], - "assumptions": [ - "Existing Git-based version control with commit hashes available for spec_ref grounding", - "OpenAI-compatible chat endpoints with configurable timeouts available for LLM strategy", - "Filesystem artifacts are the authoritative shared state for all state transitions", - "Seed-manifest governance is maintained and updated before each milestone run", - "Deterministic tool schemas are versioned and available at schema URI references", - "Parent agents consume child artifacts and never child chat transcripts", - "Context pack budget tokens are sufficient to cover required spec refs and seed files", - "SpecRefResolver can resolve git history, line ranges, and commit hashes for all spec_refs", - "Workspace artifact versioning supports Draft→Audit→Refine loops with rollback capability", - "Session logs can be rotated and archived without loss of replay metadata", - "Terminal dashboard does not require persistent storage or external dependencies", - "Secret scanning/redaction can be applied to persisted artifacts without blocking execution", - "Spec baseline commit policy allows controlled re-plan cycles when seed/spec changes are required mid-run", - "SpecRefResolver can handle missing or invalid spec_ref entries gracefully and surface explicit ambiguity findings", - "Git repository has sufficient history depth to resolve all required spec_ref commit hashes for the active milestone", - "SpecRefResolver line_range lookups are accurate and deterministic regardless of repository state (clean or with uncommitted changes)" - ], - "risks": [ - "Dependency readiness: OpenAI-compatible endpoint availability and latency may block unattended runs", - "Spec drift: Changes to seed-manifest or governed artifacts may cause context pack mismatches", - "Token budget exhaustion: Long-running LLM generations may exceed soft/hard token limits", - "Retry cap exhaustion: Infinite repair loops may exceed configured retry caps without human intervention", - "Secret leakage: Failed secret scanning/redaction may persist raw secrets to disk", - "Context loss: Long execution windows may lose critical state across resume boundaries", - "Schema drift: Runtime protocol schemas may diverge from prompt-side catalog contracts", - "Tool-call ambiguity: Compact tool catalog may be insufficient for complex deterministic planning", - "Evidence truncation: Long command outputs may lose critical pass/fail markers in evidence excerpts", - "Audit scope creep: L2 Collective Audit may miss cross-cutting quality issues across checklist items", - "SpecRefResolver failure: Missing or invalid spec_ref entries may block progress without clear remediation path", - "Spec baseline commit policy failure: Mid-run spec changes may require complex re-plan cycles that exceed retry caps", - "Agent hallucination: LLM may hallucinate APIs, file paths, or contracts that are not in governed spec artifacts", - "Workspace cleanup failure: Intermediate draft/audit versions may be incorrectly archived or deleted, losing evidence for debugging", - "SpecRefResolver stale content: Git history may contain stale spec_ref references that are not detected until runtime validation" - ], - "stakeholders": [ - { - "role": "Engineering Lead", - "needs": [ - "Milestone execution completes with verified code changes and evidence bindings", - "Unattended runs can resume from checkpoint without manual intervention", - "Schema validation gates prevent invalid artifacts from entering repository", - "Session logs provide deterministic replay for post-mortem analysis", - "Spec authority enforcement prevents unauthorized spec changes", - "Retry caps prevent infinite repair loops without human escalation", - "Error recovery success rate indicates system resilience", - "Blocker resolution rate ensures progress can be maintained", - "Implementation loop completion rate guarantees milestone progress" - ] - }, - { - "role": "DevOps Engineer", - "needs": [ - "Incremental commit checkpoints with governance gates prevent dirty working trees", - "Secret scanning/redaction prevents credential leakage in logs and artifacts", - "Token budget limits prevent runaway context inflation in long runs", - "Session log rotation and archival preserve replay metadata for long-term analysis", - "Workspace cleanup operations properly archive intermediate artifacts", - "Agent failure detection identifies runtime issues early", - "SpecRefResolver failures surface spec grounding problems" - ] - }, - { - "role": "Security Lead", - "needs": [ - "No raw secrets persist to disk in any artifact or log file", - "Redaction profiles can be tuned per environment (dev/staging/prod)", - "Secret detection confidence thresholds can be adjusted", - "Command denylist patterns prevent sensitive command execution", - "Pre-persist secret scanning blocks unsafe artifact writes", - "SpecRefResolver staleness detection prevents outdated spec references", - "Workspace cleanup safety prevents accidental secret leakage in archived artifacts" - ] - }, - { - "role": "QA Engineer", - "needs": [ - "Evidence excerpts contain verbatim pass/fail markers for test verification", - "Step 16 evidence fields are never paraphrased and include SHA-256 hashes", - "Session logs capture validation gate outcomes with full lineage metadata", - "Spec drift detection surfaces stale references before runtime execution", - "Session log completeness ensures all events are replayable and verifiable", - "Agent failure detection provides visibility into runtime issues", - "Error recovery success rate indicates system robustness" - ] - }, - { - "role": "Product Manager", - "needs": [ - "Milestone completion status is visible in terminal dashboard", - "Roadmap progress syncs automatically after verified milestones", - "Execution can be paused/resumed without losing state", - "Agent turn counts provide visibility into milestone effort and scope", - "Resume success rate indicates reliability of unattended operations", - "Implementation loop completion rate guarantees milestone progress", - "Error recovery success rate indicates system resilience" - ] - }, - { - "role": "ML Engineer / Eval Researcher", - "needs": [ - "Eval-grade structured session logs with deterministic replay metadata", - "OpenAI-style messages export for ML training dataset generation", - "Redaction profiles applied consistently across all persisted artifacts", - "Full lineage metadata (event_sequence, prev_event_sha256, artifact_sha256) for replay reconstruction", - "Session log rotation preserves archived segments for long-term eval pipelines", - "Agent failure detection provides clean data for eval dataset generation", - "SpecRefResolver errors surface spec grounding issues for eval improvement" - ] - }, - { - "role": "Platform Engineer", - "needs": [ - "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", - "Child spawn success rate indicates runtime stability", - "State transition success rate ensures reliable orchestration", - "Scratchpad recovery success rate enables crash-tolerant execution", - "Validation gate pass rate guarantees artifact quality before commit", - "Error recovery success rate indicates system robustness", - "Blocker resolution rate ensures progress can be maintained", - "Agent failure detection identifies runtime issues early" - ] - }, - { - "role": "Compliance Officer", - "needs": [ - "Spec authority enforcement prevents unauthorized spec modifications", - "Secret safety enforcement maintains auditability of sensitive data", - "Session logs capture complete execution history for regulatory review", - "Governance gate compliance ensures all commits meet policy requirements", - "Spec drift detection surfaces historical spec changes for compliance verification", - "Agent failure detection provides complete audit trail", - "Error recovery success rate indicates system reliability" - ] - }, - { - "role": "Site Reliability Engineer", - "needs": [ - "Error recovery success rate indicates system resilience", - "Blocker resolution rate ensures progress can be maintained", - "Agent failure detection identifies runtime issues early", - "SpecRefResolver failures surface spec grounding problems", - "Session log rotation prevents disk space exhaustion", - "Context budget utilization ensures optimal resource usage" - ] - } - ], - "user_segments": [ - { - "segment_id": "trinity-operator", - "description": "Platform engineers who invoke `specdev trinity` for unattended milestone execution", - "jobs_to_be_done": [ - "Execute one-milestone vertical slice from roadmap without manual intervention", - "Monitor execution progress via terminal dashboard", - "Resume interrupted runs from checkpoint", - "Review findings and verdicts after milestone completion", - "Debug blocked or deferred milestones via session logs", - "Configure context pack budgets and token limits for different milestone sizes", - "Review spec drift warnings and remediate stale references", - "Validate governance gate failures and override scope budgets when appropriate", - "Configure error recovery and retry cap parameters", - "Monitor implementation loop completion and agent failure detection", - "Review SpecRefResolver errors and address spec grounding issues", - "Handle spec baseline commit policy mid-run re-plan cycles" - ], - "pains": [ - "Context loss across long-running LLM generations", - "Infinite repair loops without clear stop conditions", - "Unverified code changes without evidence-backed tests", - "Hallucinated APIs or file paths breaking implementation", - "Spec drift causing unexpected behavior mid-run", - "Token budget exhaustion causing premature truncation", - "Retry cap exhaustion without human intervention", - "Secret leakage in persisted artifacts or logs", - "Agent failure detection failures masking runtime issues", - "Error recovery failures blocking progress", - "SpecRefResolver errors blocking execution without clear remediation" - ], - "gains": [ - "Lossless persistence allows resume from any checkpoint", - "Evidence bindings provide verifiable proof of test pass/fail", - "Spec authority prevents unauthorized changes to governed artifacts", - "Retry caps prevent infinite loops", - "Terminal dashboard provides real-time visibility into milestone state", - "Context pack budget enforcement prevents runaway token inflation", - "Spec drift detection surfaces stale references proactively", - "Secret safety enforcement prevents credential leakage", - "Agent failure detection identifies runtime issues early", - "Error recovery success rate indicates system resilience", - "Implementation loop completion rate guarantees milestone progress", - "SpecRefResolver errors surface spec grounding problems clearly" - ] - }, - { - "segment_id": "spec-author", - "description": "Domain experts who author and maintain governed specification artifacts", - "jobs_to_be_done": [ - "Update seed-manifest and governed spec files (FRs, NFRs, governance, etc.)", - "Maintain seed-manifest governance and authority set", - "Validate spec changes before committing as baseline", - "Review spec_ref grounding and line_range accuracy in implementation artifacts", - "Ensure spec drift detection identifies stale references", - "Add new spec artifacts to authority set with proper grounding", - "Update spec_ref line ranges when implementing changes", - "Review drift warnings after spec changes are committed", - "Handle SpecRefResolver staleness detection for historical references", - "Verify spec baseline commit policy allows controlled re-plan cycles" - ], - "pains": [ - "Spec changes breaking existing implementation traces", - "Spec_ref grounding failing due to missing or stale line ranges", - "Seed-manifest governance becoming outdated", - "Spec drift not detected until runtime failures occur", - "Manual verification of spec_ref commit hashes and line ranges", - "Reconciliation of spec changes across multiple milestones", - "SpecRefResolver staleness not detected until runtime", - "Spec baseline commit policy complexity for mid-run changes" - ], - "gains": [ - "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", - "Drift warnings surface stale references before runtime execution", - "Spec authority prevents unauthorized mutations to governed artifacts", - "Seed-manifest order ensures consistent context resolution across runs", - "Automated drift detection saves manual verification effort", - "Commit hash grounding ensures reproducible spec references", - "SpecRefResolver staleness detection surfaces historical changes", - "Spec baseline commit policy allows controlled re-plan cycles" - ] - }, - { - "segment_id": "eval-researcher", - "description": "Researchers and ML engineers who analyze execution traces for model training datasets", - "jobs_to_be_done": [ - "Export session logs to OpenAI-style messages format", - "Validate export rows against eval_export_row.schema.json", - "Apply deterministic redaction profiles to sensitive data", - "Create replay artifacts for dataset generation", - "Analyze validation gate outcomes and evidence bindings", - "Validate session log completeness and replay metadata", - "Create eval datasets from verified milestones", - "Analyze retry cap usage patterns and failure modes", - "Review agent failure detection events for eval improvement", - "Analyze error recovery success rates and failure patterns" - ], - "pains": [ - "Sensitive data (API keys, tokens) in session logs or prompt artifacts", - "Lack of structured export format for ML pipelines", - "Redaction applied inconsistently across different artifact types", - "Loss of replay metadata makes dataset reconstruction difficult", - "Incomplete session logs missing events or lineage", - "Redaction profiles not consistently applied across all artifacts", - "Agent failure detection events not captured in export", - "Error recovery failures not included in eval datasets" - ], - "gains": [ - "Eval strategy provides native event schema as source-of-truth", - "OpenAI-style messages export with validation", - "Deterministic redaction profiles applied to all persisted artifacts", - "Full lineage metadata (event_sequence, prev_event_sha256, artifact_sha256) for replay", - "Session log rotation preserves archival segments for long-term eval", - "Secret safety enforcement ensures export datasets are safe for external use", - "Agent failure detection events captured for eval improvement", - "Error recovery success rates tracked for eval dataset quality" - ] - }, - { - "segment_id": "platform-engineer", - "description": "Platform engineers responsible for Trinity runtime infrastructure and tooling", - "jobs_to_be_done": [ - "Implement and maintain SpecRefResolver with git grounding", - "Implement and maintain validation gate schemas and validators", - "Implement and maintain scratchpad serialization/deserialization", - "Implement and maintain session log rotation and archival", - "Implement and maintain secret scanning and redaction pipelines", - "Implement and maintain tool protocol schemas and validators", - "Monitor spawn success rates and state transition success rates", - "Debug validation gate failures and optimize performance", - "Implement error recovery and blocker resolution mechanisms", - "Implement agent failure detection and handling", - "Implement Spec baseline commit policy and re-plan cycles", - "Implement SpecRefResolver staleness detection" - ], - "pains": [ - "SpecRefResolver failures due to missing git history or invalid commit hashes", - "Validation gate performance bottlenecks in long-running sessions", - "Scratchpad corruption after crashes or token boundary handling", - "Session log rotation causing data loss or replay failures", - "Secret scanning false positives blocking legitimate artifacts", - "Tool protocol schema drift between prompt and runtime", - "Agent turn count explosion due to inefficient planning", - "Error recovery failures blocking progress", - "Blocker resolution failures preventing progress", - "Spec baseline commit policy complexity for re-plan cycles" - ], - "gains": [ - "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", - "Validation gates prevent invalid artifacts before commit", - "Scratchpad recovery ensures crash-tolerant execution", - "Session log rotation preserves archival segments without data loss", - "Secret scanning prevents leakage without false positives", - "Tool protocol schemas guarantee type safety and deterministic behavior", - "Spawn success and state transition success metrics indicate runtime health", - "Error recovery success rate indicates system resilience", - "Implementation loop completion rate guarantees milestone progress", - "Blocker resolution rate ensures progress can be maintained" - ] - }, - { - "segment_id": "compliance-officer", - "description": "Compliance officers who ensure Trinity execution adheres to organizational policies and regulations", - "jobs_to_be_done": [ - "Review session logs for compliance violations", - "Validate governance gate compliance on all commits", - "Verify spec authority enforcement prevents unauthorized changes", - "Ensure secret safety enforcement meets regulatory requirements", - "Audit spec drift detection for historical spec changes", - "Review session log completeness for audit trail requirements", - "Validate agent turn counts and execution effort for scope verification", - "Review error recovery success rates for system resilience", - "Verify agent failure detection captures all relevant events" - ], - "pains": [ - "Unable to trace execution history for audit requirements", - "Unauthorized spec changes slipping through governance gates", - "Credential leakage in logs or artifacts", - "Incomplete execution records missing events or lineage", - "Spec drift not detected until after compliance review", - "Governance gate failures not properly documented", - "Agent failure detection events not captured in audit trail", - "Error recovery failures not included in compliance review" - ], - "gains": [ - "Spec authority enforcement prevents unauthorized spec modifications", - "Secret safety enforcement maintains auditability of sensitive data", - "Session logs capture complete execution history for regulatory review", - "Governance gate compliance ensures all commits meet policy requirements", - "Spec drift detection surfaces historical spec changes for compliance verification", - "Session log completeness ensures replayable audit trails", - "Agent failure detection events captured for complete audit trail", - "Error recovery success rates tracked for compliance review" - ] - }, - { - "segment_id": "site-reliability-engineer", - "description": "Site Reliability Engineers responsible for system reliability, monitoring, and incident response", - "jobs_to_be_done": [ - "Monitor error recovery success rates and alert on failures", - "Track blocker resolution rates and alert on bottlenecks", - "Review agent failure detection events for incident response", - "Validate SpecRefResolver stability and error handling", - "Monitor session log rotation and archival for disk space management", - "Track context budget utilization for resource optimization", - "Review implementation loop completion for capacity planning", - "Validate governance gate compliance for system health" - ], - "pains": [ - "Error recovery failures blocking progress without visibility", - "Blocker resolution failures preventing milestone completion", - "Agent failure detection events not surfaced for incident response", - "SpecRefResolver instability causing runtime failures", - "Session log rotation causing disk space exhaustion", - "Context budget exhaustion causing premature truncation", - "Implementation loop completion failures indicating system issues" - ], - "gains": [ - "Error recovery success rate indicates system resilience", - "Blocker resolution rate ensures progress can be maintained", - "Agent failure detection identifies runtime issues early", - "SpecRefResolver errors surface spec grounding problems", - "Session log rotation prevents disk space exhaustion", - "Context budget utilization ensures optimal resource usage", - "Implementation loop completion rate guarantees milestone progress" - ] - } - ], - "success_metrics": [ - { - "metric_id": "trinity-m1-coverage", - "name": "Spec authority compliance", - "baseline": 95, - "target": 99, - "unit": "percent", - "measurement_method": "Validate that all tool calls and file writes reference governed seed-manifest artifacts; track violations in session logs and aggregate as percentage of total operations" - }, - { - "metric_id": "trinity-m2-evidence-binding", - "name": "Evidence binding coverage", - "baseline": 0, - "target": 100, - "unit": "percent of checklist items", - "measurement_method": "Count checklist items with valid evidence excerpts and SHA-256 hashes in execution.execution_results[]; divide by total checklist items for milestone" - }, - { - "metric_id": "trinity-m3-resume-success", - "name": "Resume success rate", - "baseline": 0, - "target": 95, - "unit": "percent of resumed runs", - "measurement_method": "Track resume attempts from checkpoint; record success if milestone completes within token budget and without context loss; measure success rate over 50 resume attempts" - }, - { - "metric_id": "trinity-m4-secret-detection", - "name": "Secret detection coverage", - "baseline": 0, - "target": 100, - "unit": "percent of persisted artifacts", - "measurement_method": "Scan all persisted prompt/response/session/artifact files using secret_scanner_v1; track detections and false negatives; calculate coverage as files scanned / files persisted" - }, - { - "metric_id": "trinity-m5-retry-cap-usage", - "name": "Retry cap utilization", - "baseline": 0, - "target": 20, - "unit": "percent of milestone runs", - "measurement_method": "Count milestone runs where retry caps are exceeded; divide by total milestone runs; track by state (16a/16b/16c) to identify failure modes" - }, - { - "metric_id": "trinity-m6-spawn-success-rate", - "name": "Child spawn success rate", - "baseline": 95, - "target": 99, - "unit": "percent of spawns", - "measurement_method": "Track all child spawns (16a/16b/16c/utility) and count successful spawns; divide by total spawns; exclude blocked/deferred states" - }, - { - "metric_id": "trinity-m7-state-transition-success-rate", - "name": "State transition success rate", - "baseline": 95, - "target": 99, - "unit": "percent of transitions", - "measurement_method": "Count valid state transitions (16a→16b→16c) that complete without validation failures; divide by total attempted transitions" - }, - { - "metric_id": "trinity-m8-validation-gate-pass-rate", - "name": "Validation gate pass rate", - "baseline": 90, - "target": 98, - "unit": "percent of gates", - "measurement_method": "Track all validation gates (schema, deep, governance, spec authority); count successful passes; divide by total gate evaluations" - }, - { - "metric_id": "trinity-m9-spec-drift-detection-coverage", - "name": "Spec drift detection coverage", - "baseline": 0, - "target": 100, - "unit": "percent of spec_ref references", - "measurement_method": "Count spec_ref records with successful drift detection (stale content or commit hash mismatch); divide by total spec_ref references in session logs" - }, - { - "metric_id": "trinity-m10-context-budget-utilization", - "name": "Context budget utilization", - "baseline": 0, - "target": 85, - "unit": "percent of hard token limit", - "measurement_method": "Calculate average context_pack token usage across all state transitions; divide by hard_token_limit; track by phase (16a/16b/16c)" - }, - { - "metric_id": "trinity-m11-context-truncation-events", - "name": "Context truncation events", - "baseline": 0, - "target": 5, - "unit": "events per milestone run", - "measurement_method": "Count context_budget_truncation events in session logs; track by cause (token overflow, priority item truncation)" - }, - { - "metric_id": "trinity-m12-specref-resolver-success-rate", - "name": "SpecRefResolver success rate", - "baseline": 95, - "target": 99, - "unit": "percent of lookups", - "measurement_method": "Count successful SpecRefResolver calls (path, line_range, commit_hash resolved) divided by total lookups; track failures by type" - }, - { - "metric_id": "trinity-m13-session-log-completeness", - "name": "Session log completeness", - "baseline": 90, - "target": 99, - "unit": "percent of events", - "measurement_method": "Validate session log events contain required fields (event_sequence, prev_event_sha256, event_sha256, artifact_ref, artifact_sha256); count complete events divided by total events" - }, - { - "metric_id": "trinity-m14-scratchpad-recovery-success", - "name": "Scratchpad recovery success rate", - "baseline": 0, - "target": 95, - "unit": "percent of recovery attempts", - "measurement_method": "Track scratchpad recovery attempts after crash or token boundary; count successful loads; measure over 50 recovery events" - }, - { - "metric_id": "trinity-m15-governance-gate-compliance", - "name": "Governance gate compliance", - "baseline": 95, - "target": 99, - "unit": "percent of commits", - "measurement_method": "Count commits passing governance checks (seed-manifest authority, spec_ref grounding, scope adherence) divided by total commits; track failures by gate type" - }, - { - "metric_id": "trinity-m16-agent-turn-count", - "name": "Agent turn count per milestone", - "baseline": 50, - "target": 100, - "unit": "turns", - "measurement_method": "Count total agent turns (messages sent/received) per milestone run; track distribution by state (16a/16b/16c/utility)" - }, - { - "metric_id": "trinity-m17-session-log-rotation-events", - "name": "Session log rotation events", - "baseline": 0, - "target": 3, - "unit": "events per milestone run", - "measurement_method": "Count session_log_compaction_threshold exceeded events; track by rotation cause (event count, time boundary)" - }, - { - "metric_id": "trinity-m18-workspace-cleanup-completeness", - "name": "Workspace cleanup completeness", - "baseline": 95, - "target": 100, - "unit": "percent of cleanup operations", - "measurement_method": "Track workspace cleanup operations after milestone closure; verify workspace artifacts and session logs are archived for future reference; count successful archive operations divided by total cleanup attempts; ensure logs are preserved, not deleted" - }, - { - "metric_id": "trinity-m19-spawn-efficiency-overhead", - "name": "Spawn efficiency and overhead", - "baseline": 5, - "target": 2, - "unit": "seconds per spawn", - "measurement_method": "Measure average time from spawn initiation to child task_input.json creation; track overhead reduction over time; exclude blocked/deferred spawns" - }, - { - "metric_id": "trinity-m20-spawn-utilization-rate", - "name": "Spawn utilization rate", - "baseline": 70, - "target": 90, - "unit": "percent of spawns", - "measurement_method": "Count effective spawns that contribute to milestone progress vs redundant or blocked spawns; divide effective spawns by total spawns; track by utility type (Researcher, ToolUser, Summarizer, Auditor)" - }, - { - "metric_id": "trinity-m21-spawn-loop-prevention-effectiveness", - "name": "Spawn loop prevention effectiveness", - "baseline": 0, - "target": 95, - "unit": "percent of duplicate spawns prevented", - "measurement_method": "Count duplicate spawn attempts detected by spawn_log loop detection; verify spawn_log prevents exceeding configured retry caps; measure prevention effectiveness over 100 milestone runs" - }, - { - "metric_id": "trinity-m22-error-recovery-success-rate", - "name": "Error recovery success rate", - "baseline": 0, - "target": 90, - "unit": "percent of error states", - "measurement_method": "Track error states (validation failures, schema violations, blocked conditions) and count successful recoveries; measure success rate over 100 error recovery attempts" - }, - { - "metric_id": "trinity-m23-blocker-resolution-rate", - "name": "Blocker resolution rate", - "baseline": 0, - "target": 85, - "unit": "percent of blockers", - "measurement_method": "Track blocker conditions (missing seed, out-of-scope writes, security concerns, retry cap exhaustion) and count successful resolutions before cap exhaustion; measure success rate over 100 blocker events" - }, - { - "metric_id": "trinity-m24-implementation-loop-completion-rate", - "name": "Implementation loop completion rate", - "baseline": 0, - "target": 95, - "unit": "percent of checklist units", - "measurement_method": "Count checklist units (atomic L3 loops) that complete successfully vs fail; track by state (16a/16b/16c); measure success rate over 100 units" - }, - { - "metric_id": "trinity-m25-agent-failure-detection-rate", - "name": "Agent failure detection rate", - "baseline": 95, - "target": 99, - "unit": "percent of agent failures", - "measurement_method": "Track agent failures (timeout, blocked, ambiguous, hallucination) and count successful detection in session logs; measure detection rate over 100 agent failure events" - } - ], - "links": [ - { - "source": "project_charter-trinity", - "target": "fr-16a-planner", - "relation": "upstream" - }, - { - "source": "project_charter-trinity", - "target": "fr-16b-builder", - "relation": "upstream" - }, - { - "source": "project_charter-trinity", - "target": "fr-16c-verifier", - "relation": "upstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-context-budget", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-evidence-integrity", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-secret-safety", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "governance-16-spec-authority", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "seed-manifest-common", - "relation": "upstream" - }, - { - "source": "project_charter-trinity", - "target": "trinity_spec", - "relation": "upstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-session-management", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-validation-gates", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-spawn-protocol", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "governance-16-agent-spawning", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "governance-16-context-pack", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "governance-16-secret-scanning", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-spawn-success-rate", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-state-transition-success", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-validation-gate-pass", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-spec-drift-detection", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-context-truncation", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-specref-resolver", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-session-log-completeness", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-scratchpad-recovery", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-governance-gate-compliance", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-agent-turn-count", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-session-rotation", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-workspace-cleanup", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-error-recovery-success", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-blocker-resolution-rate", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-implementation-loop-completion", - "relation": "downstream" - }, - { - "source": "project_charter-trinity", - "target": "nfr-16-agent-failure-detection", - "relation": "downstream" - } - ] -} \ No newline at end of file From c68f97f5625e9a0b69e298ee6fd61a783f652bc4 Mon Sep 17 00:00:00 2001 From: Shantanu Agarwal Date: Sun, 15 Feb 2026 00:06:14 +0530 Subject: [PATCH 6/6] Correcting spec path --- toolkit_agent/spec/00_charter.json | 782 +++++++++++++++++++++++++++++ 1 file changed, 782 insertions(+) create mode 100644 toolkit_agent/spec/00_charter.json diff --git a/toolkit_agent/spec/00_charter.json b/toolkit_agent/spec/00_charter.json new file mode 100644 index 00000000..504a5cca --- /dev/null +++ b/toolkit_agent/spec/00_charter.json @@ -0,0 +1,782 @@ +{ + "id": "project_charter-trinity", + "owner": "engineering", + "created_at": "2026-02-14T00:00:00Z", + "title": "Trinity Automation System Specification", + "problem_statement": "Long-running agentic implementation loops suffer from context loss from window limits, spec drift from seeded constraints, hallucinated APIs/files/contracts, infinite repair loops without stop conditions, and unverified code changes without evidence-bound tests. These failures prevent reliable unattended AI-driven development execution across multi-milestone roadmaps. Local LLMs have shorter context token window and are not capable to run long sessions.", + "in_scope": [ + "Three-level fractal orchestration (L1 milestone, L2 persona, L3 atomic) with strict parent-child process boundaries", + "Disk-first two-phase artifact exchange contract (questions only → filesystem artifacts)", + "Checklist-first state machine with deterministic tool-call protocols and retry caps", + "Spec authority enforced through seed-manifest governance and SpecRefResolver", + "Evidence binding with verbatim excerpts, SHA-256 hashes, and evidence_ref format for all Step 16 execution results", + "Eval-grade structured session logging with deterministic replay metadata, validation gate lineage, and redaction profiles", + "Context pack budget enforcement with soft/hard token limits and truncation policies", + "Context management including SpecRefResolver lookups, context pack regeneration, and Spec drift detection", + "Three utility personas (Researcher, ToolUser, Summarizer, Auditor) with bounded context discovery and extraction-only evidence support", + "Checkpoint-based incremental commits with governance gates on every state transition and milestone closure", + "Terminal dashboard with three-panel live reporting for L1 orchestration monitoring", + "Session spawning and management with parent-child spawn protocols, spawn logging, and loop detection", + "Scratchpad-based state recovery and serialization for crash tolerance and token boundary handling", + "Session log rotation and compaction with configurable thresholds and archival preservation", + "Workspace artifact versioning (Draft→Audit→Refine loops) with versioned artifacts and archive cleanup", + "Secret safety enforcement including pre-persist scanning, redaction profiles, and denylist command patterns", + "Validation gate enforcement (schema, deep, governance, spec authority) on all state transitions and artifacts", + "Agent turn tracking and session event capture for replay and eval pipeline integration", + "Deterministic tool-call protocol with typed envelopes, schema validation, and tool argument safety", + "Spec drift detection and warnings for spec_ref references with commit hash and line_range grounding checks", + "Error recovery and blocker resolution with retry caps, blocked states, and human escalation paths", + "Agent failure detection and handling for timeouts, hallucinations, ambiguous conditions, and blocked states", + "Implementation loop completion tracking for L3 atomic units with retry caps and pass/fail verification", + "Spec baseline commit policy with controlled re-plan cycles for mid-run spec changes", + "SpecRefResolver error handling for missing or invalid spec_ref entries with explicit ambiguity findings", + "Workspace cleanup safety checks to prevent incorrect archival or deletion of debug artifacts", + "SpecRefResolver staleness detection for spec_ref references across git history and repository state" + ], + "out_of_scope": [ + "Real-time collaborative editing or multi-user concurrent session management", + "Machine learning model training or fine-tuning pipelines (eval-only dataset export)", + "External CI/CD integration beyond governance gate enforcement and checkpoint commits", + "Natural language query interface or conversational UI layer", + "Automated test generation or test suite optimization algorithms", + "Plugin ecosystem or third-party extension framework for custom tools", + "Automated deployment to cloud infrastructure or container orchestration", + "Real-time telemetry streaming to external monitoring platforms", + "Automated security vulnerability scanning beyond secret detection in logs", + "Automated performance profiling or optimization recommendations" + ], + "assumptions": [ + "Existing Git-based version control with commit hashes available for spec_ref grounding", + "OpenAI-compatible chat endpoints with configurable timeouts available for LLM strategy", + "Filesystem artifacts are the authoritative shared state for all state transitions", + "Seed-manifest governance is maintained and updated before each milestone run", + "Deterministic tool schemas are versioned and available at schema URI references", + "Parent agents consume child artifacts and never child chat transcripts", + "Context pack budget tokens are sufficient to cover required spec refs and seed files", + "SpecRefResolver can resolve git history, line ranges, and commit hashes for all spec_refs", + "Workspace artifact versioning supports Draft→Audit→Refine loops with rollback capability", + "Session logs can be rotated and archived without loss of replay metadata", + "Terminal dashboard does not require persistent storage or external dependencies", + "Secret scanning/redaction can be applied to persisted artifacts without blocking execution", + "Spec baseline commit policy allows controlled re-plan cycles when seed/spec changes are required mid-run", + "SpecRefResolver can handle missing or invalid spec_ref entries gracefully and surface explicit ambiguity findings", + "Git repository has sufficient history depth to resolve all required spec_ref commit hashes for the active milestone", + "SpecRefResolver line_range lookups are accurate and deterministic regardless of repository state (clean or with uncommitted changes)" + ], + "risks": [ + "Dependency readiness: OpenAI-compatible endpoint availability and latency may block unattended runs", + "Spec drift: Changes to seed-manifest or governed artifacts may cause context pack mismatches", + "Token budget exhaustion: Long-running LLM generations may exceed soft/hard token limits", + "Retry cap exhaustion: Infinite repair loops may exceed configured retry caps without human intervention", + "Secret leakage: Failed secret scanning/redaction may persist raw secrets to disk", + "Context loss: Long execution windows may lose critical state across resume boundaries", + "Schema drift: Runtime protocol schemas may diverge from prompt-side catalog contracts", + "Tool-call ambiguity: Compact tool catalog may be insufficient for complex deterministic planning", + "Evidence truncation: Long command outputs may lose critical pass/fail markers in evidence excerpts", + "Audit scope creep: L2 Collective Audit may miss cross-cutting quality issues across checklist items", + "SpecRefResolver failure: Missing or invalid spec_ref entries may block progress without clear remediation path", + "Spec baseline commit policy failure: Mid-run spec changes may require complex re-plan cycles that exceed retry caps", + "Agent hallucination: LLM may hallucinate APIs, file paths, or contracts that are not in governed spec artifacts", + "Workspace cleanup failure: Intermediate draft/audit versions may be incorrectly archived or deleted, losing evidence for debugging", + "SpecRefResolver stale content: Git history may contain stale spec_ref references that are not detected until runtime validation" + ], + "stakeholders": [ + { + "role": "Engineering Lead", + "needs": [ + "Milestone execution completes with verified code changes and evidence bindings", + "Unattended runs can resume from checkpoint without manual intervention", + "Schema validation gates prevent invalid artifacts from entering repository", + "Session logs provide deterministic replay for post-mortem analysis", + "Spec authority enforcement prevents unauthorized spec changes", + "Retry caps prevent infinite repair loops without human escalation", + "Error recovery success rate indicates system resilience", + "Blocker resolution rate ensures progress can be maintained", + "Implementation loop completion rate guarantees milestone progress" + ] + }, + { + "role": "DevOps Engineer", + "needs": [ + "Incremental commit checkpoints with governance gates prevent dirty working trees", + "Secret scanning/redaction prevents credential leakage in logs and artifacts", + "Token budget limits prevent runaway context inflation in long runs", + "Session log rotation and archival preserve replay metadata for long-term analysis", + "Workspace cleanup operations properly archive intermediate artifacts", + "Agent failure detection identifies runtime issues early", + "SpecRefResolver failures surface spec grounding problems" + ] + }, + { + "role": "Security Lead", + "needs": [ + "No raw secrets persist to disk in any artifact or log file", + "Redaction profiles can be tuned per environment (dev/staging/prod)", + "Secret detection confidence thresholds can be adjusted", + "Command denylist patterns prevent sensitive command execution", + "Pre-persist secret scanning blocks unsafe artifact writes", + "SpecRefResolver staleness detection prevents outdated spec references", + "Workspace cleanup safety prevents accidental secret leakage in archived artifacts" + ] + }, + { + "role": "QA Engineer", + "needs": [ + "Evidence excerpts contain verbatim pass/fail markers for test verification", + "Step 16 evidence fields are never paraphrased and include SHA-256 hashes", + "Session logs capture validation gate outcomes with full lineage metadata", + "Spec drift detection surfaces stale references before runtime execution", + "Session log completeness ensures all events are replayable and verifiable", + "Agent failure detection provides visibility into runtime issues", + "Error recovery success rate indicates system robustness" + ] + }, + { + "role": "Product Manager", + "needs": [ + "Milestone completion status is visible in terminal dashboard", + "Roadmap progress syncs automatically after verified milestones", + "Execution can be paused/resumed without losing state", + "Agent turn counts provide visibility into milestone effort and scope", + "Resume success rate indicates reliability of unattended operations", + "Implementation loop completion rate guarantees milestone progress", + "Error recovery success rate indicates system resilience" + ] + }, + { + "role": "ML Engineer / Eval Researcher", + "needs": [ + "Eval-grade structured session logs with deterministic replay metadata", + "OpenAI-style messages export for ML training dataset generation", + "Redaction profiles applied consistently across all persisted artifacts", + "Full lineage metadata (event_sequence, prev_event_sha256, artifact_sha256) for replay reconstruction", + "Session log rotation preserves archived segments for long-term eval pipelines", + "Agent failure detection provides clean data for eval dataset generation", + "SpecRefResolver errors surface spec grounding issues for eval improvement" + ] + }, + { + "role": "Platform Engineer", + "needs": [ + "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", + "Child spawn success rate indicates runtime stability", + "State transition success rate ensures reliable orchestration", + "Scratchpad recovery success rate enables crash-tolerant execution", + "Validation gate pass rate guarantees artifact quality before commit", + "Error recovery success rate indicates system robustness", + "Blocker resolution rate ensures progress can be maintained", + "Agent failure detection identifies runtime issues early" + ] + }, + { + "role": "Compliance Officer", + "needs": [ + "Spec authority enforcement prevents unauthorized spec modifications", + "Secret safety enforcement maintains auditability of sensitive data", + "Session logs capture complete execution history for regulatory review", + "Governance gate compliance ensures all commits meet policy requirements", + "Spec drift detection surfaces historical spec changes for compliance verification", + "Agent failure detection provides complete audit trail", + "Error recovery success rate indicates system reliability" + ] + }, + { + "role": "Site Reliability Engineer", + "needs": [ + "Error recovery success rate indicates system resilience", + "Blocker resolution rate ensures progress can be maintained", + "Agent failure detection identifies runtime issues early", + "SpecRefResolver failures surface spec grounding problems", + "Session log rotation prevents disk space exhaustion", + "Context budget utilization ensures optimal resource usage" + ] + } + ], + "user_segments": [ + { + "segment_id": "trinity-operator", + "description": "Platform engineers who invoke `specdev trinity` for unattended milestone execution", + "jobs_to_be_done": [ + "Execute one-milestone vertical slice from roadmap without manual intervention", + "Monitor execution progress via terminal dashboard", + "Resume interrupted runs from checkpoint", + "Review findings and verdicts after milestone completion", + "Debug blocked or deferred milestones via session logs", + "Configure context pack budgets and token limits for different milestone sizes", + "Review spec drift warnings and remediate stale references", + "Validate governance gate failures and override scope budgets when appropriate", + "Configure error recovery and retry cap parameters", + "Monitor implementation loop completion and agent failure detection", + "Review SpecRefResolver errors and address spec grounding issues", + "Handle spec baseline commit policy mid-run re-plan cycles" + ], + "pains": [ + "Context loss across long-running LLM generations", + "Infinite repair loops without clear stop conditions", + "Unverified code changes without evidence-backed tests", + "Hallucinated APIs or file paths breaking implementation", + "Spec drift causing unexpected behavior mid-run", + "Token budget exhaustion causing premature truncation", + "Retry cap exhaustion without human intervention", + "Secret leakage in persisted artifacts or logs", + "Agent failure detection failures masking runtime issues", + "Error recovery failures blocking progress", + "SpecRefResolver errors blocking execution without clear remediation" + ], + "gains": [ + "Lossless persistence allows resume from any checkpoint", + "Evidence bindings provide verifiable proof of test pass/fail", + "Spec authority prevents unauthorized changes to governed artifacts", + "Retry caps prevent infinite loops", + "Terminal dashboard provides real-time visibility into milestone state", + "Context pack budget enforcement prevents runaway token inflation", + "Spec drift detection surfaces stale references proactively", + "Secret safety enforcement prevents credential leakage", + "Agent failure detection identifies runtime issues early", + "Error recovery success rate indicates system resilience", + "Implementation loop completion rate guarantees milestone progress", + "SpecRefResolver errors surface spec grounding problems clearly" + ] + }, + { + "segment_id": "spec-author", + "description": "Domain experts who author and maintain governed specification artifacts", + "jobs_to_be_done": [ + "Update seed-manifest and governed spec files (FRs, NFRs, governance, etc.)", + "Maintain seed-manifest governance and authority set", + "Validate spec changes before committing as baseline", + "Review spec_ref grounding and line_range accuracy in implementation artifacts", + "Ensure spec drift detection identifies stale references", + "Add new spec artifacts to authority set with proper grounding", + "Update spec_ref line ranges when implementing changes", + "Review drift warnings after spec changes are committed", + "Handle SpecRefResolver staleness detection for historical references", + "Verify spec baseline commit policy allows controlled re-plan cycles" + ], + "pains": [ + "Spec changes breaking existing implementation traces", + "Spec_ref grounding failing due to missing or stale line ranges", + "Seed-manifest governance becoming outdated", + "Spec drift not detected until runtime failures occur", + "Manual verification of spec_ref commit hashes and line ranges", + "Reconciliation of spec changes across multiple milestones", + "SpecRefResolver staleness not detected until runtime", + "Spec baseline commit policy complexity for mid-run changes" + ], + "gains": [ + "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", + "Drift warnings surface stale references before runtime execution", + "Spec authority prevents unauthorized mutations to governed artifacts", + "Seed-manifest order ensures consistent context resolution across runs", + "Automated drift detection saves manual verification effort", + "Commit hash grounding ensures reproducible spec references", + "SpecRefResolver staleness detection surfaces historical changes", + "Spec baseline commit policy allows controlled re-plan cycles" + ] + }, + { + "segment_id": "eval-researcher", + "description": "Researchers and ML engineers who analyze execution traces for model training datasets", + "jobs_to_be_done": [ + "Export session logs to OpenAI-style messages format", + "Validate export rows against eval_export_row.schema.json", + "Apply deterministic redaction profiles to sensitive data", + "Create replay artifacts for dataset generation", + "Analyze validation gate outcomes and evidence bindings", + "Validate session log completeness and replay metadata", + "Create eval datasets from verified milestones", + "Analyze retry cap usage patterns and failure modes", + "Review agent failure detection events for eval improvement", + "Analyze error recovery success rates and failure patterns" + ], + "pains": [ + "Sensitive data (API keys, tokens) in session logs or prompt artifacts", + "Lack of structured export format for ML pipelines", + "Redaction applied inconsistently across different artifact types", + "Loss of replay metadata makes dataset reconstruction difficult", + "Incomplete session logs missing events or lineage", + "Redaction profiles not consistently applied across all artifacts", + "Agent failure detection events not captured in export", + "Error recovery failures not included in eval datasets" + ], + "gains": [ + "Eval strategy provides native event schema as source-of-truth", + "OpenAI-style messages export with validation", + "Deterministic redaction profiles applied to all persisted artifacts", + "Full lineage metadata (event_sequence, prev_event_sha256, artifact_sha256) for replay", + "Session log rotation preserves archival segments for long-term eval", + "Secret safety enforcement ensures export datasets are safe for external use", + "Agent failure detection events captured for eval improvement", + "Error recovery success rates tracked for eval dataset quality" + ] + }, + { + "segment_id": "platform-engineer", + "description": "Platform engineers responsible for Trinity runtime infrastructure and tooling", + "jobs_to_be_done": [ + "Implement and maintain SpecRefResolver with git grounding", + "Implement and maintain validation gate schemas and validators", + "Implement and maintain scratchpad serialization/deserialization", + "Implement and maintain session log rotation and archival", + "Implement and maintain secret scanning and redaction pipelines", + "Implement and maintain tool protocol schemas and validators", + "Monitor spawn success rates and state transition success rates", + "Debug validation gate failures and optimize performance", + "Implement error recovery and blocker resolution mechanisms", + "Implement agent failure detection and handling", + "Implement Spec baseline commit policy and re-plan cycles", + "Implement SpecRefResolver staleness detection" + ], + "pains": [ + "SpecRefResolver failures due to missing git history or invalid commit hashes", + "Validation gate performance bottlenecks in long-running sessions", + "Scratchpad corruption after crashes or token boundary handling", + "Session log rotation causing data loss or replay failures", + "Secret scanning false positives blocking legitimate artifacts", + "Tool protocol schema drift between prompt and runtime", + "Agent turn count explosion due to inefficient planning", + "Error recovery failures blocking progress", + "Blocker resolution failures preventing progress", + "Spec baseline commit policy complexity for re-plan cycles" + ], + "gains": [ + "SpecRefResolver provides deterministic provenance with commit hashes and line ranges", + "Validation gates prevent invalid artifacts before commit", + "Scratchpad recovery ensures crash-tolerant execution", + "Session log rotation preserves archival segments without data loss", + "Secret scanning prevents leakage without false positives", + "Tool protocol schemas guarantee type safety and deterministic behavior", + "Spawn success and state transition success metrics indicate runtime health", + "Error recovery success rate indicates system resilience", + "Implementation loop completion rate guarantees milestone progress", + "Blocker resolution rate ensures progress can be maintained" + ] + }, + { + "segment_id": "compliance-officer", + "description": "Compliance officers who ensure Trinity execution adheres to organizational policies and regulations", + "jobs_to_be_done": [ + "Review session logs for compliance violations", + "Validate governance gate compliance on all commits", + "Verify spec authority enforcement prevents unauthorized changes", + "Ensure secret safety enforcement meets regulatory requirements", + "Audit spec drift detection for historical spec changes", + "Review session log completeness for audit trail requirements", + "Validate agent turn counts and execution effort for scope verification", + "Review error recovery success rates for system resilience", + "Verify agent failure detection captures all relevant events" + ], + "pains": [ + "Unable to trace execution history for audit requirements", + "Unauthorized spec changes slipping through governance gates", + "Credential leakage in logs or artifacts", + "Incomplete execution records missing events or lineage", + "Spec drift not detected until after compliance review", + "Governance gate failures not properly documented", + "Agent failure detection events not captured in audit trail", + "Error recovery failures not included in compliance review" + ], + "gains": [ + "Spec authority enforcement prevents unauthorized spec modifications", + "Secret safety enforcement maintains auditability of sensitive data", + "Session logs capture complete execution history for regulatory review", + "Governance gate compliance ensures all commits meet policy requirements", + "Spec drift detection surfaces historical spec changes for compliance verification", + "Session log completeness ensures replayable audit trails", + "Agent failure detection events captured for complete audit trail", + "Error recovery success rates tracked for compliance review" + ] + }, + { + "segment_id": "site-reliability-engineer", + "description": "Site Reliability Engineers responsible for system reliability, monitoring, and incident response", + "jobs_to_be_done": [ + "Monitor error recovery success rates and alert on failures", + "Track blocker resolution rates and alert on bottlenecks", + "Review agent failure detection events for incident response", + "Validate SpecRefResolver stability and error handling", + "Monitor session log rotation and archival for disk space management", + "Track context budget utilization for resource optimization", + "Review implementation loop completion for capacity planning", + "Validate governance gate compliance for system health" + ], + "pains": [ + "Error recovery failures blocking progress without visibility", + "Blocker resolution failures preventing milestone completion", + "Agent failure detection events not surfaced for incident response", + "SpecRefResolver instability causing runtime failures", + "Session log rotation causing disk space exhaustion", + "Context budget exhaustion causing premature truncation", + "Implementation loop completion failures indicating system issues" + ], + "gains": [ + "Error recovery success rate indicates system resilience", + "Blocker resolution rate ensures progress can be maintained", + "Agent failure detection identifies runtime issues early", + "SpecRefResolver errors surface spec grounding problems", + "Session log rotation prevents disk space exhaustion", + "Context budget utilization ensures optimal resource usage", + "Implementation loop completion rate guarantees milestone progress" + ] + } + ], + "success_metrics": [ + { + "metric_id": "trinity-m1-coverage", + "name": "Spec authority compliance", + "baseline": 95, + "target": 99, + "unit": "percent", + "measurement_method": "Validate that all tool calls and file writes reference governed seed-manifest artifacts; track violations in session logs and aggregate as percentage of total operations" + }, + { + "metric_id": "trinity-m2-evidence-binding", + "name": "Evidence binding coverage", + "baseline": 0, + "target": 100, + "unit": "percent of checklist items", + "measurement_method": "Count checklist items with valid evidence excerpts and SHA-256 hashes in execution.execution_results[]; divide by total checklist items for milestone" + }, + { + "metric_id": "trinity-m3-resume-success", + "name": "Resume success rate", + "baseline": 0, + "target": 95, + "unit": "percent of resumed runs", + "measurement_method": "Track resume attempts from checkpoint; record success if milestone completes within token budget and without context loss; measure success rate over 50 resume attempts" + }, + { + "metric_id": "trinity-m4-secret-detection", + "name": "Secret detection coverage", + "baseline": 0, + "target": 100, + "unit": "percent of persisted artifacts", + "measurement_method": "Scan all persisted prompt/response/session/artifact files using secret_scanner_v1; track detections and false negatives; calculate coverage as files scanned / files persisted" + }, + { + "metric_id": "trinity-m5-retry-cap-usage", + "name": "Retry cap utilization", + "baseline": 0, + "target": 20, + "unit": "percent of milestone runs", + "measurement_method": "Count milestone runs where retry caps are exceeded; divide by total milestone runs; track by state (16a/16b/16c) to identify failure modes" + }, + { + "metric_id": "trinity-m6-spawn-success-rate", + "name": "Child spawn success rate", + "baseline": 95, + "target": 99, + "unit": "percent of spawns", + "measurement_method": "Track all child spawns (16a/16b/16c/utility) and count successful spawns; divide by total spawns; exclude blocked/deferred states" + }, + { + "metric_id": "trinity-m7-state-transition-success-rate", + "name": "State transition success rate", + "baseline": 95, + "target": 99, + "unit": "percent of transitions", + "measurement_method": "Count valid state transitions (16a→16b→16c) that complete without validation failures; divide by total attempted transitions" + }, + { + "metric_id": "trinity-m8-validation-gate-pass-rate", + "name": "Validation gate pass rate", + "baseline": 90, + "target": 98, + "unit": "percent of gates", + "measurement_method": "Track all validation gates (schema, deep, governance, spec authority); count successful passes; divide by total gate evaluations" + }, + { + "metric_id": "trinity-m9-spec-drift-detection-coverage", + "name": "Spec drift detection coverage", + "baseline": 0, + "target": 100, + "unit": "percent of spec_ref references", + "measurement_method": "Count spec_ref records with successful drift detection (stale content or commit hash mismatch); divide by total spec_ref references in session logs" + }, + { + "metric_id": "trinity-m10-context-budget-utilization", + "name": "Context budget utilization", + "baseline": 0, + "target": 85, + "unit": "percent of hard token limit", + "measurement_method": "Calculate average context_pack token usage across all state transitions; divide by hard_token_limit; track by phase (16a/16b/16c)" + }, + { + "metric_id": "trinity-m11-context-truncation-events", + "name": "Context truncation events", + "baseline": 0, + "target": 5, + "unit": "events per milestone run", + "measurement_method": "Count context_budget_truncation events in session logs; track by cause (token overflow, priority item truncation)" + }, + { + "metric_id": "trinity-m12-specref-resolver-success-rate", + "name": "SpecRefResolver success rate", + "baseline": 95, + "target": 99, + "unit": "percent of lookups", + "measurement_method": "Count successful SpecRefResolver calls (path, line_range, commit_hash resolved) divided by total lookups; track failures by type" + }, + { + "metric_id": "trinity-m13-session-log-completeness", + "name": "Session log completeness", + "baseline": 90, + "target": 99, + "unit": "percent of events", + "measurement_method": "Validate session log events contain required fields (event_sequence, prev_event_sha256, event_sha256, artifact_ref, artifact_sha256); count complete events divided by total events" + }, + { + "metric_id": "trinity-m14-scratchpad-recovery-success", + "name": "Scratchpad recovery success rate", + "baseline": 0, + "target": 95, + "unit": "percent of recovery attempts", + "measurement_method": "Track scratchpad recovery attempts after crash or token boundary; count successful loads; measure over 50 recovery events" + }, + { + "metric_id": "trinity-m15-governance-gate-compliance", + "name": "Governance gate compliance", + "baseline": 95, + "target": 99, + "unit": "percent of commits", + "measurement_method": "Count commits passing governance checks (seed-manifest authority, spec_ref grounding, scope adherence) divided by total commits; track failures by gate type" + }, + { + "metric_id": "trinity-m16-agent-turn-count", + "name": "Agent turn count per milestone", + "baseline": 50, + "target": 100, + "unit": "turns", + "measurement_method": "Count total agent turns (messages sent/received) per milestone run; track distribution by state (16a/16b/16c/utility)" + }, + { + "metric_id": "trinity-m17-session-log-rotation-events", + "name": "Session log rotation events", + "baseline": 0, + "target": 3, + "unit": "events per milestone run", + "measurement_method": "Count session_log_compaction_threshold exceeded events; track by rotation cause (event count, time boundary)" + }, + { + "metric_id": "trinity-m18-workspace-cleanup-completeness", + "name": "Workspace cleanup completeness", + "baseline": 95, + "target": 100, + "unit": "percent of cleanup operations", + "measurement_method": "Track workspace cleanup operations after milestone closure; verify workspace artifacts and session logs are archived for future reference; count successful archive operations divided by total cleanup attempts; ensure logs are preserved, not deleted" + }, + { + "metric_id": "trinity-m19-spawn-efficiency-overhead", + "name": "Spawn efficiency and overhead", + "baseline": 5, + "target": 2, + "unit": "seconds per spawn", + "measurement_method": "Measure average time from spawn initiation to child task_input.json creation; track overhead reduction over time; exclude blocked/deferred spawns" + }, + { + "metric_id": "trinity-m20-spawn-utilization-rate", + "name": "Spawn utilization rate", + "baseline": 70, + "target": 90, + "unit": "percent of spawns", + "measurement_method": "Count effective spawns that contribute to milestone progress vs redundant or blocked spawns; divide effective spawns by total spawns; track by utility type (Researcher, ToolUser, Summarizer, Auditor)" + }, + { + "metric_id": "trinity-m21-spawn-loop-prevention-effectiveness", + "name": "Spawn loop prevention effectiveness", + "baseline": 0, + "target": 95, + "unit": "percent of duplicate spawns prevented", + "measurement_method": "Count duplicate spawn attempts detected by spawn_log loop detection; verify spawn_log prevents exceeding configured retry caps; measure prevention effectiveness over 100 milestone runs" + }, + { + "metric_id": "trinity-m22-error-recovery-success-rate", + "name": "Error recovery success rate", + "baseline": 0, + "target": 90, + "unit": "percent of error states", + "measurement_method": "Track error states (validation failures, schema violations, blocked conditions) and count successful recoveries; measure success rate over 100 error recovery attempts" + }, + { + "metric_id": "trinity-m23-blocker-resolution-rate", + "name": "Blocker resolution rate", + "baseline": 0, + "target": 85, + "unit": "percent of blockers", + "measurement_method": "Track blocker conditions (missing seed, out-of-scope writes, security concerns, retry cap exhaustion) and count successful resolutions before cap exhaustion; measure success rate over 100 blocker events" + }, + { + "metric_id": "trinity-m24-implementation-loop-completion-rate", + "name": "Implementation loop completion rate", + "baseline": 0, + "target": 95, + "unit": "percent of checklist units", + "measurement_method": "Count checklist units (atomic L3 loops) that complete successfully vs fail; track by state (16a/16b/16c); measure success rate over 100 units" + }, + { + "metric_id": "trinity-m25-agent-failure-detection-rate", + "name": "Agent failure detection rate", + "baseline": 95, + "target": 99, + "unit": "percent of agent failures", + "measurement_method": "Track agent failures (timeout, blocked, ambiguous, hallucination) and count successful detection in session logs; measure detection rate over 100 agent failure events" + } + ], + "links": [ + { + "source": "project_charter-trinity", + "target": "fr-16a-planner", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "fr-16b-builder", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "fr-16c-verifier", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-context-budget", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-evidence-integrity", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-secret-safety", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-spec-authority", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "seed-manifest-common", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "trinity_spec", + "relation": "upstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-session-management", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-validation-gates", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-spawn-protocol", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-agent-spawning", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-context-pack", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "governance-16-secret-scanning", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-spawn-success-rate", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-state-transition-success", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-validation-gate-pass", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-spec-drift-detection", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-context-truncation", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-specref-resolver", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-session-log-completeness", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-scratchpad-recovery", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-governance-gate-compliance", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-agent-turn-count", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-session-rotation", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-workspace-cleanup", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-error-recovery-success", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-blocker-resolution-rate", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-implementation-loop-completion", + "relation": "downstream" + }, + { + "source": "project_charter-trinity", + "target": "nfr-16-agent-failure-detection", + "relation": "downstream" + } + ] +} \ No newline at end of file