From db693cc00933d84869bb198c44763b88c8693a7e Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sat, 3 Oct 2026 21:40:49 +0200 Subject: [PATCH] =?UTF-8?q?test(tutor-quality):=20run=20Tutor=20Quality=20?= =?UTF-8?q?scenarios=20with=20the=20real=20planner=20only=20(TutorQuality?= =?UTF-8?q?=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TutorQuality SSOT §5: the scenario runner "MUST use TutorService and, where Course Content is supplied, CurriculumTutorAdapter. It MUST fake only the external provider." The runner accepted a `planning` override, and five scenarios used it to script transition, blocked, and plan outcomes; the TQ-REG-001 progression scenario even faked the planner while Course Content was supplied. Those scenarios therefore checked TutorService against invented planning outcomes, not the real planner. The runner no longer takes a planning override. The scenarios now reach the same outcomes through Course Content states: - transition: a strong answer completes LEARN mastery of a Topic with no further LEARN question; - blocked: a DEEPEN answer with no further post-mastery question ends in content-exhausted; - plan authority: the real initial plan replaces the provider question; - TQ-REG-001 progression: a minimal PWM test Topic, mastered by one strong answer, transitions LEARN -> DEEPEN; the answer is served under LEARN (R-PH-1) and the state commits DEEPEN. A mutation that drops the repaired question for transition/blocked outcomes fails all five rewritten scenarios. Co-Authored-By: Claude Opus 5.5 --- .../support/tutor-quality-scenario-runner.ts | 7 +- .../services/tutor/tq-reg-001-pwm.test.ts | 85 ++++++---- .../tutor/tutor-quality-scenarios.test.ts | 157 ++++++++---------- 3 files changed, 125 insertions(+), 124 deletions(-) diff --git a/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts b/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts index 203b05048..543c3f4c2 100644 --- a/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts +++ b/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts @@ -2,7 +2,7 @@ import type { TutorContentResult, TutorDialogTurn, TutorDifficulty } from "@shar import type { LLMProvider, LLMProviderRequest } from "../../../../../server/services/tutor/llm-provider"; import { TutorService } from "../../../../../server/services/tutor/tutor-service"; import { CurriculumTutorAdapter } from "../../../../../server/services/tutor/curriculum-tutor-adapter"; -import type { TutorPlanningContentContext, TutorPlanningExtension } from "../../../../../server/services/tutor/tutor-planning"; +import type { TutorPlanningContentContext } from "../../../../../server/services/tutor/tutor-planning"; import type { TutorProgressionState } from "../../../../../server/services/tutor/curriculum/progression-state"; type ProviderBehavior = @@ -25,7 +25,6 @@ export interface TutorQualityScenario { readonly provider: ProviderBehavior; readonly difficulty?: TutorDifficulty; readonly courseContent?: TutorPlanningContentContext; - readonly planning?: TutorPlanningExtension; } export interface TutorQualityScenarioTrace { @@ -53,7 +52,9 @@ export async function runTutorQualityScenario(scenario: TutorQualityScenario): P }, }; const stateBefore = cloneState(scenario.courseContent?.progressionState); - const planning = scenario.planning ?? (scenario.courseContent ? new CurriculumTutorAdapter() : undefined); + // TutorQuality SSOT §5: only the external provider is faked; Course Content is always planned by + // the real adapter, so a scenario cannot script a planning outcome. + const planning = scenario.courseContent ? new CurriculumTutorAdapter() : undefined; const service = new TutorService(provider, planning); try { const response = scenario.action.kind === "initial" diff --git a/tests/server/services/tutor/tq-reg-001-pwm.test.ts b/tests/server/services/tutor/tq-reg-001-pwm.test.ts index 0052953a9..3df9c9113 100644 --- a/tests/server/services/tutor/tq-reg-001-pwm.test.ts +++ b/tests/server/services/tutor/tq-reg-001-pwm.test.ts @@ -1,10 +1,9 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { parse as parseYaml } from "yaml"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { curriculumTopicSchema, validateCurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; import { createTutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state"; -import type { TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning"; import { runTutorQualityScenario } from "./support/tutor-quality-scenario-runner"; const revision = "1".repeat(40); @@ -17,13 +16,62 @@ function fixture(name: string): string { return readFileSync(fileURLToPath(new URL(`../../../fixtures/tutor-quality/${name}`, import.meta.url)), "utf8"); } -function courseContent() { +const repeatedQuestion = "Welchen Wert verwendet der Sketch an der betrachteten Integer-Variablen?"; + +// Minimal test Topic for the PWM sketch. The fact extractor yields only `type-used: int` for it, so +// the Topic activates on that fact. Its single question is mastered by one strong answer, so the +// real planner ends the turn in a LEARN -> DEEPEN transition (TutorQuality SSOT §5). +const pwmTopic = validateCurriculumTopic(curriculumTopicSchema.parse({ + schemaVersion: 1, + id: "pwm-output", + title: "PWM-Ausgabe", + locale: "de-DE", + activation: { any: [{ fact: "type-used", values: ["int"] }] }, + concepts: [{ + id: "pwm-duty-value", + title: "Tastgrad-Wert", + objective: "Den Integerwert erklären, der den PWM-Tastgrad bestimmt.", + prerequisites: [], + difficulty: { entry: [1, 60], transfer: [20, 80] }, + misconceptions: [], + indicators: [{ id: "relates-value-to-duty", description: "Ordnet den Integerwert dem Tastgrad zu." }], + mastery: { + minimumSuccessfulProbes: 1, + successRatingAtLeast: 3, + requiredIndicators: ["relates-value-to-duty"], + minimumDistinctQuestionKinds: 1, + recentWeakAnswersAllowed: 0, + }, + }], + questions: [{ + id: "pwm-duty-value-question", + concept: "pwm-duty-value", + indicator: "relates-value-to-duty", + kind: "concept", + difficulty: [1, 60], + requires: [{ fact: "type-used", values: ["int"] }], + text: repeatedQuestion, + }], + scaffolds: [], + progression: { + entryConcepts: ["pwm-duty-value"], + preferredOrder: ["pwm-duty-value"], + onRating: { + "1-2": "remediate", + "3": "clarify-same-indicator", + "4": "probe-missing-indicator", + "5": "evaluate-mastery-and-advance", + }, + }, +})); + +function courseContent(topics = [variablesTopic]) { return { revision, tutor: { status: "valid" as const, manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, - topics: [variablesTopic], + topics, strategies: [], }, progressionState: createTutorProgressionState(revision), @@ -48,29 +96,7 @@ describe("TQ-REG-001 PWM question quality regression", () => { }); it("repairs repetition after a strong answer and commits coherent transition metadata", async () => { - const content = courseContent(); - const repeatedQuestion = "Welchen Wert verwendet der Sketch an der betrachteten Integer-Variablen?"; - const planning: TutorPlanningExtension = { - planInitial: vi.fn().mockResolvedValue(null), - planFollowup: vi.fn(async ({ courseContent: workingContent }) => { - const state = workingContent?.progressionState; - if (state) { - state.activeTopicId = "pwm-output"; - state.phase = "DEEPEN"; - state.masteredTopicIds.push("pwm-output"); - state.retainedPhases["pwm-output"] = "DEEPEN"; - } - return { - kind: "transition" as const, - contentRevision: revision, - learningPhase: "DEEPEN" as const, - activeTopicId: "pwm-output", - masteredTopicIds: ["pwm-output"], - strategyId: "built-in-default", - strategySource: "built-in" as const, - }; - }), - }; + const content = courseContent([variablesTopic, pwmTopic]); const trace = await runTutorQualityScenario({ id: "TQ-REG-001/progression", code: pwmSketch, @@ -89,18 +115,19 @@ describe("TQ-REG-001 PWM question quality regression", () => { }, }, courseContent: content, - planning, }); expect(trace.error).toBeUndefined(); expect(trace.result?.question).not.toContain("betrachtete Integer-Variable"); + // R-PH-1/R-PH-2: the answer is served under LEARN; the transition is committed to state. expect(trace.result).toMatchObject({ answerRating: 5, contentRevision: revision, - learningPhase: "DEEPEN", + learningPhase: "LEARN", activeTopicId: "pwm-output", masteredTopicIds: ["pwm-output"], }); + expect(trace.result).not.toHaveProperty("topicId", "variables-and-serial"); expect(trace.stateAfter).toMatchObject({ revision, phase: "DEEPEN", diff --git a/tests/server/services/tutor/tutor-quality-scenarios.test.ts b/tests/server/services/tutor/tutor-quality-scenarios.test.ts index 945fa0ad9..24d8bff35 100644 --- a/tests/server/services/tutor/tutor-quality-scenarios.test.ts +++ b/tests/server/services/tutor/tutor-quality-scenarios.test.ts @@ -1,60 +1,35 @@ -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { TutorProviderError } from "../../../../server/services/tutor/llm-provider"; -import type { TutorPlan, TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning"; import { BUILT_IN_TUTOR_STRATEGY } from "../../../../server/services/tutor/strategy/effective-tutor-strategy"; import { createTutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state"; -import type { CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; +import type { CurriculumQuestion, CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; import { runTutorQualityScenario } from "./support/tutor-quality-scenario-runner"; const revision = "a".repeat(40); const nextRevision = "b".repeat(40); const sketch = "int value = 3; void setup() { Serial.println(value); } void loop() {}"; -function metadataOutcome(kind: "transition" | "blocked"): Awaited> { - const metadata = { - contentRevision: revision, - learningPhase: "DEEPEN", - activeTopicId: "variables-and-serial", - masteredTopicIds: ["variables-and-serial"], - strategyId: "built-in-default", - strategySource: "built-in", - } as const; - return kind === "blocked" - ? { kind, progressionBlockedReason: "content-exhausted", ...metadata, masteredTopicIds: [...metadata.masteredTopicIds] } - : { kind, ...metadata, masteredTopicIds: [...metadata.masteredTopicIds] }; -} +const FIRST_QUESTION: CurriculumQuestion = { + id: "first-question", + concept: "variable-values", + indicator: "value-use", + kind: "concept", + difficulty: [1, 50], + requires: [{ fact: "serial-call", values: ["print"] }], + text: "Welche Rolle hat value im Sketch?", +}; -function planningWithFollowup(kind: "transition" | "blocked"): TutorPlanningExtension { - return { - planInitial: vi.fn().mockResolvedValue(null), - planFollowup: vi.fn().mockResolvedValue(metadataOutcome(kind)), - }; -} +const SECOND_QUESTION: CurriculumQuestion = { + id: "second-question", + concept: "variable-values", + indicator: "value-use", + kind: "application", + difficulty: [1, 50], + requires: [{ fact: "serial-call", values: ["print"] }], + text: "Wie wird value bei der seriellen Ausgabe verwendet?", +}; -function tutorPlan(): TutorPlan { - return { - topicId: "variables-and-serial", - topicTitle: "Variablen", - conceptId: "variable-values", - conceptTitle: "Variablenwerte", - objective: "Werte erklären", - questionId: "planned-question", - questionKind: "application", - indicatorId: "value-use", - indicator: "Ordnet Wert und Verwendung zu", - question: "Wie wird value im Serial.println-Aufruf des Sketches verwendet?", - misconceptions: [], - contentRevision: revision, - strategyId: BUILT_IN_TUTOR_STRATEGY.id, - strategySource: "built-in", - effectiveStrategy: BUILT_IN_TUTOR_STRATEGY, - learningPhase: "LEARN", - activeTopicId: "variables-and-serial", - masteredTopicIds: [], - }; -} - -function topic(): CurriculumTopic { +function topic(questions: readonly CurriculumQuestion[] = [FIRST_QUESTION, SECOND_QUESTION]): CurriculumTopic { return { schemaVersion: 1, id: "variables-and-serial", @@ -77,26 +52,7 @@ function topic(): CurriculumTopic { recentWeakAnswersAllowed: 0, }, }], - questions: [ - { - id: "first-question", - concept: "variable-values", - indicator: "value-use", - kind: "concept", - difficulty: [1, 50], - requires: [{ fact: "serial-call", values: ["print"] }], - text: "Welche Rolle hat value im Sketch?", - }, - { - id: "second-question", - concept: "variable-values", - indicator: "value-use", - kind: "application", - difficulty: [1, 50], - requires: [{ fact: "serial-call", values: ["print"] }], - text: "Wie wird value bei der seriellen Ausgabe verwendet?", - }, - ], + questions: questions.map((question) => ({ ...question })), scaffolds: [], progression: { entryConcepts: ["variable-values"], @@ -111,56 +67,72 @@ function topic(): CurriculumTopic { }; } -function courseContent(contentRevision = revision) { +function courseContent(contentRevision = revision, questions?: readonly CurriculumQuestion[]) { return { revision: contentRevision, tutor: { status: "valid" as const, manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, - topics: [topic()], + topics: [topic(questions)], strategies: [], }, progressionState: createTutorProgressionState(contentRevision), }; } +// The Topic is mastered through FIRST_QUESTION; SECOND_QUESTION is its only post-mastery question. +function deepenContent() { + const content = courseContent(); + content.progressionState = { + ...createTutorProgressionState(revision), + activeTopicId: "variables-and-serial", + phase: "DEEPEN", + masteredTopicIds: ["variables-and-serial"], + masteryEvidence: { + "variables-and-serial": [{ questionId: FIRST_QUESTION.id, conceptId: "variable-values", indicatorId: "value-use", kind: "concept", rating: 5 }], + }, + retainedPhases: { "variables-and-serial": "DEEPEN" }, + }; + return content; +} + describe("Tutor Quality deterministic scenarios", () => { + // §3 items 5 and 6 with real planning outcomes (§5): a strong answer that completes LEARN mastery + // ends in a transition (the Topic has no further LEARN question), and a DEEPEN answer with no + // further post-mastery question ends blocked. Neither outcome carries a question, so the repaired + // provider question must survive. it.each([ - ["transition", "exact", "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"], - ["transition", "near", "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?"], - ["blocked", "exact", "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"], - ["blocked", "near", "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?"], + ["transition", "exact", FIRST_QUESTION.text!], + ["transition", "near", "Welche Rolle spielt value in diesem Sketch?"], + ["blocked", "exact", SECOND_QUESTION.text!], + ["blocked", "near", "Wie wird value in der seriellen Ausgabe verwendet?"], ] as const)("retains a repaired %s/%s duplicate question", async (kind, _similarity, candidate) => { - const currentQuestion = "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"; + const content = kind === "transition" ? courseContent(revision, [FIRST_QUESTION]) : deepenContent(); + const currentQuestion = kind === "transition" ? FIRST_QUESTION.text! : SECOND_QUESTION.text!; const trace = await runTutorQualityScenario({ id: `TQ-ADV-repeat-${kind}`, code: sketch, action: { kind: "dialog", question: currentQuestion, answer: "value hat den Wert 3 und wird ausgegeben." }, - provider: { - kind: "result", - result: { - answerRating: 5, - question: candidate, - }, - }, - planning: planningWithFollowup(kind), + provider: { kind: "result", result: { answerRating: 5, question: candidate } }, + courseContent: content, }); expect(trace.error).toBeUndefined(); + expect(trace.result?.question).not.toBe(candidate); + expect(trace.result?.question).not.toBe(currentQuestion); + expect(trace.result).not.toHaveProperty("questionId"); expect(trace.result).toMatchObject({ - learningPhase: "DEEPEN", contentRevision: revision, activeTopicId: "variables-and-serial", + masteredTopicIds: ["variables-and-serial"], + ...(kind === "transition" + ? { learningPhase: "LEARN" } + : { learningPhase: "DEEPEN", progressionBlockedReason: "content-exhausted" }), }); - expect(trace.result?.question).not.toBe(candidate); + expect(trace.stateAfter).toMatchObject({ activeTopicId: "variables-and-serial", phase: "DEEPEN" }); }); it("lets an application-owned TutorPlan replace provider question and metadata", async () => { - const plan = tutorPlan(); - const planning: TutorPlanningExtension = { - planInitial: vi.fn().mockResolvedValue(plan), - planFollowup: vi.fn().mockResolvedValue(null), - }; const trace = await runTutorQualityScenario({ id: "TQ-ADV-plan-authority", code: sketch, @@ -174,13 +146,14 @@ describe("Tutor Quality deterministic scenarios", () => { learningPhase: "EXPAND", }, }, - planning, + courseContent: courseContent(revision, [FIRST_QUESTION]), }); + expect(trace.error).toBeUndefined(); expect(trace.result).toMatchObject({ - question: plan.question, - topicId: plan.topicId, - questionId: plan.questionId, + question: FIRST_QUESTION.text, + topicId: "variables-and-serial", + questionId: FIRST_QUESTION.id, learningPhase: "LEARN", }); });