test(tutor-quality): run Tutor Quality scenarios with the real planner only (TutorQuality §5)

tests/server/services/tutor/support/tutor-quality-scenario-runner.ts

@@ -2,7 +2,7 @@ import type { TutorContentResult, TutorDialogTurn, TutorDifficulty } from "@shar
 import type { LLMProvider, LLMProviderRequest } from "../../../../../server/services/tutor/llm-provider";
 import { TutorService } from "../../../../../server/services/tutor/tutor-service";
 import { CurriculumTutorAdapter } from "../../../../../server/services/tutor/curriculum-tutor-adapter";
-import type { TutorPlanningContentContext, TutorPlanningExtension } from "../../../../../server/services/tutor/tutor-planning";
+import type { TutorPlanningContentContext } from "../../../../../server/services/tutor/tutor-planning";
 import type { TutorProgressionState } from "../../../../../server/services/tutor/curriculum/progression-state";
 
 type ProviderBehavior =
@@ -25,7 +25,6 @@ export interface TutorQualityScenario {
   readonly provider: ProviderBehavior;
   readonly difficulty?: TutorDifficulty;
   readonly courseContent?: TutorPlanningContentContext;
-  readonly planning?: TutorPlanningExtension;
 }
 
 export interface TutorQualityScenarioTrace {
@@ -53,7 +52,9 @@ export async function runTutorQualityScenario(scenario: TutorQualityScenario): P
     },
   };
   const stateBefore = cloneState(scenario.courseContent?.progressionState);
-  const planning = scenario.planning ?? (scenario.courseContent ? new CurriculumTutorAdapter() : undefined);
+  // TutorQuality SSOT §5: only the external provider is faked; Course Content is always planned by
+  // the real adapter, so a scenario cannot script a planning outcome.
+  const planning = scenario.courseContent ? new CurriculumTutorAdapter() : undefined;
   const service = new TutorService(provider, planning);
   try {
     const response = scenario.action.kind === "initial"

tests/server/services/tutor/tq-reg-001-pwm.test.ts

@@ -1,10 +1,9 @@
 import { readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { parse as parseYaml } from "yaml";
-import { describe, expect, it, vi } from "vitest";
+import { describe, expect, it } from "vitest";
 import { curriculumTopicSchema, validateCurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema";
 import { createTutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state";
-import type { TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning";
 import { runTutorQualityScenario } from "./support/tutor-quality-scenario-runner";
 
 const revision = "1".repeat(40);
@@ -17,13 +16,62 @@ function fixture(name: string): string {
   return readFileSync(fileURLToPath(new URL(`../../../fixtures/tutor-quality/${name}`, import.meta.url)), "utf8");
 }
 
-function courseContent() {
+const repeatedQuestion = "Welchen Wert verwendet der Sketch an der betrachteten Integer-Variablen?";
+
+// Minimal test Topic for the PWM sketch. The fact extractor yields only `type-used: int` for it, so
+// the Topic activates on that fact. Its single question is mastered by one strong answer, so the
+// real planner ends the turn in a LEARN -> DEEPEN transition (TutorQuality SSOT §5).
+const pwmTopic = validateCurriculumTopic(curriculumTopicSchema.parse({
+  schemaVersion: 1,
+  id: "pwm-output",
+  title: "PWM-Ausgabe",
+  locale: "de-DE",
+  activation: { any: [{ fact: "type-used", values: ["int"] }] },
+  concepts: [{
+    id: "pwm-duty-value",
+    title: "Tastgrad-Wert",
+    objective: "Den Integerwert erklären, der den PWM-Tastgrad bestimmt.",
+    prerequisites: [],
+    difficulty: { entry: [1, 60], transfer: [20, 80] },
+    misconceptions: [],
+    indicators: [{ id: "relates-value-to-duty", description: "Ordnet den Integerwert dem Tastgrad zu." }],
+    mastery: {
+      minimumSuccessfulProbes: 1,
+      successRatingAtLeast: 3,
+      requiredIndicators: ["relates-value-to-duty"],
+      minimumDistinctQuestionKinds: 1,
+      recentWeakAnswersAllowed: 0,
+    },
+  }],
+  questions: [{
+    id: "pwm-duty-value-question",
+    concept: "pwm-duty-value",
+    indicator: "relates-value-to-duty",
+    kind: "concept",
+    difficulty: [1, 60],
+    requires: [{ fact: "type-used", values: ["int"] }],
+    text: repeatedQuestion,
+  }],
+  scaffolds: [],
+  progression: {
+    entryConcepts: ["pwm-duty-value"],
+    preferredOrder: ["pwm-duty-value"],
+    onRating: {
+      "1-2": "remediate",
+      "3": "clarify-same-indicator",
+      "4": "probe-missing-indicator",
+      "5": "evaluate-mastery-and-advance",
+    },
+  },
+}));
+
+function courseContent(topics = [variablesTopic]) {
   return {
     revision,
     tutor: {
       status: "valid" as const,
       manifest: { schemaVersion: 1 as const, topics: [], strategies: [] },
-      topics: [variablesTopic],
+      topics,
       strategies: [],
     },
     progressionState: createTutorProgressionState(revision),
@@ -48,29 +96,7 @@ describe("TQ-REG-001 PWM question quality regression", () => {
   });
 
   it("repairs repetition after a strong answer and commits coherent transition metadata", async () => {
-    const content = courseContent();
-    const repeatedQuestion = "Welchen Wert verwendet der Sketch an der betrachteten Integer-Variablen?";
-    const planning: TutorPlanningExtension = {
-      planInitial: vi.fn().mockResolvedValue(null),
-      planFollowup: vi.fn(async ({ courseContent: workingContent }) => {
-        const state = workingContent?.progressionState;
-        if (state) {
-          state.activeTopicId = "pwm-output";
-          state.phase = "DEEPEN";
-          state.masteredTopicIds.push("pwm-output");
-          state.retainedPhases["pwm-output"] = "DEEPEN";
-        }
-        return {
-          kind: "transition" as const,
-          contentRevision: revision,
-          learningPhase: "DEEPEN" as const,
-          activeTopicId: "pwm-output",
-          masteredTopicIds: ["pwm-output"],
-          strategyId: "built-in-default",
-          strategySource: "built-in" as const,
-        };
-      }),
-    };
+    const content = courseContent([variablesTopic, pwmTopic]);
     const trace = await runTutorQualityScenario({
       id: "TQ-REG-001/progression",
       code: pwmSketch,
@@ -89,18 +115,19 @@ describe("TQ-REG-001 PWM question quality regression", () => {
         },
       },
       courseContent: content,
-      planning,
     });
 
     expect(trace.error).toBeUndefined();
     expect(trace.result?.question).not.toContain("betrachtete Integer-Variable");
+    // R-PH-1/R-PH-2: the answer is served under LEARN; the transition is committed to state.
     expect(trace.result).toMatchObject({
       answerRating: 5,
       contentRevision: revision,
-      learningPhase: "DEEPEN",
+      learningPhase: "LEARN",
       activeTopicId: "pwm-output",
       masteredTopicIds: ["pwm-output"],
     });
+    expect(trace.result).not.toHaveProperty("topicId", "variables-and-serial");
     expect(trace.stateAfter).toMatchObject({
       revision,
       phase: "DEEPEN",

tests/server/services/tutor/tutor-quality-scenarios.test.ts

@@ -1,60 +1,35 @@
-import { describe, expect, it, vi } from "vitest";
+import { describe, expect, it } from "vitest";
 import { TutorProviderError } from "../../../../server/services/tutor/llm-provider";
-import type { TutorPlan, TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning";
 import { BUILT_IN_TUTOR_STRATEGY } from "../../../../server/services/tutor/strategy/effective-tutor-strategy";
 import { createTutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state";
-import type { CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema";
+import type { CurriculumQuestion, CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema";
 import { runTutorQualityScenario } from "./support/tutor-quality-scenario-runner";
 
 const revision = "a".repeat(40);
 const nextRevision = "b".repeat(40);
 const sketch = "int value = 3; void setup() { Serial.println(value); } void loop() {}";
 
-function metadataOutcome(kind: "transition" | "blocked"): Awaited<ReturnType<TutorPlanningExtension["planFollowup"]>> {
-  const metadata = {
-    contentRevision: revision,
-    learningPhase: "DEEPEN",
-    activeTopicId: "variables-and-serial",
-    masteredTopicIds: ["variables-and-serial"],
-    strategyId: "built-in-default",
-    strategySource: "built-in",
-  } as const;
-  return kind === "blocked"
-    ? { kind, progressionBlockedReason: "content-exhausted", ...metadata, masteredTopicIds: [...metadata.masteredTopicIds] }
-    : { kind, ...metadata, masteredTopicIds: [...metadata.masteredTopicIds] };
-}
+const FIRST_QUESTION: CurriculumQuestion = {
+  id: "first-question",
+  concept: "variable-values",
+  indicator: "value-use",
+  kind: "concept",
+  difficulty: [1, 50],
+  requires: [{ fact: "serial-call", values: ["print"] }],
+  text: "Welche Rolle hat value im Sketch?",
+};
 
-function planningWithFollowup(kind: "transition" | "blocked"): TutorPlanningExtension {
-  return {
-    planInitial: vi.fn().mockResolvedValue(null),
-    planFollowup: vi.fn().mockResolvedValue(metadataOutcome(kind)),
-  };
-}
+const SECOND_QUESTION: CurriculumQuestion = {
+  id: "second-question",
+  concept: "variable-values",
+  indicator: "value-use",
+  kind: "application",
+  difficulty: [1, 50],
+  requires: [{ fact: "serial-call", values: ["print"] }],
+  text: "Wie wird value bei der seriellen Ausgabe verwendet?",
+};
 
-function tutorPlan(): TutorPlan {
-  return {
-    topicId: "variables-and-serial",
-    topicTitle: "Variablen",
-    conceptId: "variable-values",
-    conceptTitle: "Variablenwerte",
-    objective: "Werte erklären",
-    questionId: "planned-question",
-    questionKind: "application",
-    indicatorId: "value-use",
-    indicator: "Ordnet Wert und Verwendung zu",
-    question: "Wie wird value im Serial.println-Aufruf des Sketches verwendet?",
-    misconceptions: [],
-    contentRevision: revision,
-    strategyId: BUILT_IN_TUTOR_STRATEGY.id,
-    strategySource: "built-in",
-    effectiveStrategy: BUILT_IN_TUTOR_STRATEGY,
-    learningPhase: "LEARN",
-    activeTopicId: "variables-and-serial",
-    masteredTopicIds: [],
-  };
-}
-
-function topic(): CurriculumTopic {
+function topic(questions: readonly CurriculumQuestion[] = [FIRST_QUESTION, SECOND_QUESTION]): CurriculumTopic {
   return {
     schemaVersion: 1,
     id: "variables-and-serial",
@@ -77,26 +52,7 @@ function topic(): CurriculumTopic {
         recentWeakAnswersAllowed: 0,
       },
     }],
-    questions: [
-      {
-        id: "first-question",
-        concept: "variable-values",
-        indicator: "value-use",
-        kind: "concept",
-        difficulty: [1, 50],
-        requires: [{ fact: "serial-call", values: ["print"] }],
-        text: "Welche Rolle hat value im Sketch?",
-      },
-      {
-        id: "second-question",
-        concept: "variable-values",
-        indicator: "value-use",
-        kind: "application",
-        difficulty: [1, 50],
-        requires: [{ fact: "serial-call", values: ["print"] }],
-        text: "Wie wird value bei der seriellen Ausgabe verwendet?",
-      },
-    ],
+    questions: questions.map((question) => ({ ...question })),
     scaffolds: [],
     progression: {
       entryConcepts: ["variable-values"],
@@ -111,56 +67,72 @@ function topic(): CurriculumTopic {
   };
 }
 
-function courseContent(contentRevision = revision) {
+function courseContent(contentRevision = revision, questions?: readonly CurriculumQuestion[]) {
   return {
     revision: contentRevision,
     tutor: {
       status: "valid" as const,
       manifest: { schemaVersion: 1 as const, topics: [], strategies: [] },
-      topics: [topic()],
+      topics: [topic(questions)],
       strategies: [],
     },
     progressionState: createTutorProgressionState(contentRevision),
   };
 }
 
+// The Topic is mastered through FIRST_QUESTION; SECOND_QUESTION is its only post-mastery question.
+function deepenContent() {
+  const content = courseContent();
+  content.progressionState = {
+    ...createTutorProgressionState(revision),
+    activeTopicId: "variables-and-serial",
+    phase: "DEEPEN",
+    masteredTopicIds: ["variables-and-serial"],
+    masteryEvidence: {
+      "variables-and-serial": [{ questionId: FIRST_QUESTION.id, conceptId: "variable-values", indicatorId: "value-use", kind: "concept", rating: 5 }],
+    },
+    retainedPhases: { "variables-and-serial": "DEEPEN" },
+  };
+  return content;
+}
+
 describe("Tutor Quality deterministic scenarios", () => {
+  // §3 items 5 and 6 with real planning outcomes (§5): a strong answer that completes LEARN mastery
+  // ends in a transition (the Topic has no further LEARN question), and a DEEPEN answer with no
+  // further post-mastery question ends blocked. Neither outcome carries a question, so the repaired
+  // provider question must survive.
   it.each([
-    ["transition", "exact", "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"],
-    ["transition", "near", "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?"],
-    ["blocked", "exact", "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"],
-    ["blocked", "near", "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?"],
+    ["transition", "exact", FIRST_QUESTION.text!],
+    ["transition", "near", "Welche Rolle spielt value in diesem Sketch?"],
+    ["blocked", "exact", SECOND_QUESTION.text!],
+    ["blocked", "near", "Wie wird value in der seriellen Ausgabe verwendet?"],
   ] as const)("retains a repaired %s/%s duplicate question", async (kind, _similarity, candidate) => {
-    const currentQuestion = "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?";
+    const content = kind === "transition" ? courseContent(revision, [FIRST_QUESTION]) : deepenContent();
+    const currentQuestion = kind === "transition" ? FIRST_QUESTION.text! : SECOND_QUESTION.text!;
     const trace = await runTutorQualityScenario({
       id: `TQ-ADV-repeat-${kind}`,
       code: sketch,
       action: { kind: "dialog", question: currentQuestion, answer: "value hat den Wert 3 und wird ausgegeben." },
-      provider: {
-        kind: "result",
-        result: {
-          answerRating: 5,
-          question: candidate,
-        },
-      },
-      planning: planningWithFollowup(kind),
+      provider: { kind: "result", result: { answerRating: 5, question: candidate } },
+      courseContent: content,
     });
 
     expect(trace.error).toBeUndefined();
+    expect(trace.result?.question).not.toBe(candidate);
+    expect(trace.result?.question).not.toBe(currentQuestion);
+    expect(trace.result).not.toHaveProperty("questionId");
     expect(trace.result).toMatchObject({
-      learningPhase: "DEEPEN",
       contentRevision: revision,
       activeTopicId: "variables-and-serial",
+      masteredTopicIds: ["variables-and-serial"],
+      ...(kind === "transition"
+        ? { learningPhase: "LEARN" }
+        : { learningPhase: "DEEPEN", progressionBlockedReason: "content-exhausted" }),
     });
-    expect(trace.result?.question).not.toBe(candidate);
+    expect(trace.stateAfter).toMatchObject({ activeTopicId: "variables-and-serial", phase: "DEEPEN" });
   });
 
   it("lets an application-owned TutorPlan replace provider question and metadata", async () => {
-    const plan = tutorPlan();
-    const planning: TutorPlanningExtension = {
-      planInitial: vi.fn().mockResolvedValue(plan),
-      planFollowup: vi.fn().mockResolvedValue(null),
-    };
     const trace = await runTutorQualityScenario({
       id: "TQ-ADV-plan-authority",
       code: sketch,
@@ -174,13 +146,14 @@ describe("Tutor Quality deterministic scenarios", () => {
           learningPhase: "EXPAND",
         },
       },
-      planning,
+      courseContent: courseContent(revision, [FIRST_QUESTION]),
     });
 
+    expect(trace.error).toBeUndefined();
     expect(trace.result).toMatchObject({
-      question: plan.question,
-      topicId: plan.topicId,
-      questionId: plan.questionId,
+      question: FIRST_QUESTION.text,
+      topicId: "variables-and-serial",
+      questionId: FIRST_QUESTION.id,
       learningPhase: "LEARN",
     });
   });