From 1f5ef6e42d751507c5a042be45e695cea2b6beff Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Thu, 2 Jul 2026 20:31:26 +0100 Subject: [PATCH 1/7] add bulk tests with parameterising --- data/longMIDIsequence.json | 1081 ++++++++++++++++++++++++ evaluation_function/evaluation_test.py | 87 +- 2 files changed, 1167 insertions(+), 1 deletion(-) create mode 100644 data/longMIDIsequence.json diff --git a/data/longMIDIsequence.json b/data/longMIDIsequence.json new file mode 100644 index 0000000..5d0ab87 --- /dev/null +++ b/data/longMIDIsequence.json @@ -0,0 +1,1081 @@ +{ + "test_cases": [ + { + "name": "twinkle_perfect", + "category": "long_melody", + "description": "Student plays all 42 notes of Twinkle Twinkle Little Star perfectly. Baseline: zero errors of any kind.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 0, "extra_notes": 0, "wrong_pitch_notes": 0} + }, + { + "name": "twinkle_scattered_errors", + "category": "long_melody", + "description": "Three errors spread across the full melody: one wrong pitch near the start (index 4, A4->B4), one missing note in the middle (index 21, G4), one extra ghost note near the end. Tests that local errors do not corrupt alignment of the surrounding notes.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 71, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 66, "start": 21.8, "duration": 0.2}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 1, "extra_notes": 1, "wrong_pitch_notes": 1} + }, + { + "name": "twinkle_repeated_note_correct", + "category": "repeated_notes", + "description": "Student plays the full melody correctly, including all natural repeated-note pairs (C C, G G, A A, F F, E E, D D). Baseline sanity check before the repeated-note error cases.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 0, "extra_notes": 0} + }, + { + "name": "twinkle_repeated_note_missing", + "category": "repeated_notes", + "description": "The opening phrase has G G (indices 2,3). Student plays only one G4 then continues. Tests that alignment uses timing to correctly identify the second G4 as missing, not the first (pitch alone cannot distinguish them).", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 69, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 67, "start": 2.5, "duration": 1.0}, + {"pitch": 65, "start": 3.5, "duration": 0.5}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 64, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 62, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 60, "start": 6.5, "duration": 1.0}, + {"pitch": 67, "start": 7.5, "duration": 0.5}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 65, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 64, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 62, "start": 10.5, "duration": 1.0}, + {"pitch": 67, "start": 11.5, "duration": 0.5}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 65, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 64, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 62, "start": 14.5, "duration": 1.0}, + {"pitch": 60, "start": 15.5, "duration": 0.5}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 67, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 69, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 67, "start": 18.5, "duration": 1.0}, + {"pitch": 65, "start": 19.5, "duration": 0.5}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 64, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 62, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 60, "start": 22.5, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 1, "extra_notes": 0} + }, + { + "name": "twinkle_repeated_note_extra", + "category": "repeated_notes", + "description": "Student double-strikes the opening C: plays C C C instead of C C. Tests that the extra repeated note is counted as one extra note without disrupting the rest of the alignment.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 60, "start": 1.0, "duration": 0.4}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 67, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 69, "start": 3.0, "duration": 0.5}, + {"pitch": 67, "start": 3.5, "duration": 1.0}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 65, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 64, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 62, "start": 7.0, "duration": 0.5}, + {"pitch": 60, "start": 7.5, "duration": 1.0}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 67, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 65, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 64, "start": 11.0, "duration": 0.5}, + {"pitch": 62, "start": 11.5, "duration": 1.0}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 67, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 65, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 64, "start": 15.0, "duration": 0.5}, + {"pitch": 62, "start": 15.5, "duration": 1.0}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 60, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 67, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 69, "start": 19.0, "duration": 0.5}, + {"pitch": 67, "start": 19.5, "duration": 1.0}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 65, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 64, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 62, "start": 23.0, "duration": 0.5}, + {"pitch": 60, "start": 23.5, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 0, "extra_notes": 1} + }, + { + "name": "twinkle_hesitation_mid_phrase", + "category": "hesitation", + "description": "Student plays phrases 1-2 correctly, then hesitates 1.2s before phrase 3 (note index 14, G4 at t=8.0), then resumes normal rhythm. Tests that global trend removal does not absorb this one-off local pause into 'global tempo drift', and that only the note at index 14 is flagged for timing.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 9.2, "duration": 0.5}, + {"pitch": 67, "start": 9.7, "duration": 0.5}, + {"pitch": 65, "start": 10.2, "duration": 0.5}, + {"pitch": 65, "start": 10.7, "duration": 0.5}, + {"pitch": 64, "start": 11.2, "duration": 0.5}, + {"pitch": 64, "start": 11.7, "duration": 0.5}, + {"pitch": 62, "start": 12.2, "duration": 1.0}, + {"pitch": 67, "start": 13.2, "duration": 0.5}, + {"pitch": 67, "start": 13.7, "duration": 0.5}, + {"pitch": 65, "start": 14.2, "duration": 0.5}, + {"pitch": 65, "start": 14.7, "duration": 0.5}, + {"pitch": 64, "start": 15.2, "duration": 0.5}, + {"pitch": 64, "start": 15.7, "duration": 0.5}, + {"pitch": 62, "start": 16.2, "duration": 1.0}, + {"pitch": 60, "start": 17.2, "duration": 0.5}, + {"pitch": 60, "start": 17.7, "duration": 0.5}, + {"pitch": 67, "start": 18.2, "duration": 0.5}, + {"pitch": 67, "start": 18.7, "duration": 0.5}, + {"pitch": 69, "start": 19.2, "duration": 0.5}, + {"pitch": 69, "start": 19.7, "duration": 0.5}, + {"pitch": 67, "start": 20.2, "duration": 1.0}, + {"pitch": 65, "start": 21.2, "duration": 0.5}, + {"pitch": 65, "start": 21.7, "duration": 0.5}, + {"pitch": 64, "start": 22.2, "duration": 0.5}, + {"pitch": 64, "start": 22.7, "duration": 0.5}, + {"pitch": 62, "start": 23.2, "duration": 0.5}, + {"pitch": 62, "start": 23.7, "duration": 0.5}, + {"pitch": 60, "start": 24.2, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 0, "extra_notes": 0, "wrong_pitch_notes": 0, "large_timing_deviation_note_index": 14} + }, + { + "name": "twinkle_hesitation_plus_missing", + "category": "hesitation", + "description": "Same 1.2s hesitation before phrase 3 as above, but student also skips the first G4 of phrase 3 (index 14). Tests that the system correctly reports one missing note AND a timing deviation, without confusing the two or reporting errors at wrong positions.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 9.7, "duration": 0.5}, + {"pitch": 65, "start": 10.2, "duration": 0.5}, + {"pitch": 65, "start": 10.7, "duration": 0.5}, + {"pitch": 64, "start": 11.2, "duration": 0.5}, + {"pitch": 64, "start": 11.7, "duration": 0.5}, + {"pitch": 62, "start": 12.2, "duration": 1.0}, + {"pitch": 67, "start": 13.2, "duration": 0.5}, + {"pitch": 67, "start": 13.7, "duration": 0.5}, + {"pitch": 65, "start": 14.2, "duration": 0.5}, + {"pitch": 65, "start": 14.7, "duration": 0.5}, + {"pitch": 64, "start": 15.2, "duration": 0.5}, + {"pitch": 64, "start": 15.7, "duration": 0.5}, + {"pitch": 62, "start": 16.2, "duration": 1.0}, + {"pitch": 60, "start": 17.2, "duration": 0.5}, + {"pitch": 60, "start": 17.7, "duration": 0.5}, + {"pitch": 67, "start": 18.2, "duration": 0.5}, + {"pitch": 67, "start": 18.7, "duration": 0.5}, + {"pitch": 69, "start": 19.2, "duration": 0.5}, + {"pitch": 69, "start": 19.7, "duration": 0.5}, + {"pitch": 67, "start": 20.2, "duration": 1.0}, + {"pitch": 65, "start": 21.2, "duration": 0.5}, + {"pitch": 65, "start": 21.7, "duration": 0.5}, + {"pitch": 64, "start": 22.2, "duration": 0.5}, + {"pitch": 64, "start": 22.7, "duration": 0.5}, + {"pitch": 62, "start": 23.2, "duration": 0.5}, + {"pitch": 62, "start": 23.7, "duration": 0.5}, + {"pitch": 60, "start": 24.2, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 1, "extra_notes": 0} + }, + { + "name": "twinkle_long_break_no_notes_lost", + "category": "long_break", + "description": "Student plays phrases 1-2 correctly, then pauses 5 seconds, then resumes from phrase 3 playing all remaining notes correctly. Key edge case for gap_penalty: the long break should NOT be misread as missing notes. All 42 notes should still align 1-to-1; only the note at index 14 should be flagged for a large timing deviation.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 13.0, "duration": 0.5}, + {"pitch": 67, "start": 13.5, "duration": 0.5}, + {"pitch": 65, "start": 14.0, "duration": 0.5}, + {"pitch": 65, "start": 14.5, "duration": 0.5}, + {"pitch": 64, "start": 15.0, "duration": 0.5}, + {"pitch": 64, "start": 15.5, "duration": 0.5}, + {"pitch": 62, "start": 16.0, "duration": 1.0}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 65, "start": 18.0, "duration": 0.5}, + {"pitch": 65, "start": 18.5, "duration": 0.5}, + {"pitch": 64, "start": 19.0, "duration": 0.5}, + {"pitch": 64, "start": 19.5, "duration": 0.5}, + {"pitch": 62, "start": 20.0, "duration": 1.0}, + {"pitch": 60, "start": 21.0, "duration": 0.5}, + {"pitch": 60, "start": 21.5, "duration": 0.5}, + {"pitch": 67, "start": 22.0, "duration": 0.5}, + {"pitch": 67, "start": 22.5, "duration": 0.5}, + {"pitch": 69, "start": 23.0, "duration": 0.5}, + {"pitch": 69, "start": 23.5, "duration": 0.5}, + {"pitch": 67, "start": 24.0, "duration": 1.0}, + {"pitch": 65, "start": 25.0, "duration": 0.5}, + {"pitch": 65, "start": 25.5, "duration": 0.5}, + {"pitch": 64, "start": 26.0, "duration": 0.5}, + {"pitch": 64, "start": 26.5, "duration": 0.5}, + {"pitch": 62, "start": 27.0, "duration": 0.5}, + {"pitch": 62, "start": 27.5, "duration": 0.5}, + {"pitch": 60, "start": 28.0, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 0, "extra_notes": 0, "large_timing_deviation_note_index": 14} + }, + { + "name": "twinkle_long_break_with_skip", + "category": "long_break", + "description": "Student pauses 5 seconds after phrase 2 AND skips the first G4 of phrase 3 (index 14) when resuming. Tests that the system correctly reports one missing note despite the long break, without misattributing the timing gap as the source of the error.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 8.0, "duration": 0.5}, + {"pitch": 67, "start": 8.5, "duration": 0.5}, + {"pitch": 65, "start": 9.0, "duration": 0.5}, + {"pitch": 65, "start": 9.5, "duration": 0.5}, + {"pitch": 64, "start": 10.0, "duration": 0.5}, + {"pitch": 64, "start": 10.5, "duration": 0.5}, + {"pitch": 62, "start": 11.0, "duration": 1.0}, + {"pitch": 67, "start": 12.0, "duration": 0.5}, + {"pitch": 67, "start": 12.5, "duration": 0.5}, + {"pitch": 65, "start": 13.0, "duration": 0.5}, + {"pitch": 65, "start": 13.5, "duration": 0.5}, + {"pitch": 64, "start": 14.0, "duration": 0.5}, + {"pitch": 64, "start": 14.5, "duration": 0.5}, + {"pitch": 62, "start": 15.0, "duration": 1.0}, + {"pitch": 60, "start": 16.0, "duration": 0.5}, + {"pitch": 60, "start": 16.5, "duration": 0.5}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 69, "start": 18.0, "duration": 0.5}, + {"pitch": 69, "start": 18.5, "duration": 0.5}, + {"pitch": 67, "start": 19.0, "duration": 1.0}, + {"pitch": 65, "start": 20.0, "duration": 0.5}, + {"pitch": 65, "start": 20.5, "duration": 0.5}, + {"pitch": 64, "start": 21.0, "duration": 0.5}, + {"pitch": 64, "start": 21.5, "duration": 0.5}, + {"pitch": 62, "start": 22.0, "duration": 0.5}, + {"pitch": 62, "start": 22.5, "duration": 0.5}, + {"pitch": 60, "start": 23.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 60, "start": 0.5, "duration": 0.5}, + {"pitch": 67, "start": 1.0, "duration": 0.5}, + {"pitch": 67, "start": 1.5, "duration": 0.5}, + {"pitch": 69, "start": 2.0, "duration": 0.5}, + {"pitch": 69, "start": 2.5, "duration": 0.5}, + {"pitch": 67, "start": 3.0, "duration": 1.0}, + {"pitch": 65, "start": 4.0, "duration": 0.5}, + {"pitch": 65, "start": 4.5, "duration": 0.5}, + {"pitch": 64, "start": 5.0, "duration": 0.5}, + {"pitch": 64, "start": 5.5, "duration": 0.5}, + {"pitch": 62, "start": 6.0, "duration": 0.5}, + {"pitch": 62, "start": 6.5, "duration": 0.5}, + {"pitch": 60, "start": 7.0, "duration": 1.0}, + {"pitch": 67, "start": 13.5, "duration": 0.5}, + {"pitch": 65, "start": 14.0, "duration": 0.5}, + {"pitch": 65, "start": 14.5, "duration": 0.5}, + {"pitch": 64, "start": 15.0, "duration": 0.5}, + {"pitch": 64, "start": 15.5, "duration": 0.5}, + {"pitch": 62, "start": 16.0, "duration": 1.0}, + {"pitch": 67, "start": 17.0, "duration": 0.5}, + {"pitch": 67, "start": 17.5, "duration": 0.5}, + {"pitch": 65, "start": 18.0, "duration": 0.5}, + {"pitch": 65, "start": 18.5, "duration": 0.5}, + {"pitch": 64, "start": 19.0, "duration": 0.5}, + {"pitch": 64, "start": 19.5, "duration": 0.5}, + {"pitch": 62, "start": 20.0, "duration": 1.0}, + {"pitch": 60, "start": 21.0, "duration": 0.5}, + {"pitch": 60, "start": 21.5, "duration": 0.5}, + {"pitch": 67, "start": 22.0, "duration": 0.5}, + {"pitch": 67, "start": 22.5, "duration": 0.5}, + {"pitch": 69, "start": 23.0, "duration": 0.5}, + {"pitch": 69, "start": 23.5, "duration": 0.5}, + {"pitch": 67, "start": 24.0, "duration": 1.0}, + {"pitch": 65, "start": 25.0, "duration": 0.5}, + {"pitch": 65, "start": 25.5, "duration": 0.5}, + {"pitch": 64, "start": 26.0, "duration": 0.5}, + {"pitch": 64, "start": 26.5, "duration": 0.5}, + {"pitch": 62, "start": 27.0, "duration": 0.5}, + {"pitch": 62, "start": 27.5, "duration": 0.5}, + {"pitch": 60, "start": 28.0, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 1, "extra_notes": 0} + }, + { + "name": "chord_all_correct", + "category": "broken_chord", + "description": "4 melody notes then 3 block chords (C major, F major, G major), all played perfectly. Baseline for chord comparison.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "expected": {"missing_notes": 0, "extra_notes": 0, "chord_correct": true} + }, + { + "name": "chord_one_wrong_note", + "category": "broken_chord", + "description": "Student plays the C major chord with Ab4 instead of G4 (2 of 3 notes correct). Should be counted as one imperfect chord, not a fully correct or fully wrong chord.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 68, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "expected": {"chord_correct": false, "chord_partial_match": true} + }, + { + "name": "chord_entirely_missing", + "category": "broken_chord", + "description": "Student skips the F major chord entirely. Should be counted as one missing chord without affecting the alignment of surrounding notes.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "expected": {"chord_missing": 1} + }, + { + "name": "chord_arpeggiated_within_window", + "category": "broken_chord", + "description": "C major chord played with onsets 0/20/40ms apart (within the 50ms onset window). Should still be grouped as one chord event and counted as fully correct.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.02, "duration": 1.0}, + {"pitch": 67, "start": 2.04, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "expected": {"chord_correct": true} + }, + { + "name": "chord_arpeggiated_beyond_window", + "category": "broken_chord", + "description": "C major chord played with onsets 0/100/200ms apart (beyond the 50ms onset window). The three notes will NOT be grouped as one chord event. Tests the boundary of the DEFAULT_CHORD_ONSET_WINDOW parameter.", + "reference": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.0, "duration": 1.0}, + {"pitch": 67, "start": 2.0, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "response": { + "notes": [ + {"pitch": 60, "start": 0.0, "duration": 0.5}, + {"pitch": 62, "start": 0.5, "duration": 0.5}, + {"pitch": 64, "start": 1.0, "duration": 0.5}, + {"pitch": 65, "start": 1.5, "duration": 0.5}, + {"pitch": 60, "start": 2.0, "duration": 1.0}, + {"pitch": 64, "start": 2.1, "duration": 1.0}, + {"pitch": 67, "start": 2.2, "duration": 1.0}, + {"pitch": 65, "start": 3.0, "duration": 1.0}, + {"pitch": 69, "start": 3.0, "duration": 1.0}, + {"pitch": 72, "start": 3.0, "duration": 1.0}, + {"pitch": 67, "start": 4.0, "duration": 1.0}, + {"pitch": 71, "start": 4.0, "duration": 1.0}, + {"pitch": 74, "start": 4.0, "duration": 1.0} + ] + }, + "expected": {"chord_correct": false} + } + ] +} \ No newline at end of file diff --git a/evaluation_function/evaluation_test.py b/evaluation_function/evaluation_test.py index 7094391..0a2e10a 100755 --- a/evaluation_function/evaluation_test.py +++ b/evaluation_function/evaluation_test.py @@ -551,4 +551,89 @@ def test_custom_chord_onset_window_affects_grouping(self): # With 50ms window: two separate notes -> chord_count=0 result_narrow = compare_performance_ED(res, ref, chord_onset_window=0.05) assert result_wide.stats["total_chords_in_reference"] == 1 - assert result_narrow.stats["total_notes_in_reference"] == 2 \ No newline at end of file + assert result_narrow.stats["total_notes_in_reference"] == 2 + + +# 10. Bulk tests using longer MIDI sequences, which are more realistic +# ------------------------------------------------------------------------------ +# Test cases live in longMIDIsequence.json, not hardcoded here. +# To add a new scenario: add a new dict to the "test_cases" list in that +# JSON file (with "name", "category", "reference", "response", "expected"). +# +# Scenarios currently covered: +# perfect performance; scattered errors across full melody +# correct; one of a pair missing; extra double-struck note +# single mid-phrase pause; pause combined with missing note +# long break with no notes lost; long break with a skip + +# Load realistic scenario test cases from the JSON file. +# Path is relative to this test file, not the current working directory, +# so the tests work no matter where pytest is run from. +import json +import os +import pytest + +cwd = os.getcwd() +dir = os.path.dirname(cwd) +path = os.path.join(dir, "data", "longMIDIsequence.json") + +with open(path, "r") as json_file: + REALISTIC_TEST_DATA = json.load(json_file) + +REALISTIC_TEST_CASES = REALISTIC_TEST_DATA["test_cases"] +REALISTIC_TEST_IDS = [case["name"] for case in REALISTIC_TEST_CASES] + +@pytest.mark.parametrize("case", REALISTIC_TEST_CASES, ids=REALISTIC_TEST_IDS) +def test_realistic_scenario(case): + """ + Run one realistic scenario through the full pipeline and check the + values in result.stats (and event_details, for timing checks) against + the "expected" section of the JSON test case. Only fields present in + "expected" are checked, since different scenarios check different things. + """ + reference_midi = case["reference"] + response_midi = case["response"] + expected = case["expected"] + + result = compare_performance_ED(response_midi, reference_midi) + stats = result.stats + + # ---- Note-level checks ---- + if "missing_notes" in expected: + assert stats["total_notes_missing"] == expected["missing_notes"] + + if "extra_notes" in expected: + assert stats["total_notes_extra"] == expected["extra_notes"] + + if "wrong_pitch_notes" in expected: + assert stats["total_notes_wrong_pitch"] == expected["wrong_pitch_notes"] + + # ---- Chord-level checks ---- + if "chord_correct" in expected: + if expected["chord_correct"]: + assert stats["total_chords_in_reference"] > 0 + assert stats["total_chords_correct"] == stats["total_chords_in_reference"] + else: + assert stats["total_chords_correct"] < stats["total_chords_in_reference"] + + if expected.get("chord_partial_match"): + assert stats["total_chords_imperfect"] >= 1 + + if "chord_missing" in expected: + assert stats["total_chords_missing"] == expected["chord_missing"] + + # ---- Timing deviation checks ---- + # "large_timing_deviation_note_index" points at the reference_index of the + # note that should be flagged with a large local timing error (e.g. the + # note right after a hesitation or a long break). + if "large_timing_deviation_note_index" in expected: + flagged_index = expected["large_timing_deviation_note_index"] + matching_notes = [ + note for note in result.event_details + if note["event_type"] == "note" + and note.get("reference_index") == flagged_index + ] + assert len(matching_notes) == 1 + flagged_note = matching_notes[0] + assert flagged_note["timing_correct"] is False + \ No newline at end of file From 5a9fc5f8e88afb2b9d9095b06ae0a9b8d6bae5dc Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Thu, 2 Jul 2026 22:56:26 +0100 Subject: [PATCH 2/7] add new dependencies --- poetry.lock | 187 +++++++++++++++++++++++++++++++++++++++++++++++-- pyproject.toml | 2 + 2 files changed, 182 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8852c54..db22435 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -939,6 +939,26 @@ perf = ["ipython"] test = ["packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] +[[package]] +name = "importlib-resources" +version = "7.1.0" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1"}, + {file = "importlib_resources-7.1.0.tar.gz", hash = "sha256:0722d4c6212489c530f2a145a34c0a7a3b4721bc96a15fada5930e2a0b760708"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] + [[package]] name = "iniconfig" version = "2.0.0" @@ -1089,7 +1109,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.3.6" +jsonschema-specifications = ">=2023.03.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -1302,7 +1322,7 @@ parsing = ["antlr4-python3-runtime (==4.13.2)", "lark (==1.2.2)", "latex2sympy @ type = "git" url = "https://github.com/lambda-feedback/toolkit-python.git" reference = "main" -resolved_reference = "713f13fec11a1d81668fefc3199942f604ba1505" +resolved_reference = "b6c3eac917e0bfa81085638cf8173d5e6b8ff12d" [[package]] name = "matplotlib" @@ -1383,6 +1403,34 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mido" +version = "1.3.3" +description = "MIDI Objects for Python" +optional = false +python-versions = "~=3.7" +groups = ["main"] +files = [ + {file = "mido-1.3.3-py3-none-any.whl", hash = "sha256:01033c9b10b049e4436fca2762194ca839b09a4334091dd3c34e7f4ae674fd8a"}, + {file = "mido-1.3.3.tar.gz", hash = "sha256:1aecb30b7f282404f17e43768cbf74a6a31bf22b3b783bdd117a1ce9d22cb74c"}, +] + +[package.dependencies] +packaging = "*" + +[package.extras] +build-docs = ["sphinx (>=4.3.2,<4.4.0)", "sphinx-rtd-theme (>=1.2.2,<1.3.0)"] +check-manifest = ["check-manifest (>=0.49)"] +dev = ["mido[build-docs]", "mido[check-manifest]", "mido[lint-code]", "mido[lint-reuse]", "mido[release]", "mido[test-code]"] +lint-code = ["ruff (>=0.1.6,<0.2.0)"] +lint-reuse = ["reuse (>=1.1.2,<1.2.0)"] +ports-all = ["mido[ports-pygame]", "mido[ports-rtmidi-python]", "mido[ports-rtmidi]"] +ports-pygame = ["PyGame (>=2.5,<3.0)"] +ports-rtmidi = ["python-rtmidi (>=1.5.4,<1.6.0)"] +ports-rtmidi-python = ["rtmidi-python (>=0.2.2,<0.3.0)"] +release = ["twine (>=4.0.2,<4.1.0)"] +test-code = ["pytest (>=7.4.0,<7.5.0)"] + [[package]] name = "more-itertools" version = "11.1.0" @@ -1598,6 +1646,98 @@ files = [ {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "pandas" +version = "3.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, + {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, + {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, + {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, + {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, + {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, + {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, + {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, + {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, + {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, + {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, + {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, + {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, + {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, + {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, + {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, + {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, + {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, + {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, + {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, + {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, + {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, + {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, + {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, + {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, + {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, + {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, + {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, + {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, + {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, + {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, + {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, + {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, + {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, + {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, + {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, + {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, + {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, + {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, + {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, + {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, + {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, + {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, + {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, + {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, + {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, + {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, + {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.26.0", markers = "python_version < \"3.14\""}, + {version = ">=2.3.3", markers = "python_version >= \"3.14\""}, +] +python-dateutil = ">=2.8.2" +tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] +aws = ["s3fs (>=2024.10.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] +compression = ["zstandard (>=0.23.0)"] +computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] +feather = ["pyarrow (>=13.0.0)"] +fss = ["fsspec (>=2024.10.0)"] +gcp = ["gcsfs (>=2024.10.0)"] +hdf5 = ["tables (>=3.10.1)"] +html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] +iceberg = ["pyiceberg (>=0.8.1)"] +mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] +output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=13.0.0)"] +performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] +plot = ["matplotlib (>=3.9.3)"] +postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] +pyarrow = ["pyarrow (>=13.0.0)"] +spss = ["pyreadstat (>=1.2.8)"] +sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +timezone = ["pytz (>=2020.1)"] +xml = ["lxml (>=5.3.0)"] + [[package]] name = "pbs-installer" version = "2025.12.17" @@ -1835,6 +1975,26 @@ poetry = ">=2.1.0,<3.0.0" poetry-core = ">=2.1.0,<3.0.0" tomlkit = ">=0.11.4,<1.0.0" +[[package]] +name = "pretty-midi" +version = "0.2.11" +description = "Functions and classes for handling MIDI data conveniently." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pretty_midi-0.2.11.tar.gz", hash = "sha256:dda81d643f71a0e91843a8644b312c0380237f1e63d5c6188225785c8d58cc67"}, +] + +[package.dependencies] +importlib_resources = "*" +mido = ">=1.1.16" +numpy = ">=1.7.0" +six = "*" + +[package.extras] +fluidsynth = ["pyfluidsynth (>=1.3.1)"] + [[package]] name = "pycodestyle" version = "2.12.1" @@ -2417,10 +2577,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a0" +botocore = ">=1.37.4,<2.0a.0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] [[package]] name = "secretstorage" @@ -2541,6 +2701,19 @@ files = [ [package.dependencies] typing-extensions = ">=4.12.0" +[[package]] +name = "tzdata" +version = "2026.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\"" +files = [ + {file = "tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7"}, + {file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"}, +] + [[package]] name = "ujson" version = "5.10.0" @@ -2866,9 +3039,9 @@ files = [ ] [package.extras] -cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and python_version < \"3.14\"", "cffi (>=2.0.0b0) ; platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""] +cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and python_version < \"3.14\"", "cffi (>=2.0.0b) ; platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "494427295e39d085dd9aaa6680cb28ad115685b73a2cbf41116fca463abc54ee" +content-hash = "e2572e64cfb65e24d91c1cf604f2e59440d86cac35507649a2e522dba0a5a318" diff --git a/pyproject.toml b/pyproject.toml index 2928e06..9a6a00e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ lf_toolkit = { git = "https://github.com/lambda-feedback/toolkit-python.git", br ] } numpy = "^2.4.6" matplotlib = "^3.11.0" +pretty-midi = "^0.2.11" +pandas = "^3.0.3" [tool.poetry.group.dev.dependencies] pytest = "^8.2.2" From f8b705ce30f78a45530befce882ce592efc58adf Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Thu, 2 Jul 2026 22:56:52 +0100 Subject: [PATCH 3/7] apply ED on the ASAP dataset --- notebooks/Alignment_on_ASAPdataset.ipynb | 740 +++++++++++++++++++++++ 1 file changed, 740 insertions(+) create mode 100644 notebooks/Alignment_on_ASAPdataset.ipynb diff --git a/notebooks/Alignment_on_ASAPdataset.ipynb b/notebooks/Alignment_on_ASAPdataset.ipynb new file mode 100644 index 0000000..ac38559 --- /dev/null +++ b/notebooks/Alignment_on_ASAPdataset.ipynb @@ -0,0 +1,740 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ASAP dataset contains paired MIDI score files (what *should* be played) and MIDI performance files (what a professional pianist *actually* played). We use the score as the reference and the performance as the student response, and run the ED alignment algorithm on each pair.\n", + "\n", + "citation:\n", + "\n", + "@inproceedings{asap-dataset,\n", + " title={{ASAP}: a dataset of aligned scores and performances for piano transcription},\n", + " author={Foscarin, Francesco and McLeod, Andrew and Rigaux, Philippe and Jacquemard, Florent and Sakai, Masahiko},\n", + " booktitle={International Society for Music Information Retrieval Conference {(ISMIR)}},\n", + " year={2020},\n", + " pages={534--541}\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Download the ASAP Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Cloning into 'asap-dataset'...\n", + "Updating files: 100% (2854/2854), done.\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "if not os.path.exists('asap-dataset'):\n", + " os.system('git clone https://github.com/fosfrancesco/asap-dataset.git')\n", + "else:\n", + " print('asap-dataset already exists, skipping download.')\n", + "\n", + "ASAP_PATH = 'asap-dataset'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Format conversion\n", + "\n", + "These functions convert a MIDI file into the `{pitch, start, duration}` format used by `compare_MIDI.py`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import pretty_midi\n", + "\n", + "def midi_file_to_notes(midi_path):\n", + " \"\"\"\n", + " Parse a MIDI file and return all its notes in compareMusic format.\n", + " Notes are sorted by onset time.\n", + " \"\"\"\n", + " midi_data = pretty_midi.PrettyMIDI(midi_path)\n", + " all_notes = []\n", + " for instrument in midi_data.instruments:\n", + " if instrument.is_drum: # Ignore drum tracks\n", + " continue\n", + " for note in instrument.notes:\n", + " all_notes.append({\n", + " \"pitch\": note.pitch,\n", + " \"start\": round(note.start, 3),\n", + " \"duration\": round(note.end - note.start, 3),\n", + " })\n", + " all_notes.sort(key=lambda n: (n[\"start\"], n[\"pitch\"]))\n", + " return all_notes\n", + "\n", + "\n", + "def build_sample(ref_path, response_path, composer, title):\n", + " \"\"\"\n", + " Convert one score/performance MIDI pair into a sample dict\n", + " ready for compare_performance_ED.\n", + " Returns None if either MIDI file produces zero notes.\n", + " \"\"\"\n", + " score_notes = midi_file_to_notes(ref_path)\n", + " perf_notes = midi_file_to_notes(response_path)\n", + "\n", + " if not score_notes or not perf_notes:\n", + " return None\n", + "\n", + " return {\n", + " \"composer\": composer,\n", + " \"title\": title,\n", + " \"reference\": {\"notes\": score_notes},\n", + " \"response\": {\"notes\": perf_notes},\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load Score/Performance Pairs from ASAP Metadata\n", + "\n", + "For the ASAP dataset, the `metadata.csv` lists every score/performance pair in the dataset, check that both MIDI files exist on disk." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 169 score/performance pairs.\n", + "\n", + "1. Bach / Fugue_bwv_846 | score notes: 755 | performance notes: 754\n", + "2. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1435\n", + "3. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1438\n", + "4. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1429\n", + "5. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1431\n", + "6. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1433\n", + "7. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1438\n", + "8. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1440\n", + "9. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1431\n", + "10. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1430\n", + "11. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 739\n", + "12. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 738\n", + "13. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 734\n", + "14. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 738\n", + "15. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 736\n", + "16. Bach / Fugue_bwv_856 | score notes: 798 | performance notes: 740\n", + "17. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1400\n", + "18. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1353\n", + "19. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1345\n", + "20. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1361\n", + "21. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1340\n", + "22. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1358\n", + "23. Bach / Fugue_bwv_858 | score notes: 930 | performance notes: 875\n", + "24. Bach / Fugue_bwv_858 | score notes: 930 | performance notes: 888\n", + "25. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1739\n", + "26. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1750\n", + "27. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1766\n", + "28. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1740\n", + "29. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1762\n", + "30. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1744\n", + "31. Bach / Fugue_bwv_862 | score notes: 886 | performance notes: 868\n", + "32. Bach / Fugue_bwv_863 | score notes: 797 | performance notes: 792\n", + "33. Bach / Fugue_bwv_863 | score notes: 797 | performance notes: 799\n", + "34. Bach / Fugue_bwv_863 | score notes: 797 | performance notes: 803\n", + "35. Bach / Fugue_bwv_864 | score notes: 1207 | performance notes: 1188\n", + "36. Bach / Fugue_bwv_865 | score notes: 2428 | performance notes: 2398\n", + "37. Bach / Fugue_bwv_865 | score notes: 2428 | performance notes: 2456\n", + "38. Bach / Fugue_bwv_865 | score notes: 2428 | performance notes: 2402\n", + "39. Bach / Fugue_bwv_866 | score notes: 946 | performance notes: 946\n", + "40. Bach / Fugue_bwv_867 | score notes: 760 | performance notes: 747\n", + "41. Bach / Fugue_bwv_867 | score notes: 760 | performance notes: 739\n", + "42. Bach / Fugue_bwv_868 | score notes: 868 | performance notes: 872\n", + "43. Bach / Fugue_bwv_870 | score notes: 1083 | performance notes: 1070\n", + "44. Bach / Fugue_bwv_870 | score notes: 1083 | performance notes: 1049\n", + "45. Bach / Fugue_bwv_870 | score notes: 1083 | performance notes: 1092\n", + "46. Bach / Fugue_bwv_873 | score notes: 1386 | performance notes: 1385\n", + "47. Bach / Fugue_bwv_873 | score notes: 1386 | performance notes: 1380\n", + "48. Bach / Fugue_bwv_874 | score notes: 865 | performance notes: 867\n", + "49. Bach / Fugue_bwv_874 | score notes: 865 | performance notes: 872\n", + "50. Bach / Fugue_bwv_874 | score notes: 865 | performance notes: 867\n", + "51. Bach / Fugue_bwv_875 | score notes: 794 | performance notes: 806\n", + "52. Bach / Fugue_bwv_875 | score notes: 794 | performance notes: 799\n", + "53. Bach / Fugue_bwv_876 | score notes: 701 | performance notes: 691\n", + "54. Bach / Fugue_bwv_876 | score notes: 701 | performance notes: 704\n", + "55. Bach / Fugue_bwv_880 | score notes: 973 | performance notes: 984\n", + "56. Bach / Fugue_bwv_880 | score notes: 973 | performance notes: 982\n", + "57. Bach / Fugue_bwv_880 | score notes: 973 | performance notes: 975\n", + "58. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1452\n", + "59. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1450\n", + "60. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1463\n", + "61. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1443\n", + "62. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1428\n", + "63. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1544\n", + "64. Bach / Fugue_bwv_884 | score notes: 748 | performance notes: 745\n", + "65. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1642\n", + "66. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1676\n", + "67. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1659\n", + "68. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1662\n", + "69. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1662\n", + "70. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1671\n", + "71. Bach / Fugue_bwv_887 | score notes: 1582 | performance notes: 1588\n", + "72. Bach / Fugue_bwv_888 | score notes: 755 | performance notes: 768\n", + "73. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 838\n", + "74. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 848\n", + "75. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 827\n", + "76. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 840\n", + "77. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 849\n", + "78. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 830\n", + "79. Bach / Fugue_bwv_891 | score notes: 1750 | performance notes: 1760\n", + "80. Bach / Fugue_bwv_892 | score notes: 1311 | performance notes: 1305\n", + "81. Bach / Fugue_bwv_892 | score notes: 1311 | performance notes: 1310\n", + "82. Bach / Fugue_bwv_893 | score notes: 1025 | performance notes: 992\n", + "83. Bach / Fugue_bwv_893 | score notes: 1025 | performance notes: 1020\n", + "84. Bach / Fugue_bwv_893 | score notes: 1025 | performance notes: 1031\n", + "85. Bach / Italian_concerto | score notes: 1096 | performance notes: 1221\n", + "86. Bach / Italian_concerto | score notes: 1096 | performance notes: 1196\n", + "87. Bach / Italian_concerto | score notes: 1096 | performance notes: 1171\n", + "88. Bach / Prelude_bwv_846 | score notes: 549 | performance notes: 548\n", + "89. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 811\n", + "90. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 816\n", + "91. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 810\n", + "92. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 810\n", + "93. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 810\n", + "94. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 811\n", + "95. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 806\n", + "96. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 811\n", + "97. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 808\n", + "98. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 459\n", + "99. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 444\n", + "100. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 443\n", + "101. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 444\n", + "102. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 465\n", + "103. Bach / Prelude_bwv_856 | score notes: 1045 | performance notes: 824\n", + "104. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 549\n", + "105. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 566\n", + "106. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 556\n", + "107. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 557\n", + "108. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 526\n", + "109. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 559\n", + "110. Bach / Prelude_bwv_858 | score notes: 497 | performance notes: 439\n", + "111. Bach / Prelude_bwv_858 | score notes: 497 | performance notes: 486\n", + "112. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 616\n", + "113. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 608\n", + "114. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 612\n", + "115. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 613\n", + "116. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 611\n", + "117. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 608\n", + "118. Bach / Prelude_bwv_862 | score notes: 696 | performance notes: 675\n", + "119. Bach / Prelude_bwv_863 | score notes: 571 | performance notes: 562\n", + "120. Bach / Prelude_bwv_863 | score notes: 571 | performance notes: 564\n", + "121. Bach / Prelude_bwv_863 | score notes: 571 | performance notes: 564\n", + "122. Bach / Prelude_bwv_864 | score notes: 618 | performance notes: 611\n", + "123. Bach / Prelude_bwv_865 | score notes: 608 | performance notes: 612\n", + "124. Bach / Prelude_bwv_865 | score notes: 608 | performance notes: 613\n", + "125. Bach / Prelude_bwv_865 | score notes: 608 | performance notes: 610\n", + "126. Bach / Prelude_bwv_866 | score notes: 634 | performance notes: 637\n", + "127. Bach / Prelude_bwv_867 | score notes: 776 | performance notes: 769\n", + "128. Bach / Prelude_bwv_867 | score notes: 776 | performance notes: 777\n", + "129. Bach / Prelude_bwv_868 | score notes: 417 | performance notes: 414\n", + "130. Bach / Prelude_bwv_870 | score notes: 832 | performance notes: 834\n", + "131. Bach / Prelude_bwv_870 | score notes: 832 | performance notes: 834\n", + "132. Bach / Prelude_bwv_870 | score notes: 832 | performance notes: 833\n", + "133. Bach / Prelude_bwv_873 | score notes: 1602 | performance notes: 1358\n", + "134. Bach / Prelude_bwv_873 | score notes: 1602 | performance notes: 1358\n", + "135. Bach / Prelude_bwv_874 | score notes: 1569 | performance notes: 1574\n", + "136. Bach / Prelude_bwv_874 | score notes: 1569 | performance notes: 1551\n", + "137. Bach / Prelude_bwv_875 | score notes: 1081 | performance notes: 1103\n", + "138. Bach / Prelude_bwv_875 | score notes: 1081 | performance notes: 1099\n", + "139. Bach / Prelude_bwv_875 | score notes: 1081 | performance notes: 1104\n", + "140. Bach / Prelude_bwv_876 | score notes: 914 | performance notes: 920\n", + "141. Bach / Prelude_bwv_876 | score notes: 914 | performance notes: 924\n", + "142. Bach / Prelude_bwv_880 | score notes: 1239 | performance notes: 1240\n", + "143. Bach / Prelude_bwv_880 | score notes: 1239 | performance notes: 1237\n", + "144. Bach / Prelude_bwv_880 | score notes: 1239 | performance notes: 1243\n", + "145. Bach / Prelude_bwv_883 | score notes: 882 | performance notes: 892\n", + "146. Bach / Prelude_bwv_883 | score notes: 882 | performance notes: 888\n", + "147. Bach / Prelude_bwv_883 | score notes: 882 | performance notes: 888\n", + "148. Bach / Prelude_bwv_884 | score notes: 902 | performance notes: 908\n", + "149. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 510\n", + "150. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 524\n", + "151. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 517\n", + "152. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 516\n", + "153. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 523\n", + "154. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 525\n", + "155. Bach / Prelude_bwv_887 | score notes: 1306 | performance notes: 1305\n", + "156. Bach / Prelude_bwv_888 | score notes: 688 | performance notes: 681\n", + "157. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 785\n", + "158. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 782\n", + "159. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 780\n", + "160. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 786\n", + "161. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 789\n", + "162. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 795\n", + "163. Bach / Prelude_bwv_891 | score notes: 1182 | performance notes: 1180\n", + "164. Bach / Prelude_bwv_891 | score notes: 1182 | performance notes: 1185\n", + "165. Bach / Prelude_bwv_892 | score notes: 1089 | performance notes: 1084\n", + "166. Bach / Prelude_bwv_892 | score notes: 1089 | performance notes: 1086\n", + "167. Bach / Prelude_bwv_893 | score notes: 882 | performance notes: 876\n", + "168. Bach / Prelude_bwv_893 | score notes: 882 | performance notes: 885\n", + "169. Bach / Prelude_bwv_893 | score notes: 882 | performance notes: 895\n" + ] + } + ], + "source": [ + "import csv\n", + "\n", + "def load_samples(asap_path, composer):\n", + " \"\"\"\n", + " Read the ASAP metadata CSV and return a list of sample dicts\n", + " for a specific composer only.\n", + " \"\"\"\n", + " metadata_path = os.path.join(asap_path, \"metadata.csv\")\n", + " samples = []\n", + "\n", + " with open(metadata_path, \"r\", encoding=\"utf-8\") as csv_file:\n", + " reader = csv.DictReader(csv_file)\n", + " for row in reader:\n", + " if row.get(\"composer\", \"\").strip() != composer:\n", + " continue\n", + "\n", + " ref_path = os.path.join(asap_path, row.get(\"midi_score\", \"\").strip())\n", + " response_path = os.path.join(asap_path, row.get(\"midi_performance\", \"\").strip())\n", + "\n", + " if os.path.isfile(ref_path) and os.path.isfile(response_path):\n", + " sample = build_sample(\n", + " ref_path, response_path,\n", + " row.get(\"composer\", \"Unknown\"),\n", + " row.get(\"title\", \"Unknown\"),\n", + " )\n", + " if sample is not None:\n", + " samples.append(sample)\n", + "\n", + " return samples\n", + "\n", + "samples = load_samples(ASAP_PATH, \"Bach\")\n", + "\n", + "print(\"Loaded\", len(samples), \"score/performance pairs.\")\n", + "print()\n", + "for i, sample in enumerate(samples):\n", + " print(\n", + " str(i + 1) + \".\",\n", + " sample[\"composer\"], \"/\", sample[\"title\"],\n", + " \" | score notes:\", len(sample[\"reference\"][\"notes\"]),\n", + " \" | performance notes:\", len(sample[\"response\"][\"notes\"])\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run Alignment on Each Pair\n", + "\n", + "Passing each score/performance pair through `compare_performance_ED`, where score MIDI is the **reference** and the performance MIDI is the **response**." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Alignment complete for 169 pieces.\n" + ] + } + ], + "source": [ + "from evaluation_function.compare_MIDI import compare_performance_ED\n", + "\n", + "def run_alignment_on_samples(samples):\n", + " \"\"\"\n", + " Run compare_performance_ED on every sample and return the results.\n", + "\n", + " Args:\n", + " samples: list of sample dicts from load_samples().\n", + "\n", + " Returns:\n", + " List of result dicts, each with keys:\n", + " composer, title, stats, event_details, is_correct\n", + " \"\"\"\n", + " results = []\n", + " for sample in samples:\n", + " result = compare_performance_ED(\n", + " sample[\"response\"],\n", + " sample[\"reference\"]\n", + " )\n", + " results.append({\n", + " \"composer\": sample[\"composer\"],\n", + " \"title\": sample[\"title\"],\n", + " \"stats\": result.stats,\n", + " \"event_details\": result.event_details,\n", + " \"is_correct\": result.is_correct,\n", + " })\n", + " return results\n", + "\n", + "all_results = run_alignment_on_samples(samples)\n", + "print(\"Alignment complete for\", len(all_results), \"pieces.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "summary" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PieceScoreMissingExtraWrongMatch %
0Bach / Fugue_bwv_846187012398.4
1Bach / Fugue_bwv_84831344796.5
2Bach / Fugue_bwv_848313412896.2
3Bach / Fugue_bwv_8483132111494.9
4Bach / Fugue_bwv_848313412995.8
.....................
164Bach / Prelude_bwv_892440167794.8
165Bach / Prelude_bwv_892440951893.9
166Bach / Prelude_bwv_89342875297.9
167Bach / Prelude_bwv_89342832398.6
168Bach / Prelude_bwv_8934281251396.7
\n", + "

169 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " Piece Score Missing Extra Wrong Match %\n", + "0 Bach / Fugue_bwv_846 187 0 12 3 98.4\n", + "1 Bach / Fugue_bwv_848 313 4 4 7 96.5\n", + "2 Bach / Fugue_bwv_848 313 4 12 8 96.2\n", + "3 Bach / Fugue_bwv_848 313 2 11 14 94.9\n", + "4 Bach / Fugue_bwv_848 313 4 12 9 95.8\n", + ".. ... ... ... ... ... ...\n", + "164 Bach / Prelude_bwv_892 440 16 7 7 94.8\n", + "165 Bach / Prelude_bwv_892 440 9 5 18 93.9\n", + "166 Bach / Prelude_bwv_893 428 7 5 2 97.9\n", + "167 Bach / Prelude_bwv_893 428 3 2 3 98.6\n", + "168 Bach / Prelude_bwv_893 428 1 25 13 96.7\n", + "\n", + "[169 rows x 6 columns]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "def make_results_table(all_results):\n", + " \"\"\"Build a summary DataFrame from alignment results.\"\"\"\n", + " rows = []\n", + " for r in all_results:\n", + " stats = r[\"stats\"]\n", + " total = stats[\"total_notes_in_reference\"]\n", + " missing = stats[\"total_notes_missing\"]\n", + " wrong = stats[\"total_notes_wrong_pitch\"]\n", + " matched = total - missing - wrong\n", + " rows.append({\n", + " \"Piece\": r[\"composer\"] + \" / \" + r[\"title\"],\n", + " \"Score\": total,\n", + " \"Missing\": missing,\n", + " \"Extra\": stats[\"total_notes_extra\"],\n", + " \"Wrong\": wrong,\n", + " \"Match %\": round(100.0 * matched / total, 1) if total > 0 else 0.0,\n", + " })\n", + " return pd.DataFrame(rows)\n", + "\n", + "make_results_table(all_results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Timing Deviation Distribution" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No Timing deviations found.\n" + ] + } + ], + "source": [ + "def collect_timing_deviations(all_results):\n", + " \"\"\"\n", + " Collect local timing deviations from all matched notes across all pieces.\n", + "\n", + " Args:\n", + " all_results: list of result dicts from run_alignment_on_samples().\n", + "\n", + " Returns:\n", + " List of floats (one per matched note), in seconds.\n", + " \"\"\"\n", + " deviations = []\n", + " for result in all_results:\n", + " for event in result[\"event_details\"]:\n", + " if event[\"event_type\"] != \"note\":\n", + " continue\n", + " if event[\"operation_type\"] != \"match\":\n", + " continue\n", + " if event.get(\"timing_deviation\") is not None:\n", + " deviations.append(event[\"timing_deviation\"])\n", + " return deviations\n", + "\n", + "\n", + "def print_deviation_summary(deviations, label):\n", + " \"\"\"\n", + " Print a text summary of any list of deviation values.\n", + "\n", + " Args:\n", + " deviations: list of floats.\n", + " label: str, printed in the header line.\n", + " \"\"\"\n", + " if not deviations:\n", + " print(\"No\", label, \"deviations found.\")\n", + " return\n", + "\n", + " sorted_devs = sorted(deviations)\n", + " n = len(sorted_devs)\n", + " mean_abs = round(sum(abs(d) for d in sorted_devs) / n, 4)\n", + " median = round(sorted_devs[n // 2], 4)\n", + " pct_90 = round(sorted_devs[int(n * 0.90)], 4)\n", + " pct_95 = round(sorted_devs[int(n * 0.95)], 4)\n", + "\n", + " within_50ms = sum(1 for d in sorted_devs if abs(d) <= 0.05)\n", + " within_100ms = sum(1 for d in sorted_devs if abs(d) <= 0.10)\n", + "\n", + " print(label, \"deviation summary across\", n, \"matched notes:\")\n", + " print(\" Mean absolute : \", mean_abs, \"s\")\n", + " print(\" Median : \", median, \"s\")\n", + " print(\" 90th pct : \", pct_90, \"s\")\n", + " print(\" 95th pct : \", pct_95, \"s\")\n", + " print(\" Within 50 ms : \", round(100.0 * within_50ms / n, 1), \"%\")\n", + " print(\" Within 100 ms : \", round(100.0 * within_100ms / n, 1), \"%\")\n", + "\n", + "\n", + "timing_deviations = collect_timing_deviations(all_results)\n", + "print_deviation_summary(timing_deviations, \"Timing\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Duration Deviation Distribution" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No Duration deviations found.\n" + ] + } + ], + "source": [ + "def collect_duration_deviations(all_results):\n", + " \"\"\"\n", + " Collect local duration deviations from all matched notes across all pieces.\n", + "\n", + " Args:\n", + " all_results: list of result dicts from run_alignment_on_samples().\n", + "\n", + " Returns:\n", + " List of floats (one per matched note), in seconds.\n", + " \"\"\"\n", + " deviations = []\n", + " for result in all_results:\n", + " for event in result[\"event_details\"]:\n", + " if event[\"event_type\"] != \"note\":\n", + " continue\n", + " if event[\"operation_type\"] != \"match\":\n", + " continue\n", + " if event.get(\"duration_deviation\") is not None:\n", + " deviations.append(event[\"duration_deviation\"])\n", + " return deviations\n", + "\n", + "\n", + "duration_deviations = collect_duration_deviations(all_results)\n", + "print_deviation_summary(duration_deviations, \"Duration\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "compareMusic", + "language": "python", + "name": "comparemusic" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 94fd4b14aa79f97ec6184381b3d6067cb0bc47c6 Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Fri, 3 Jul 2026 21:24:01 +0100 Subject: [PATCH 4/7] use an updated ASAP dataset --- notebooks/Alignment_on_ASAPdataset.ipynb | 980 ++++++++++++----------- 1 file changed, 500 insertions(+), 480 deletions(-) diff --git a/notebooks/Alignment_on_ASAPdataset.ipynb b/notebooks/Alignment_on_ASAPdataset.ipynb index ac38559..d604eae 100644 --- a/notebooks/Alignment_on_ASAPdataset.ipynb +++ b/notebooks/Alignment_on_ASAPdataset.ipynb @@ -4,63 +4,146 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The ASAP dataset contains paired MIDI score files (what *should* be played) and MIDI performance files (what a professional pianist *actually* played). We use the score as the reference and the performance as the student response, and run the ED alignment algorithm on each pair.\n", + "# ED Alignment Algorithm on Real Piano Performances\n", "\n", - "citation:\n", + "This notebook evaluates the ED alignment algorithm in `compareMusic` on real piano performances from the [(n)ASAP: the (note-)Aligned Scores And Performances dataset](https://github.com/CPJKU/asap-dataset) (Peter et al., 2023).\n", "\n", - "@inproceedings{asap-dataset,\n", - " title={{ASAP}: a dataset of aligned scores and performances for piano transcription},\n", - " author={Foscarin, Francesco and McLeod, Andrew and Rigaux, Philippe and Jacquemard, Florent and Sakai, Masahiko},\n", - " booktitle={International Society for Music Information Retrieval Conference {(ISMIR)}},\n", - " year={2020},\n", - " pages={534--541}\n", - "}" + "(n)ASAP is a dataset of aligned musical scores and performances built by extending the ASAP dataset with note-level annotations. The ASAP contains 236 distinct musical scores and 1067 performances of Western classical piano music from 15 different composers, the piece directory contains the XML and MIDI score, plus all of the performances of a specific piece, including: \n", + "- `midi_score`: the MIDI score (what should be played) — used as **reference**\n", + "- `midi_performance`: what the pianist actually played — used as **response**\n", + "- `note_alignments.tsv`: official note-level alignment annotations — used as **ground truth**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Download the ASAP Dataset" + "bib citation:\n", + "\n", + "@article{Peter-2023,\n", + " title = {Automatic Note-Level Score-to-Performance Alignments in the ASAP Dataset},\n", + " author = {Peter, Silvan David and Cancino-Chacón, Carlos Eduardo and Foscarin, Francesco and McLeod, Andrew Philip and Henkel, Florian and Karystinaios, Emmanouil and Widmer, Gerhard},\n", + " doi = {10.5334/tismir.149},\n", + " journal = {Transactions of the International Society for Music Information Retrieval {(TISMIR)}},\n", + " year = {2023}\n", + "}\n", + "\n", + "relevant paper: https://transactions.ismir.net/articles/10.5334/tismir.149#5-alignment-of-the-asap-dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download the ASAP Dataset" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 19, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "Cloning into 'asap-dataset'...\n", - "Updating files: 100% (2854/2854), done.\n" + "asap-dataset already exists, skipping download.\n" ] } ], "source": [ "import os\n", "\n", - "if not os.path.exists('asap-dataset'):\n", - " os.system('git clone https://github.com/fosfrancesco/asap-dataset.git')\n", + "# Note: use CPJKU version (not fosfrancesco) because only CPJKU has\n", + "# the note_alignments TSV files are needed for ground truth comparison.\n", + "if not os.path.exists(\"asap-dataset\"):\n", + " os.system(\"git clone https://github.com/CPJKU/asap-dataset.git\")\n", "else:\n", - " print('asap-dataset already exists, skipping download.')\n", + " print(\"asap-dataset already exists, skipping download.\")\n", "\n", - "ASAP_PATH = 'asap-dataset'" + "ASAP_PATH = \"asap-dataset\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Format conversion\n", + "## Load the ground truth\n", + "\n", + "The `note_alignment.tsv` file in each performance contains the official note-level alignment annotations. Each row in the TSV represents one note pair. \n", "\n", - "These functions convert a MIDI file into the `{pitch, start, duration}` format used by `compare_MIDI.py`." + "The `label` has three possible values:\n", + "- `match`: a score note (reference) and a performance note (response) were successfully aligned\n", + "- `deletion`: the score note was not played in the performance\n", + "- `insertion`: an extra note was played that has no counterpart in the score\n", + "\n", + "For `match` and `insertion` rows, the TSV provides `onset` (onset time in seconds) and `pitch` (MIDI pitch) for the performance note." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "\n", + "def load_ground_truth(tsv_path):\n", + " \"\"\"\n", + " Read a note_alignments TSV file from the CPJKU ASAP dataset.\n", + "\n", + " The TSV columns are: xml_id, midi_id, track, channel, pitch, onset\n", + " - if xml_id == 'insertion': extra note played with no score counterpart\n", + " - if midi_id == 'deletion': score note not played in the performance\n", + " - otherwise: matched pair, onset and pitch refer to the performance note\n", + "\n", + " Returns a list of dicts with keys: label, onset, pitch.\n", + " \"\"\"\n", + " rows = []\n", + " with open(tsv_path, \"r\", encoding=\"utf-8\") as f:\n", + " reader = csv.DictReader(f, delimiter=\"\\t\")\n", + " for row in reader:\n", + " xml_id = row[\"xml_id\"].strip()\n", + " midi_id = row[\"midi_id\"].strip()\n", + "\n", + " if midi_id == \"deletion\":\n", + " label = \"deletion\"\n", + " onset = None\n", + " pitch = None\n", + " elif xml_id == \"insertion\":\n", + " label = \"insertion\"\n", + " onset = float(row[\"onset\"])\n", + " pitch = int(row[\"pitch\"])\n", + " else:\n", + " label = \"paired\"\n", + " onset = float(row[\"onset\"])\n", + " pitch = int(row[\"pitch\"])\n", + "\n", + " rows.append({\n", + " \"label\": label, # \"paired\", \"insertion\", or \"deletion\"\n", + " \"onset\": onset,\n", + " \"pitch\": pitch,\n", + " })\n", + " return rows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Score (reference)/Performance (response) pairs from ASAP" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Helper functions for format conversion: These functions convert a MIDI file into the `{pitch, start, duration}` format used by `compare_MIDI.py`." + ] + }, + { + "cell_type": "code", + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -78,15 +161,15 @@ " continue\n", " for note in instrument.notes:\n", " all_notes.append({\n", - " \"pitch\": note.pitch,\n", - " \"start\": round(note.start, 3),\n", + " \"pitch\": note.pitch,\n", + " \"start\": round(note.start, 3),\n", " \"duration\": round(note.end - note.start, 3),\n", " })\n", " all_notes.sort(key=lambda n: (n[\"start\"], n[\"pitch\"]))\n", " return all_notes\n", "\n", "\n", - "def build_sample(ref_path, response_path, composer, title):\n", + "def build_sample(ref_path, response_path, composer, title, metadata_row):\n", " \"\"\"\n", " Convert one score/performance MIDI pair into a sample dict\n", " ready for compare_performance_ED.\n", @@ -99,10 +182,11 @@ " return None\n", "\n", " return {\n", - " \"composer\": composer,\n", - " \"title\": title,\n", + " \"composer\": composer,\n", + " \"title\": title,\n", " \"reference\": {\"notes\": score_notes},\n", - " \"response\": {\"notes\": perf_notes},\n", + " \"response\": {\"notes\": perf_notes},\n", + " \"metadata_row\": metadata_row,\n", " }" ] }, @@ -110,197 +194,41 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Load Score/Performance Pairs from ASAP Metadata\n", - "\n", "For the ASAP dataset, the `metadata.csv` lists every score/performance pair in the dataset, check that both MIDI files exist on disk." ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 22, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jz7125/compareMusic/.venv/lib/python3.13/site-packages/pretty_midi/pretty_midi.py:122: RuntimeWarning: Tempo, Key or Time signature change events found on non-zero tracks. This is not a valid type 0 or type 1 MIDI file. Tempo, Key or Time Signature may be wrong.\n", + " warnings.warn(\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Loaded 169 score/performance pairs.\n", - "\n", - "1. Bach / Fugue_bwv_846 | score notes: 755 | performance notes: 754\n", - "2. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1435\n", - "3. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1438\n", - "4. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1429\n", - "5. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1431\n", - "6. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1433\n", - "7. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1438\n", - "8. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1440\n", - "9. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1431\n", - "10. Bach / Fugue_bwv_848 | score notes: 1429 | performance notes: 1430\n", - "11. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 739\n", - "12. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 738\n", - "13. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 734\n", - "14. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 738\n", - "15. Bach / Fugue_bwv_854 | score notes: 732 | performance notes: 736\n", - "16. Bach / Fugue_bwv_856 | score notes: 798 | performance notes: 740\n", - "17. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1400\n", - "18. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1353\n", - "19. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1345\n", - "20. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1361\n", - "21. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1340\n", - "22. Bach / Fugue_bwv_857 | score notes: 1331 | performance notes: 1358\n", - "23. Bach / Fugue_bwv_858 | score notes: 930 | performance notes: 875\n", - "24. Bach / Fugue_bwv_858 | score notes: 930 | performance notes: 888\n", - "25. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1739\n", - "26. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1750\n", - "27. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1766\n", - "28. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1740\n", - "29. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1762\n", - "30. Bach / Fugue_bwv_860 | score notes: 1838 | performance notes: 1744\n", - "31. Bach / Fugue_bwv_862 | score notes: 886 | performance notes: 868\n", - "32. Bach / Fugue_bwv_863 | score notes: 797 | performance notes: 792\n", - "33. Bach / Fugue_bwv_863 | score notes: 797 | performance notes: 799\n", - "34. Bach / Fugue_bwv_863 | score notes: 797 | performance notes: 803\n", - "35. Bach / Fugue_bwv_864 | score notes: 1207 | performance notes: 1188\n", - "36. Bach / Fugue_bwv_865 | score notes: 2428 | performance notes: 2398\n", - "37. Bach / Fugue_bwv_865 | score notes: 2428 | performance notes: 2456\n", - "38. Bach / Fugue_bwv_865 | score notes: 2428 | performance notes: 2402\n", - "39. Bach / Fugue_bwv_866 | score notes: 946 | performance notes: 946\n", - "40. Bach / Fugue_bwv_867 | score notes: 760 | performance notes: 747\n", - "41. Bach / Fugue_bwv_867 | score notes: 760 | performance notes: 739\n", - "42. Bach / Fugue_bwv_868 | score notes: 868 | performance notes: 872\n", - "43. Bach / Fugue_bwv_870 | score notes: 1083 | performance notes: 1070\n", - "44. Bach / Fugue_bwv_870 | score notes: 1083 | performance notes: 1049\n", - "45. Bach / Fugue_bwv_870 | score notes: 1083 | performance notes: 1092\n", - "46. Bach / Fugue_bwv_873 | score notes: 1386 | performance notes: 1385\n", - "47. Bach / Fugue_bwv_873 | score notes: 1386 | performance notes: 1380\n", - "48. Bach / Fugue_bwv_874 | score notes: 865 | performance notes: 867\n", - "49. Bach / Fugue_bwv_874 | score notes: 865 | performance notes: 872\n", - "50. Bach / Fugue_bwv_874 | score notes: 865 | performance notes: 867\n", - "51. Bach / Fugue_bwv_875 | score notes: 794 | performance notes: 806\n", - "52. Bach / Fugue_bwv_875 | score notes: 794 | performance notes: 799\n", - "53. Bach / Fugue_bwv_876 | score notes: 701 | performance notes: 691\n", - "54. Bach / Fugue_bwv_876 | score notes: 701 | performance notes: 704\n", - "55. Bach / Fugue_bwv_880 | score notes: 973 | performance notes: 984\n", - "56. Bach / Fugue_bwv_880 | score notes: 973 | performance notes: 982\n", - "57. Bach / Fugue_bwv_880 | score notes: 973 | performance notes: 975\n", - "58. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1452\n", - "59. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1450\n", - "60. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1463\n", - "61. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1443\n", - "62. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1428\n", - "63. Bach / Fugue_bwv_883 | score notes: 1420 | performance notes: 1544\n", - "64. Bach / Fugue_bwv_884 | score notes: 748 | performance notes: 745\n", - "65. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1642\n", - "66. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1676\n", - "67. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1659\n", - "68. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1662\n", - "69. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1662\n", - "70. Bach / Fugue_bwv_885 | score notes: 1645 | performance notes: 1671\n", - "71. Bach / Fugue_bwv_887 | score notes: 1582 | performance notes: 1588\n", - "72. Bach / Fugue_bwv_888 | score notes: 755 | performance notes: 768\n", - "73. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 838\n", - "74. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 848\n", - "75. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 827\n", - "76. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 840\n", - "77. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 849\n", - "78. Bach / Fugue_bwv_889 | score notes: 821 | performance notes: 830\n", - "79. Bach / Fugue_bwv_891 | score notes: 1750 | performance notes: 1760\n", - "80. Bach / Fugue_bwv_892 | score notes: 1311 | performance notes: 1305\n", - "81. Bach / Fugue_bwv_892 | score notes: 1311 | performance notes: 1310\n", - "82. Bach / Fugue_bwv_893 | score notes: 1025 | performance notes: 992\n", - "83. Bach / Fugue_bwv_893 | score notes: 1025 | performance notes: 1020\n", - "84. Bach / Fugue_bwv_893 | score notes: 1025 | performance notes: 1031\n", - "85. Bach / Italian_concerto | score notes: 1096 | performance notes: 1221\n", - "86. Bach / Italian_concerto | score notes: 1096 | performance notes: 1196\n", - "87. Bach / Italian_concerto | score notes: 1096 | performance notes: 1171\n", - "88. Bach / Prelude_bwv_846 | score notes: 549 | performance notes: 548\n", - "89. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 811\n", - "90. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 816\n", - "91. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 810\n", - "92. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 810\n", - "93. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 810\n", - "94. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 811\n", - "95. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 806\n", - "96. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 811\n", - "97. Bach / Prelude_bwv_848 | score notes: 810 | performance notes: 808\n", - "98. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 459\n", - "99. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 444\n", - "100. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 443\n", - "101. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 444\n", - "102. Bach / Prelude_bwv_854 | score notes: 471 | performance notes: 465\n", - "103. Bach / Prelude_bwv_856 | score notes: 1045 | performance notes: 824\n", - "104. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 549\n", - "105. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 566\n", - "106. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 556\n", - "107. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 557\n", - "108. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 526\n", - "109. Bach / Prelude_bwv_857 | score notes: 575 | performance notes: 559\n", - "110. Bach / Prelude_bwv_858 | score notes: 497 | performance notes: 439\n", - "111. Bach / Prelude_bwv_858 | score notes: 497 | performance notes: 486\n", - "112. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 616\n", - "113. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 608\n", - "114. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 612\n", - "115. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 613\n", - "116. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 611\n", - "117. Bach / Prelude_bwv_860 | score notes: 607 | performance notes: 608\n", - "118. Bach / Prelude_bwv_862 | score notes: 696 | performance notes: 675\n", - "119. Bach / Prelude_bwv_863 | score notes: 571 | performance notes: 562\n", - "120. Bach / Prelude_bwv_863 | score notes: 571 | performance notes: 564\n", - "121. Bach / Prelude_bwv_863 | score notes: 571 | performance notes: 564\n", - "122. Bach / Prelude_bwv_864 | score notes: 618 | performance notes: 611\n", - "123. Bach / Prelude_bwv_865 | score notes: 608 | performance notes: 612\n", - "124. Bach / Prelude_bwv_865 | score notes: 608 | performance notes: 613\n", - "125. Bach / Prelude_bwv_865 | score notes: 608 | performance notes: 610\n", - "126. Bach / Prelude_bwv_866 | score notes: 634 | performance notes: 637\n", - "127. Bach / Prelude_bwv_867 | score notes: 776 | performance notes: 769\n", - "128. Bach / Prelude_bwv_867 | score notes: 776 | performance notes: 777\n", - "129. Bach / Prelude_bwv_868 | score notes: 417 | performance notes: 414\n", - "130. Bach / Prelude_bwv_870 | score notes: 832 | performance notes: 834\n", - "131. Bach / Prelude_bwv_870 | score notes: 832 | performance notes: 834\n", - "132. Bach / Prelude_bwv_870 | score notes: 832 | performance notes: 833\n", - "133. Bach / Prelude_bwv_873 | score notes: 1602 | performance notes: 1358\n", - "134. Bach / Prelude_bwv_873 | score notes: 1602 | performance notes: 1358\n", - "135. Bach / Prelude_bwv_874 | score notes: 1569 | performance notes: 1574\n", - "136. Bach / Prelude_bwv_874 | score notes: 1569 | performance notes: 1551\n", - "137. Bach / Prelude_bwv_875 | score notes: 1081 | performance notes: 1103\n", - "138. Bach / Prelude_bwv_875 | score notes: 1081 | performance notes: 1099\n", - "139. Bach / Prelude_bwv_875 | score notes: 1081 | performance notes: 1104\n", - "140. Bach / Prelude_bwv_876 | score notes: 914 | performance notes: 920\n", - "141. Bach / Prelude_bwv_876 | score notes: 914 | performance notes: 924\n", - "142. Bach / Prelude_bwv_880 | score notes: 1239 | performance notes: 1240\n", - "143. Bach / Prelude_bwv_880 | score notes: 1239 | performance notes: 1237\n", - "144. Bach / Prelude_bwv_880 | score notes: 1239 | performance notes: 1243\n", - "145. Bach / Prelude_bwv_883 | score notes: 882 | performance notes: 892\n", - "146. Bach / Prelude_bwv_883 | score notes: 882 | performance notes: 888\n", - "147. Bach / Prelude_bwv_883 | score notes: 882 | performance notes: 888\n", - "148. Bach / Prelude_bwv_884 | score notes: 902 | performance notes: 908\n", - "149. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 510\n", - "150. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 524\n", - "151. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 517\n", - "152. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 516\n", - "153. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 523\n", - "154. Bach / Prelude_bwv_885 | score notes: 524 | performance notes: 525\n", - "155. Bach / Prelude_bwv_887 | score notes: 1306 | performance notes: 1305\n", - "156. Bach / Prelude_bwv_888 | score notes: 688 | performance notes: 681\n", - "157. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 785\n", - "158. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 782\n", - "159. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 780\n", - "160. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 786\n", - "161. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 789\n", - "162. Bach / Prelude_bwv_889 | score notes: 780 | performance notes: 795\n", - "163. Bach / Prelude_bwv_891 | score notes: 1182 | performance notes: 1180\n", - "164. Bach / Prelude_bwv_891 | score notes: 1182 | performance notes: 1185\n", - "165. Bach / Prelude_bwv_892 | score notes: 1089 | performance notes: 1084\n", - "166. Bach / Prelude_bwv_892 | score notes: 1089 | performance notes: 1086\n", - "167. Bach / Prelude_bwv_893 | score notes: 882 | performance notes: 876\n", - "168. Bach / Prelude_bwv_893 | score notes: 882 | performance notes: 885\n", - "169. Bach / Prelude_bwv_893 | score notes: 882 | performance notes: 895\n" + "Loaded 169 score (reference) /performance (response) pairs.\n", + "1. Bach(Fugue_bwv_846) | ref notes: 755 | response notes: 754\n", + "2. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1435\n", + "3. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1438\n", + "4. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1429\n", + "5. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1431\n", + "6. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1433\n", + "7. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1438\n", + "8. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1440\n", + "9. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1431\n", + "10. Bach(Fugue_bwv_848) | ref notes: 1429 | response notes: 1430\n" ] } ], "source": [ - "import csv\n", - "\n", "def load_samples(asap_path, composer):\n", " \"\"\"\n", " Read the ASAP metadata CSV and return a list of sample dicts\n", @@ -312,34 +240,29 @@ " with open(metadata_path, \"r\", encoding=\"utf-8\") as csv_file:\n", " reader = csv.DictReader(csv_file)\n", " for row in reader:\n", - " if row.get(\"composer\", \"\").strip() != composer:\n", - " continue\n", - "\n", - " ref_path = os.path.join(asap_path, row.get(\"midi_score\", \"\").strip())\n", - " response_path = os.path.join(asap_path, row.get(\"midi_performance\", \"\").strip())\n", - "\n", - " if os.path.isfile(ref_path) and os.path.isfile(response_path):\n", - " sample = build_sample(\n", - " ref_path, response_path,\n", - " row.get(\"composer\", \"Unknown\"),\n", - " row.get(\"title\", \"Unknown\"),\n", - " )\n", - " if sample is not None:\n", - " samples.append(sample)\n", + " if row.get(\"composer\", \"\").strip() == composer:\n", + " ref_path = os.path.join(asap_path, row.get(\"midi_score\", \"\").strip())\n", + " response_path = os.path.join(asap_path, row.get(\"midi_performance\", \"\").strip())\n", + "\n", + " if os.path.isfile(ref_path) and os.path.isfile(response_path):\n", + " sample = build_sample(\n", + " ref_path, response_path,\n", + " row.get(\"composer\", \"Unknown\"),\n", + " row.get(\"title\", \"Unknown\"),\n", + " dict(row),\n", + " )\n", + " if sample is not None:\n", + " samples.append(sample)\n", "\n", " return samples\n", "\n", "samples = load_samples(ASAP_PATH, \"Bach\")\n", "\n", - "print(\"Loaded\", len(samples), \"score/performance pairs.\")\n", - "print()\n", - "for i, sample in enumerate(samples):\n", - " print(\n", - " str(i + 1) + \".\",\n", - " sample[\"composer\"], \"/\", sample[\"title\"],\n", - " \" | score notes:\", len(sample[\"reference\"][\"notes\"]),\n", - " \" | performance notes:\", len(sample[\"response\"][\"notes\"])\n", - " )" + "print(\"Loaded\", len(samples), \"score (reference) /performance (response) pairs.\")\n", + "for i, sample in enumerate(samples[:10]):\n", + " print(f\"{str(i + 1)}. {sample['composer']}({sample['title']}) |\"\n", + " f\" ref notes: {len(sample['reference']['notes'])} |\"\n", + " f\" response notes: {len(sample['response']['notes'])}\")" ] }, { @@ -348,22 +271,14 @@ "source": [ "# Run Alignment on Each Pair\n", "\n", - "Passing each score/performance pair through `compare_performance_ED`, where score MIDI is the **reference** and the performance MIDI is the **response**." + "Passing each score/performance pair through `compare_performance_ED`, where score MIDI is the reference and the performance MIDI is the response." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 23, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Alignment complete for 169 pieces.\n" - ] - } - ], + "outputs": [], "source": [ "from evaluation_function.compare_MIDI import compare_performance_ED\n", "\n", @@ -385,28 +300,247 @@ " sample[\"reference\"]\n", " )\n", " results.append({\n", - " \"composer\": sample[\"composer\"],\n", - " \"title\": sample[\"title\"],\n", - " \"stats\": result.stats,\n", + " \"composer\": sample[\"composer\"],\n", + " \"title\": sample[\"title\"],\n", + " \"stats\": result.stats,\n", " \"event_details\": result.event_details,\n", - " \"is_correct\": result.is_correct,\n", + " \"is_correct\": result.is_correct,\n", " })\n", " return results\n", "\n", - "all_results = run_alignment_on_samples(samples)\n", - "print(\"Alignment complete for\", len(all_results), \"pieces.\")" + "all_results = run_alignment_on_samples(samples)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Precision, Recall, and F1 " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "summary" + "Compares our alignment output against the ASAP ground truth to compute standard information retrieval metrics:\n", + "\n", + "- **Precision**: TP / (TP + FP) \n", + " - i.e. of all note pairs we called a match, how many the ground truth also call a match\n", + "- **Recall**: TP / (TP + FN)\n", + " - i.e of all note pairs the ground truth called a match, how many we find\n", + "- **F1**: harmonic mean of precision and recall\n", + "\n", + "The alignment is correct if the pitch is correct and the number of matches at that pitch does not exceed the ground truth count. Onset time is intentionally excluded from this metric, consistent with the pipeline's design: the cost function uses pitch distance only, so that timing evaluation remains a separate step performed after structural alignment rather than influencing the alignment itself." ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "def convert_pipeline_output(event_details, response_notes):\n", + " \"\"\"\n", + " Convert event_details from compare_performance_ED into a list of\n", + " dicts with keys: onset, pitch, label.\n", + " Labels follow the TSV convention:\n", + " 'paired': correctly aligned note pair (pitch matches)\n", + " 'deletion' : score note not found in response (missing)\n", + " 'insertion': response note has no score counterpart (extra)\n", + "\n", + " Chord events are expanded note-by-note using correct_pitches /\n", + " missing_pitches / extra_pitches from event_level_feedback.\n", + "\n", + " Args:\n", + " event_details: list of dicts from compare_performance_ED\n", + " response_notes: list of note dicts (the flat, normalised note list\n", + " that was passed into compare_performance_ED as\n", + " responseMIDI[\"notes\"], after normalize_start_times)\n", + "\n", + " Returns:\n", + " list of dicts with keys: onset, pitch, label\n", + " \"\"\"\n", + " my_pairs = []\n", + "\n", + " for event in event_details:\n", + " op = event[\"operation_type\"]\n", + "\n", + " if event[\"event_type\"] == \"note\":\n", + " if op == \"match\" or op == \"replacement\":\n", + " note = response_notes[event[\"response_index\"] - 1]\n", + " my_pairs.append({\n", + " \"onset\": note[\"start\"],\n", + " \"pitch\": note[\"pitch\"],\n", + " \"label\": \"paired\",\n", + " })\n", + " elif op == \"missing\":\n", + " my_pairs.append({\n", + " \"onset\": None,\n", + " \"pitch\": None,\n", + " \"label\": \"deletion\",\n", + " })\n", + " elif op == \"extra\":\n", + " note = response_notes[event[\"response_index\"] - 1]\n", + " my_pairs.append({\n", + " \"onset\": note[\"start\"],\n", + " \"pitch\": note[\"pitch\"],\n", + " \"label\": \"insertion\",\n", + " })\n", + "\n", + " elif event[\"event_type\"] == \"chord\":\n", + " if op == \"missing\":\n", + " # Entire chord missing: one deletion per note in the ref chord\n", + " # We know how many notes because correct + missing = all ref notes\n", + " num_ref_notes = len(event[\"correct_pitches\"] or []) + \\\n", + " len(event[\"missing_pitches\"] or [])\n", + " for _ in range(num_ref_notes):\n", + " my_pairs.append({\n", + " \"onset\": None,\n", + " \"pitch\": None,\n", + " \"label\": \"deletion\",\n", + " })\n", + " elif op == \"extra\":\n", + " # Entire chord is extra: one insertion per note played\n", + " res_idx_0 = event[\"response_index\"] - 1\n", + " note = response_notes[res_idx_0]\n", + " my_pairs.append({\n", + " \"onset\": note[\"start\"],\n", + " \"pitch\": note[\"pitch\"],\n", + " \"label\": \"insertion\",\n", + " })\n", + " else:\n", + " # Aligned chord pair (match or replacement)\n", + " # correct_pitches -> match for each\n", + " # missing_pitches -> deletion for each\n", + " # extra_pitches -> insertion for each\n", + " res_idx_0 = event[\"response_index\"] - 1\n", + " note = response_notes[res_idx_0]\n", + " onset = note[\"start\"]\n", + " for _ in (event[\"correct_pitches\"] or []):\n", + " my_pairs.append({\n", + " \"onset\": onset,\n", + " \"pitch\": note[\"pitch\"],\n", + " \"label\": \"paired\",\n", + " })\n", + " for _ in (event[\"missing_pitches\"] or []):\n", + " my_pairs.append({\n", + " \"onset\": None,\n", + " \"pitch\": None,\n", + " \"label\": \"deletion\",\n", + " })\n", + " for _ in (event[\"extra_pitches\"] or []):\n", + " my_pairs.append({\n", + " \"onset\": onset,\n", + " \"pitch\": note[\"pitch\"],\n", + " \"label\": \"insertion\",\n", + " })\n", + "\n", + " return my_pairs\n", + "\n", + "\n", + "def compute_metrics(ground_truth, my_pairs):\n", + " \"\"\"\n", + " Compute the metrics in a fairer way for pipelines that align \n", + " by pitch without considering onset time.\n", + "\n", + " For my match, count as a True Positive if:\n", + " - its pitch appears in the GT match set for that pitch,\n", + " and the GT has at least one unaccounted match at that pitch.\n", + "\n", + " This uses a multiset comparison: if GT has 3 matches at pitch 60\n", + " and we predict 3 matches at pitch 60, all 3 are TPs. If we predict\n", + " 4, the extra one is an FP.\n", + "\n", + " Args:\n", + " ground_truth: list of dicts from load_ground_truth()\n", + " my_pairs: list of dicts from convert_pipeline_output()\n", + "\n", + " Returns:\n", + " dict with precision, recall, f1, tp, fp, fn\n", + " \"\"\"\n", + " # Build a pitch count multiset for GT matches\n", + " gt_pitch_counts = {}\n", + " for row in ground_truth:\n", + " if row[\"label\"] == \"paired\":\n", + " p = int(row[\"pitch\"])\n", + " if p not in gt_pitch_counts:\n", + " gt_pitch_counts[p] = 0\n", + " gt_pitch_counts[p] += 1\n", + "\n", + " # Build a pitch count multiset for my matches\n", + " my_pitch_counts = {}\n", + " for row in my_pairs:\n", + " if row[\"label\"] == \"paired\" and row[\"onset\"] is not None:\n", + " p = int(row[\"pitch\"])\n", + " if p not in my_pitch_counts:\n", + " my_pitch_counts[p] = 0\n", + " my_pitch_counts[p] += 1\n", + "\n", + " tp = 0\n", + " fp = 0\n", + " fn = 0\n", + "\n", + " all_pitches = set(gt_pitch_counts.keys()).union(set(my_pitch_counts.keys()))\n", + " for pitch in all_pitches:\n", + " gt_count = gt_pitch_counts.get(pitch, 0) # Count of GT matches at this pitch\n", + " my_count = my_pitch_counts.get(pitch, 0) # Count of my matches at this pitch\n", + " tp += min(gt_count, my_count) \n", + " fp += max(0, my_count - gt_count) \n", + " fn += max(0, gt_count - my_count)\n", + "\n", + " if tp + fp > 0:\n", + " precision = tp / (tp + fp)\n", + " else:\n", + " precision = 0.0\n", + "\n", + " if tp + fn > 0:\n", + " recall = tp / (tp + fn)\n", + " else:\n", + " recall = 0.0\n", + "\n", + " if precision + recall > 0:\n", + " f1 = 2 * precision * recall / (precision + recall)\n", + " else:\n", + " f1 = 0.0\n", + "\n", + " return {\n", + " \"precision\": round(precision, 4),\n", + " \"recall\": round(recall, 4),\n", + " \"f1\": round(f1, 4),\n", + " \"tp\": tp, \"fp\": fp, \"fn\": fn,\n", + " }\n", + "\n", + "\n", + "def evaluate_one_piece(asap_path, metadata_row, event_details, response_notes_original):\n", + " \"\"\"\n", + " Run ground truth comparison for one score/performance pair.\n", + "\n", + " Args:\n", + " asap_path: str, path to ASAP repo root\n", + " metadata_row: dict, one row from metadata.csv\n", + " event_details: list from compare_performance_ED result\n", + " response_notes_original: list of note dicts with original\n", + " (absolute) onset times — NOT normalised\n", + "\n", + " Returns:\n", + " metrics dict, or None if the TSV file is not found\n", + " \"\"\"\n", + " tsv_rel = metadata_row.get(\"note_alignments\", \"\").strip()\n", + " tsv_path = os.path.join(asap_path, tsv_rel)\n", + "\n", + " if not os.path.isfile(tsv_path):\n", + " print(\"TSV not found:\", tsv_path)\n", + " return None\n", + "\n", + " ground_truth = load_ground_truth(tsv_path)\n", + " my_pairs = convert_pipeline_output(event_details, response_notes_original)\n", + "\n", + " return compute_metrics(ground_truth, my_pairs)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -431,288 +565,174 @@ " \n", " \n", " Piece\n", - " Score\n", - " Missing\n", - " Extra\n", - " Wrong\n", - " Match %\n", + " Precision\n", + " Recall\n", + " F1\n", + " TP\n", + " FP\n", + " FN\n", " \n", " \n", " \n", " \n", " 0\n", - " Bach / Fugue_bwv_846\n", - " 187\n", - " 0\n", - " 12\n", - " 3\n", - " 98.4\n", + " Bach (Fugue_bwv_846)\n", + " 0.9092\n", + " 0.8415\n", + " 0.8740\n", + " 621\n", + " 62\n", + " 117\n", " \n", " \n", " 1\n", - " Bach / Fugue_bwv_848\n", - " 313\n", - " 4\n", - " 4\n", - " 7\n", - " 96.5\n", + " Bach (Fugue_bwv_848)\n", + " 0.9293\n", + " 0.9004\n", + " 0.9146\n", + " 1275\n", + " 97\n", + " 141\n", " \n", " \n", " 2\n", - " Bach / Fugue_bwv_848\n", - " 313\n", - " 4\n", - " 12\n", - " 8\n", - " 96.2\n", + " Bach (Fugue_bwv_848)\n", + " 0.9288\n", + " 0.8946\n", + " 0.9114\n", + " 1265\n", + " 97\n", + " 149\n", " \n", " \n", " 3\n", - " Bach / Fugue_bwv_848\n", - " 313\n", - " 2\n", - " 11\n", - " 14\n", - " 94.9\n", + " Bach (Fugue_bwv_848)\n", + " 0.9420\n", + " 0.9100\n", + " 0.9257\n", + " 1284\n", + " 79\n", + " 127\n", " \n", " \n", " 4\n", - " Bach / Fugue_bwv_848\n", - " 313\n", - " 4\n", - " 12\n", - " 9\n", - " 95.8\n", + " Bach (Fugue_bwv_848)\n", + " 0.9435\n", + " 0.9147\n", + " 0.9289\n", + " 1287\n", + " 77\n", + " 120\n", " \n", " \n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", + " 5\n", + " Bach (Fugue_bwv_848)\n", + " 0.9470\n", + " 0.9190\n", + " 0.9328\n", + " 1304\n", + " 73\n", + " 115\n", " \n", " \n", - " 164\n", - " Bach / Prelude_bwv_892\n", - " 440\n", - " 16\n", - " 7\n", - " 7\n", - " 94.8\n", + " 6\n", + " Bach (Fugue_bwv_848)\n", + " 0.9184\n", + " 0.8905\n", + " 0.9043\n", + " 1261\n", + " 112\n", + " 155\n", " \n", " \n", - " 165\n", - " Bach / Prelude_bwv_892\n", - " 440\n", - " 9\n", - " 5\n", - " 18\n", - " 93.9\n", + " 7\n", + " Bach (Fugue_bwv_848)\n", + " 0.9394\n", + " 0.9014\n", + " 0.9200\n", + " 1271\n", + " 82\n", + " 139\n", " \n", " \n", - " 166\n", - " Bach / Prelude_bwv_893\n", - " 428\n", - " 7\n", - " 5\n", - " 2\n", - " 97.9\n", + " 8\n", + " Bach (Fugue_bwv_848)\n", + " 0.9381\n", + " 0.9096\n", + " 0.9236\n", + " 1288\n", + " 85\n", + " 128\n", " \n", " \n", - " 167\n", - " Bach / Prelude_bwv_893\n", - " 428\n", - " 3\n", - " 2\n", - " 3\n", - " 98.6\n", - " \n", - " \n", - " 168\n", - " Bach / Prelude_bwv_893\n", - " 428\n", - " 1\n", - " 25\n", - " 13\n", - " 96.7\n", + " 9\n", + " Bach (Fugue_bwv_848)\n", + " 0.9366\n", + " 0.9082\n", + " 0.9222\n", + " 1286\n", + " 87\n", + " 130\n", " \n", " \n", "\n", - "

169 rows × 6 columns

\n", "" ], "text/plain": [ - " Piece Score Missing Extra Wrong Match %\n", - "0 Bach / Fugue_bwv_846 187 0 12 3 98.4\n", - "1 Bach / Fugue_bwv_848 313 4 4 7 96.5\n", - "2 Bach / Fugue_bwv_848 313 4 12 8 96.2\n", - "3 Bach / Fugue_bwv_848 313 2 11 14 94.9\n", - "4 Bach / Fugue_bwv_848 313 4 12 9 95.8\n", - ".. ... ... ... ... ... ...\n", - "164 Bach / Prelude_bwv_892 440 16 7 7 94.8\n", - "165 Bach / Prelude_bwv_892 440 9 5 18 93.9\n", - "166 Bach / Prelude_bwv_893 428 7 5 2 97.9\n", - "167 Bach / Prelude_bwv_893 428 3 2 3 98.6\n", - "168 Bach / Prelude_bwv_893 428 1 25 13 96.7\n", - "\n", - "[169 rows x 6 columns]" + " Piece Precision Recall F1 TP FP FN\n", + "0 Bach (Fugue_bwv_846) 0.9092 0.8415 0.8740 621 62 117\n", + "1 Bach (Fugue_bwv_848) 0.9293 0.9004 0.9146 1275 97 141\n", + "2 Bach (Fugue_bwv_848) 0.9288 0.8946 0.9114 1265 97 149\n", + "3 Bach (Fugue_bwv_848) 0.9420 0.9100 0.9257 1284 79 127\n", + "4 Bach (Fugue_bwv_848) 0.9435 0.9147 0.9289 1287 77 120\n", + "5 Bach (Fugue_bwv_848) 0.9470 0.9190 0.9328 1304 73 115\n", + "6 Bach (Fugue_bwv_848) 0.9184 0.8905 0.9043 1261 112 155\n", + "7 Bach (Fugue_bwv_848) 0.9394 0.9014 0.9200 1271 82 139\n", + "8 Bach (Fugue_bwv_848) 0.9381 0.9096 0.9236 1288 85 128\n", + "9 Bach (Fugue_bwv_848) 0.9366 0.9082 0.9222 1286 87 130" ] }, - "execution_count": 9, "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "def make_results_table(all_results):\n", - " \"\"\"Build a summary DataFrame from alignment results.\"\"\"\n", - " rows = []\n", - " for r in all_results:\n", - " stats = r[\"stats\"]\n", - " total = stats[\"total_notes_in_reference\"]\n", - " missing = stats[\"total_notes_missing\"]\n", - " wrong = stats[\"total_notes_wrong_pitch\"]\n", - " matched = total - missing - wrong\n", - " rows.append({\n", - " \"Piece\": r[\"composer\"] + \" / \" + r[\"title\"],\n", - " \"Score\": total,\n", - " \"Missing\": missing,\n", - " \"Extra\": stats[\"total_notes_extra\"],\n", - " \"Wrong\": wrong,\n", - " \"Match %\": round(100.0 * matched / total, 1) if total > 0 else 0.0,\n", - " })\n", - " return pd.DataFrame(rows)\n", - "\n", - "make_results_table(all_results)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Timing Deviation Distribution" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", "text": [ - "No Timing deviations found.\n" + "Mean Precision: 0.9238\n", + "Mean Recall : 0.8876\n", + "Mean F1 : 0.9051\n" ] } ], "source": [ - "def collect_timing_deviations(all_results):\n", - " \"\"\"\n", - " Collect local timing deviations from all matched notes across all pieces.\n", - "\n", - " Args:\n", - " all_results: list of result dicts from run_alignment_on_samples().\n", - "\n", - " Returns:\n", - " List of floats (one per matched note), in seconds.\n", - " \"\"\"\n", - " deviations = []\n", - " for result in all_results:\n", - " for event in result[\"event_details\"]:\n", - " if event[\"event_type\"] != \"note\":\n", - " continue\n", - " if event[\"operation_type\"] != \"match\":\n", - " continue\n", - " if event.get(\"timing_deviation\") is not None:\n", - " deviations.append(event[\"timing_deviation\"])\n", - " return deviations\n", - "\n", - "\n", - "def print_deviation_summary(deviations, label):\n", - " \"\"\"\n", - " Print a text summary of any list of deviation values.\n", + "import pandas as pd\n", "\n", - " Args:\n", - " deviations: list of floats.\n", - " label: str, printed in the header line.\n", - " \"\"\"\n", - " if not deviations:\n", - " print(\"No\", label, \"deviations found.\")\n", - " return\n", - "\n", - " sorted_devs = sorted(deviations)\n", - " n = len(sorted_devs)\n", - " mean_abs = round(sum(abs(d) for d in sorted_devs) / n, 4)\n", - " median = round(sorted_devs[n // 2], 4)\n", - " pct_90 = round(sorted_devs[int(n * 0.90)], 4)\n", - " pct_95 = round(sorted_devs[int(n * 0.95)], 4)\n", - "\n", - " within_50ms = sum(1 for d in sorted_devs if abs(d) <= 0.05)\n", - " within_100ms = sum(1 for d in sorted_devs if abs(d) <= 0.10)\n", - "\n", - " print(label, \"deviation summary across\", n, \"matched notes:\")\n", - " print(\" Mean absolute : \", mean_abs, \"s\")\n", - " print(\" Median : \", median, \"s\")\n", - " print(\" 90th pct : \", pct_90, \"s\")\n", - " print(\" 95th pct : \", pct_95, \"s\")\n", - " print(\" Within 50 ms : \", round(100.0 * within_50ms / n, 1), \"%\")\n", - " print(\" Within 100 ms : \", round(100.0 * within_100ms / n, 1), \"%\")\n", - "\n", - "\n", - "timing_deviations = collect_timing_deviations(all_results)\n", - "print_deviation_summary(timing_deviations, \"Timing\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Duration Deviation Distribution" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No Duration deviations found.\n" - ] - } - ], - "source": [ - "def collect_duration_deviations(all_results):\n", - " \"\"\"\n", - " Collect local duration deviations from all matched notes across all pieces.\n", + "eval_rows = []\n", + "for sample, result in zip(samples, all_results):\n", + " metrics = evaluate_one_piece(\n", + " ASAP_PATH,\n", + " sample[\"metadata_row\"],\n", + " result[\"event_details\"],\n", + " sample[\"response\"][\"notes\"], \n", + " )\n", + " if metrics is not None:\n", + " eval_rows.append({\n", + " \"Piece\": result[\"composer\"] + \" (\" + result[\"title\"] + \")\",\n", + " \"Precision\": metrics[\"precision\"],\n", + " \"Recall\": metrics[\"recall\"],\n", + " \"F1\": metrics[\"f1\"],\n", + " \"TP\": metrics[\"tp\"],\n", + " \"FP\": metrics[\"fp\"],\n", + " \"FN\": metrics[\"fn\"],\n", + " })\n", "\n", - " Args:\n", - " all_results: list of result dicts from run_alignment_on_samples().\n", + "df_eval = pd.DataFrame(eval_rows)\n", + "display(df_eval.head(10))\n", "\n", - " Returns:\n", - " List of floats (one per matched note), in seconds.\n", - " \"\"\"\n", - " deviations = []\n", - " for result in all_results:\n", - " for event in result[\"event_details\"]:\n", - " if event[\"event_type\"] != \"note\":\n", - " continue\n", - " if event[\"operation_type\"] != \"match\":\n", - " continue\n", - " if event.get(\"duration_deviation\") is not None:\n", - " deviations.append(event[\"duration_deviation\"])\n", - " return deviations\n", - "\n", - "\n", - "duration_deviations = collect_duration_deviations(all_results)\n", - "print_deviation_summary(duration_deviations, \"Duration\")" + "print(\"Mean Precision:\", round(df_eval[\"Precision\"].mean(), 4))\n", + "print(\"Mean Recall :\", round(df_eval[\"Recall\"].mean(), 4))\n", + "print(\"Mean F1 :\", round(df_eval[\"F1\"].mean(), 4))" ] } ], From 210f57c2d9e9fcd53010a01f6041660bb62126a4 Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Sat, 4 Jul 2026 00:04:27 +0100 Subject: [PATCH 5/7] fix indexing and onset mismatch problem, modify the metrics computation --- notebooks/Alignment_on_ASAPdataset.ipynb | 491 +++++++++++++---------- 1 file changed, 284 insertions(+), 207 deletions(-) diff --git a/notebooks/Alignment_on_ASAPdataset.ipynb b/notebooks/Alignment_on_ASAPdataset.ipynb index d604eae..09b76cb 100644 --- a/notebooks/Alignment_on_ASAPdataset.ipynb +++ b/notebooks/Alignment_on_ASAPdataset.ipynb @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -96,8 +96,15 @@ " - if xml_id == 'insertion': extra note played with no score counterpart\n", " - if midi_id == 'deletion': score note not played in the performance\n", " - otherwise: matched pair, onset and pitch refer to the performance note\n", + " \n", + " Args:\n", + " tsv_path: str, path to the note_alignments.tsv file\n", "\n", - " Returns a list of dicts with keys: label, onset, pitch.\n", + " Returns:\n", + " list of dicts, each with keys:\n", + " label -> 'paired', 'insertion', or 'deletion'\n", + " onset -> float (seconds) or None for deletions\n", + " pitch -> int (MIDI pitch number) or None for deletions\n", " \"\"\"\n", " rows = []\n", " with open(tsv_path, \"r\", encoding=\"utf-8\") as f:\n", @@ -107,23 +114,19 @@ " midi_id = row[\"midi_id\"].strip()\n", "\n", " if midi_id == \"deletion\":\n", - " label = \"deletion\"\n", - " onset = None\n", - " pitch = None\n", + " rows.append({\"label\": \"deletion\", \"onset\": None, \"pitch\": None})\n", " elif xml_id == \"insertion\":\n", - " label = \"insertion\"\n", - " onset = float(row[\"onset\"])\n", - " pitch = int(row[\"pitch\"])\n", + " rows.append({\n", + " \"label\": \"insertion\",\n", + " \"onset\": float(row[\"onset\"]),\n", + " \"pitch\": int(row[\"pitch\"]),\n", + " })\n", " else:\n", - " label = \"paired\"\n", - " onset = float(row[\"onset\"])\n", - " pitch = int(row[\"pitch\"])\n", - "\n", - " rows.append({\n", - " \"label\": label, # \"paired\", \"insertion\", or \"deletion\"\n", - " \"onset\": onset,\n", - " \"pitch\": pitch,\n", - " })\n", + " rows.append({\n", + " \"label\": \"paired\",\n", + " \"onset\": float(row[\"onset\"]),\n", + " \"pitch\": int(row[\"pitch\"]),\n", + " })\n", " return rows" ] }, @@ -143,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -152,7 +155,15 @@ "def midi_file_to_notes(midi_path):\n", " \"\"\"\n", " Parse a MIDI file and return all its notes in compareMusic format.\n", - " Notes are sorted by onset time.\n", + " \n", + " Args:\n", + " midi_path: str, path to the MIDI file\n", + "\n", + " Returns:\n", + " list of dicts sorted by onset time, each with keys:\n", + " pitch -> int, MIDI note number\n", + " start -> float, onset time in seconds\n", + " duration -> float, note length in seconds\n", " \"\"\"\n", " midi_data = pretty_midi.PrettyMIDI(midi_path)\n", " all_notes = []\n", @@ -173,7 +184,17 @@ " \"\"\"\n", " Convert one score/performance MIDI pair into a sample dict\n", " ready for compare_performance_ED.\n", - " Returns None if either MIDI file produces zero notes.\n", + "\n", + " Args:\n", + " ref_path: str, path to the score (reference) MIDI file\n", + " response_path: str, path to the performance (response) MIDI file\n", + " composer: str\n", + " title: str\n", + " metadata_row: dict, one row from metadata.csv\n", + "\n", + " Returns:\n", + " sample dict with keys: composer, title, reference, response, metadata_row\n", + " Returns None if either MIDI file produces zero notes.\n", " \"\"\"\n", " score_notes = midi_file_to_notes(ref_path)\n", " perf_notes = midi_file_to_notes(response_path)\n", @@ -199,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -233,6 +254,13 @@ " \"\"\"\n", " Read the ASAP metadata CSV and return a list of sample dicts\n", " for a specific composer only.\n", + "\n", + " Args:\n", + " asap_path: str, path to the cloned ASAP repo\n", + " composer: str, e.g. \"Bach\"\n", + "\n", + " Returns:\n", + " list of sample dicts (see build_sample)\n", " \"\"\"\n", " metadata_path = os.path.join(asap_path, \"metadata.csv\")\n", " samples = []\n", @@ -271,44 +299,84 @@ "source": [ "# Run Alignment on Each Pair\n", "\n", - "Passing each score/performance pair through `compare_performance_ED`, where score MIDI is the reference and the performance MIDI is the response." + "Passing each score/performance pair through `compare_performance_ED`, where score MIDI is the reference and the performance MIDI is the response.\n", + "\n", + "**Why do we need the normalised events? - to fix the onset mismatch bug**\n", + "- Inside `compare_performance_ED`, `normalize_start_times` shifts the first note to t = 0, and `group_notes_into_events` groups simultaneous notes into chords. The `response_index` values stored in `event_details` refer to positions in these grouped events, not in the original flat note list. Therefore, need to reproduce these two steps here so that the correct onset and pitch for each event during ground truth comparison can be looked up." ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Alignment complete for 169 pieces.\n" + ] + } + ], "source": [ - "from evaluation_function.compare_MIDI import compare_performance_ED\n", + "from evaluation_function.compare_MIDI import (\n", + " compare_performance_ED,\n", + " normalize_start_times,\n", + " group_notes_into_events,\n", + ")\n", + "\n", "\n", "def run_alignment_on_samples(samples):\n", " \"\"\"\n", - " Run compare_performance_ED on every sample and return the results.\n", + " Run compare_performance_ED on every sample and collect results.\n", + "\n", + " We also store:\n", + " - response_events_normalized: the grouped event list after start-time\n", + " normalisation, needed to look up correct onset/pitch during evaluation.\n", + " - response_onset_offset: the original first-note onset time, needed to\n", + " convert normalised onsets back to absolute time for GT comparison.\n", "\n", " Args:\n", - " samples: list of sample dicts from load_samples().\n", + " samples: list of sample dicts from load_samples()\n", "\n", " Returns:\n", - " List of result dicts, each with keys:\n", - " composer, title, stats, event_details, is_correct\n", + " list of result dicts, each with keys:\n", + " composer, title, stats, event_details, is_correct,\n", + " response_events_normalized, response_onset_offset\n", " \"\"\"\n", " results = []\n", " for sample in samples:\n", " result = compare_performance_ED(\n", " sample[\"response\"],\n", - " sample[\"reference\"]\n", + " sample[\"reference\"],\n", " )\n", + "\n", + " # Reproduce the same normalisation + grouping done inside the pipeline.\n", + " # This gives us the event list whose indices match event_details.\n", + " response_notes_norm = normalize_start_times(sample[\"response\"][\"notes\"])\n", + " response_events_norm = group_notes_into_events(response_notes_norm)\n", + "\n", + " # Record the offset that normalize_start_times subtracted from every onset.\n", + " # Adding it back can convert a normalised onset to the original onset.\n", + " if sample[\"response\"][\"notes\"]:\n", + " response_onset_offset = sample[\"response\"][\"notes\"][0][\"start\"]\n", + " else:\n", + " response_onset_offset = 0.0\n", + "\n", " results.append({\n", " \"composer\": sample[\"composer\"],\n", " \"title\": sample[\"title\"],\n", " \"stats\": result.stats,\n", " \"event_details\": result.event_details,\n", " \"is_correct\": result.is_correct,\n", + " \"response_events_normalized\": response_events_norm,\n", + " \"response_onset_offset\": response_onset_offset,\n", " })\n", + "\n", " return results\n", "\n", - "all_results = run_alignment_on_samples(samples)" + "all_results = run_alignment_on_samples(samples)\n", + "print(\"Alignment complete for\", len(all_results), \"pieces.\")" ] }, { @@ -330,16 +398,18 @@ " - i.e of all note pairs the ground truth called a match, how many we find\n", "- **F1**: harmonic mean of precision and recall\n", "\n", - "The alignment is correct if the pitch is correct and the number of matches at that pitch does not exceed the ground truth count. Onset time is intentionally excluded from this metric, consistent with the pipeline's design: the cost function uses pitch distance only, so that timing evaluation remains a separate step performed after structural alignment rather than influencing the alignment itself." + "A predicted match is a True Positive only if both pitch and onset time (rounded to 1 decimal place) agree with the ground truth.\n", + "\n", + "**fix the index problem caused by the grouping in the pipeline**: look up pitches and onsets from `response_events_normalized` (the grouped events) instead of the flat note list. Also, need to add back `response_onset_offset` to every onset so that the normalised times match the original times stored in the ground truth TSV" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "def convert_pipeline_output(event_details, response_notes):\n", + "def convert_pipeline_output(event_details, response_events_normalized):\n", " \"\"\"\n", " Convert event_details from compare_performance_ED into a list of\n", " dicts with keys: onset, pitch, label.\n", @@ -353,12 +423,15 @@ "\n", " Args:\n", " event_details: list of dicts from compare_performance_ED\n", - " response_notes: list of note dicts (the flat, normalised note list\n", - " that was passed into compare_performance_ED as\n", - " responseMIDI[\"notes\"], after normalize_start_times)\n", + " response_events_normalized: list of EVENT dicts produced by\n", + " group_notes_into_events on the normalised\n", + " response notes — NOT the flat note list.\n", "\n", " Returns:\n", - " list of dicts with keys: onset, pitch, label\n", + " list of dicts with keys:\n", + " onset -> float (normalised seconds) or None for deletions\n", + " pitch -> int (MIDI pitch) or None for deletions\n", + " label -> 'paired', 'insertion', or 'deletion'\n", " \"\"\"\n", " my_pairs = []\n", "\n", @@ -366,127 +439,113 @@ " op = event[\"operation_type\"]\n", "\n", " if event[\"event_type\"] == \"note\":\n", - " if op == \"match\" or op == \"replacement\":\n", - " note = response_notes[event[\"response_index\"] - 1]\n", + " if op in (\"match\", \"replacement\"):\n", + " # Look up onset and pitch from the grouped event list.\n", + " ev = response_events_normalized[event[\"response_index\"] - 1]\n", " my_pairs.append({\n", - " \"onset\": note[\"start\"],\n", - " \"pitch\": note[\"pitch\"],\n", + " \"onset\": ev[\"event_start\"],\n", + " \"pitch\": ev[\"notes\"][0][\"pitch\"],\n", " \"label\": \"paired\",\n", " })\n", " elif op == \"missing\":\n", - " my_pairs.append({\n", - " \"onset\": None,\n", - " \"pitch\": None,\n", - " \"label\": \"deletion\",\n", - " })\n", + " my_pairs.append({\"onset\": None, \"pitch\": None, \"label\": \"deletion\"})\n", " elif op == \"extra\":\n", - " note = response_notes[event[\"response_index\"] - 1]\n", + " ev = response_events_normalized[event[\"response_index\"] - 1]\n", " my_pairs.append({\n", - " \"onset\": note[\"start\"],\n", - " \"pitch\": note[\"pitch\"],\n", + " \"onset\": ev[\"event_start\"],\n", + " \"pitch\": ev[\"notes\"][0][\"pitch\"],\n", " \"label\": \"insertion\",\n", " })\n", "\n", " elif event[\"event_type\"] == \"chord\":\n", " if op == \"missing\":\n", " # Entire chord missing: one deletion per note in the ref chord\n", - " # We know how many notes because correct + missing = all ref notes\n", - " num_ref_notes = len(event[\"correct_pitches\"] or []) + \\\n", - " len(event[\"missing_pitches\"] or [])\n", - " for _ in range(num_ref_notes):\n", - " my_pairs.append({\n", - " \"onset\": None,\n", - " \"pitch\": None,\n", - " \"label\": \"deletion\",\n", - " })\n", + " # correct_pitches + missing_pitches = all ref notes in this chord\n", + " num_ref_notes = (\n", + " len(event[\"correct_pitches\"] or [])\n", + " + len(event[\"missing_pitches\"] or [])\n", + " )\n", + " for i in range(num_ref_notes):\n", + " my_pairs.append({\"onset\": None, \"pitch\": None, \"label\": \"deletion\"})\n", " elif op == \"extra\":\n", - " # Entire chord is extra: one insertion per note played\n", - " res_idx_0 = event[\"response_index\"] - 1\n", - " note = response_notes[res_idx_0]\n", - " my_pairs.append({\n", - " \"onset\": note[\"start\"],\n", - " \"pitch\": note[\"pitch\"],\n", - " \"label\": \"insertion\",\n", - " })\n", - " else:\n", - " # Aligned chord pair (match or replacement)\n", - " # correct_pitches -> match for each\n", - " # missing_pitches -> deletion for each\n", - " # extra_pitches -> insertion for each\n", - " res_idx_0 = event[\"response_index\"] - 1\n", - " note = response_notes[res_idx_0]\n", - " onset = note[\"start\"]\n", - " for _ in (event[\"correct_pitches\"] or []):\n", - " my_pairs.append({\n", - " \"onset\": onset,\n", - " \"pitch\": note[\"pitch\"],\n", - " \"label\": \"paired\",\n", - " })\n", - " for _ in (event[\"missing_pitches\"] or []):\n", - " my_pairs.append({\n", - " \"onset\": None,\n", - " \"pitch\": None,\n", - " \"label\": \"deletion\",\n", - " })\n", - " for _ in (event[\"extra_pitches\"] or []):\n", + " # Every note in the extra response chord counts as an insertion.\n", + " ev = response_events_normalized[event[\"response_index\"] - 1]\n", + " for note in ev[\"notes\"]:\n", " my_pairs.append({\n", - " \"onset\": onset,\n", + " \"onset\": ev[\"event_start\"],\n", " \"pitch\": note[\"pitch\"],\n", " \"label\": \"insertion\",\n", " })\n", - "\n", - " return my_pairs\n", - "\n", - "\n", - "def compute_metrics(ground_truth, my_pairs):\n", + " else:\n", + " # Aligned chord pair (match or replacement).\n", + " ev = response_events_normalized[event[\"response_index\"] - 1]\n", + " onset = ev[\"event_start\"]\n", + "\n", + " # Build a pitch-class -> MIDI-pitch lookup from the response chord.\n", + " # This fixes the bug where the original code always used the first\n", + " # note's pitch regardless of which pitch class was being expanded.\n", + " pc_to_midi = {}\n", + " for note in ev[\"notes\"]:\n", + " pc = note[\"pitch\"] % 12\n", + " pc_to_midi[pc] = note[\"pitch\"]\n", + "\n", + " # Correctly matched pitch classes -> paired\n", + " for pc in (event[\"correct_pitches\"] or []):\n", + " midi_pitch = pc_to_midi.get(pc, pc) # fallback if lookup fails\n", + " my_pairs.append({\"onset\": onset, \"pitch\": midi_pitch, \"label\": \"paired\"})\n", + "\n", + " # Reference pitch classes not played -> deletion\n", + " for pc in (event[\"missing_pitches\"] or []):\n", + " my_pairs.append({\"onset\": None, \"pitch\": None, \"label\": \"deletion\"})\n", + "\n", + " # Response pitch classes with no reference counterpart -> insertion\n", + " for pc in (event[\"extra_pitches\"] or []):\n", + " midi_pitch = pc_to_midi.get(pc, pc)\n", + " my_pairs.append({\"onset\": onset, \"pitch\": midi_pitch, \"label\": \"insertion\"})\n", + "\n", + " return my_pairs" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def compute_metrics(ground_truth, my_pairs, onset_offset=0.0):\n", " \"\"\"\n", - " Compute the metrics in a fairer way for pipelines that align \n", - " by pitch without considering onset time.\n", + " Compare pipeline alignment against ASAP ground truth.\n", "\n", - " For my match, count as a True Positive if:\n", - " - its pitch appears in the GT match set for that pitch,\n", - " and the GT has at least one unaccounted match at that pitch.\n", - "\n", - " This uses a multiset comparison: if GT has 3 matches at pitch 60\n", - " and we predict 3 matches at pitch 60, all 3 are TPs. If we predict\n", - " 4, the extra one is an FP.\n", + " A predicted match is a True Positive only if both pitch and\n", + " onset time (rounded to 1 decimal place) agree with the GT.\n", "\n", " Args:\n", " ground_truth: list of dicts from load_ground_truth()\n", " my_pairs: list of dicts from convert_pipeline_output()\n", + " onset_offset: float, the value that was subtracted from all response\n", + " onsets by normalize_start_times. Added back here so that\n", + " our normalised onsets match the absolute GT onsets.\n", "\n", " Returns:\n", - " dict with precision, recall, f1, tp, fp, fn\n", + " dict with keys: precision, recall, f1, tp, fp, fn\n", " \"\"\"\n", - " # Build a pitch count multiset for GT matches\n", - " gt_pitch_counts = {}\n", + " # Build a set of GT matched (pitch, onset) pairs\n", + " gt_matches = set()\n", " for row in ground_truth:\n", " if row[\"label\"] == \"paired\":\n", - " p = int(row[\"pitch\"])\n", - " if p not in gt_pitch_counts:\n", - " gt_pitch_counts[p] = 0\n", - " gt_pitch_counts[p] += 1\n", + " gt_matches.add((int(row[\"pitch\"]), round(float(row[\"onset\"]), 1)))\n", "\n", - " # Build a pitch count multiset for my matches\n", - " my_pitch_counts = {}\n", + " # Build a set of our matched (pitch, onset) pairs.\n", + " # Add the onset_offset back so our normalised times match GT absolute times.\n", + " my_matches = set()\n", " for row in my_pairs:\n", " if row[\"label\"] == \"paired\" and row[\"onset\"] is not None:\n", - " p = int(row[\"pitch\"])\n", - " if p not in my_pitch_counts:\n", - " my_pitch_counts[p] = 0\n", - " my_pitch_counts[p] += 1\n", - "\n", - " tp = 0\n", - " fp = 0\n", - " fn = 0\n", - "\n", - " all_pitches = set(gt_pitch_counts.keys()).union(set(my_pitch_counts.keys()))\n", - " for pitch in all_pitches:\n", - " gt_count = gt_pitch_counts.get(pitch, 0) # Count of GT matches at this pitch\n", - " my_count = my_pitch_counts.get(pitch, 0) # Count of my matches at this pitch\n", - " tp += min(gt_count, my_count) \n", - " fp += max(0, my_count - gt_count) \n", - " fn += max(0, gt_count - my_count)\n", + " absolute_onset = row[\"onset\"] + onset_offset\n", + " my_matches.add((int(row[\"pitch\"]), round(float(absolute_onset), 1)))\n", + "\n", + " tp = len(my_matches & gt_matches)\n", + " fp = len(my_matches - gt_matches)\n", + " fn = len(gt_matches - my_matches)\n", "\n", " if tp + fp > 0:\n", " precision = tp / (tp + fp)\n", @@ -508,19 +567,35 @@ " \"recall\": round(recall, 4),\n", " \"f1\": round(f1, 4),\n", " \"tp\": tp, \"fp\": fp, \"fn\": fn,\n", - " }\n", - "\n", - "\n", - "def evaluate_one_piece(asap_path, metadata_row, event_details, response_notes_original):\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evaluate against ground truth:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_one_piece(asap_path, metadata_row, event_details,\n", + " response_events_normalized, response_onset_offset):\n", " \"\"\"\n", - " Run ground truth comparison for one score/performance pair.\n", + " Run ground truth comparison for each score/performance pair.\n", "\n", " Args:\n", " asap_path: str, path to ASAP repo root\n", " metadata_row: dict, one row from metadata.csv\n", " event_details: list from compare_performance_ED result\n", - " response_notes_original: list of note dicts with original\n", - " (absolute) onset times — NOT normalised\n", + " response_events_normalized: list of event dicts (normalized + grouped)\n", + " response_onset_offset: float, onset of the first response note\n", + " before normalization — added back to convert\n", + " normalised onsets to absolute times for GT comparison.\n", "\n", " Returns:\n", " metrics dict, or None if the TSV file is not found\n", @@ -533,14 +608,15 @@ " return None\n", "\n", " ground_truth = load_ground_truth(tsv_path)\n", - " my_pairs = convert_pipeline_output(event_details, response_notes_original)\n", + " my_pairs = convert_pipeline_output(event_details, response_events_normalized)\n", + " metrics = compute_metrics(ground_truth, my_pairs, onset_offset=response_onset_offset)\n", "\n", - " return compute_metrics(ground_truth, my_pairs)" + " return metrics" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -577,52 +653,52 @@ " \n", " 0\n", " Bach (Fugue_bwv_846)\n", - " 0.9092\n", - " 0.8415\n", + " 0.9444\n", " 0.8740\n", - " 621\n", - " 62\n", - " 117\n", + " 0.9078\n", + " 645\n", + " 38\n", + " 93\n", " \n", " \n", " 1\n", " Bach (Fugue_bwv_848)\n", - " 0.9293\n", - " 0.9004\n", - " 0.9146\n", - " 1275\n", - " 97\n", - " 141\n", + " 0.9614\n", + " 0.9322\n", + " 0.9465\n", + " 1319\n", + " 53\n", + " 96\n", " \n", " \n", " 2\n", " Bach (Fugue_bwv_848)\n", - " 0.9288\n", - " 0.8946\n", - " 0.9114\n", - " 1265\n", - " 97\n", - " 149\n", + " 0.9376\n", + " 0.9031\n", + " 0.9200\n", + " 1277\n", + " 85\n", + " 137\n", " \n", " \n", " 3\n", " Bach (Fugue_bwv_848)\n", - " 0.9420\n", - " 0.9100\n", - " 0.9257\n", - " 1284\n", - " 79\n", - " 127\n", + " 0.9494\n", + " 0.9171\n", + " 0.9329\n", + " 1294\n", + " 69\n", + " 117\n", " \n", " \n", " 4\n", " Bach (Fugue_bwv_848)\n", - " 0.9435\n", - " 0.9147\n", - " 0.9289\n", - " 1287\n", - " 77\n", - " 120\n", + " 0.9370\n", + " 0.9083\n", + " 0.9224\n", + " 1278\n", + " 86\n", + " 129\n", " \n", " \n", " 5\n", @@ -637,59 +713,59 @@ " \n", " 6\n", " Bach (Fugue_bwv_848)\n", - " 0.9184\n", - " 0.8905\n", - " 0.9043\n", - " 1261\n", - " 112\n", - " 155\n", + " 0.9585\n", + " 0.9294\n", + " 0.9437\n", + " 1316\n", + " 57\n", + " 100\n", " \n", " \n", " 7\n", " Bach (Fugue_bwv_848)\n", - " 0.9394\n", - " 0.9014\n", - " 0.9200\n", - " 1271\n", - " 82\n", - " 139\n", + " 0.9453\n", + " 0.9071\n", + " 0.9258\n", + " 1279\n", + " 74\n", + " 131\n", " \n", " \n", " 8\n", " Bach (Fugue_bwv_848)\n", - " 0.9381\n", - " 0.9096\n", - " 0.9236\n", - " 1288\n", - " 85\n", - " 128\n", + " 0.9563\n", + " 0.9273\n", + " 0.9416\n", + " 1313\n", + " 60\n", + " 103\n", " \n", " \n", " 9\n", " Bach (Fugue_bwv_848)\n", - " 0.9366\n", - " 0.9082\n", - " 0.9222\n", - " 1286\n", - " 87\n", - " 130\n", + " 0.9541\n", + " 0.9251\n", + " 0.9394\n", + " 1310\n", + " 63\n", + " 106\n", " \n", " \n", "\n", "" ], "text/plain": [ - " Piece Precision Recall F1 TP FP FN\n", - "0 Bach (Fugue_bwv_846) 0.9092 0.8415 0.8740 621 62 117\n", - "1 Bach (Fugue_bwv_848) 0.9293 0.9004 0.9146 1275 97 141\n", - "2 Bach (Fugue_bwv_848) 0.9288 0.8946 0.9114 1265 97 149\n", - "3 Bach (Fugue_bwv_848) 0.9420 0.9100 0.9257 1284 79 127\n", - "4 Bach (Fugue_bwv_848) 0.9435 0.9147 0.9289 1287 77 120\n", - "5 Bach (Fugue_bwv_848) 0.9470 0.9190 0.9328 1304 73 115\n", - "6 Bach (Fugue_bwv_848) 0.9184 0.8905 0.9043 1261 112 155\n", - "7 Bach (Fugue_bwv_848) 0.9394 0.9014 0.9200 1271 82 139\n", - "8 Bach (Fugue_bwv_848) 0.9381 0.9096 0.9236 1288 85 128\n", - "9 Bach (Fugue_bwv_848) 0.9366 0.9082 0.9222 1286 87 130" + " Piece Precision Recall F1 TP FP FN\n", + "0 Bach (Fugue_bwv_846) 0.9444 0.8740 0.9078 645 38 93\n", + "1 Bach (Fugue_bwv_848) 0.9614 0.9322 0.9465 1319 53 96\n", + "2 Bach (Fugue_bwv_848) 0.9376 0.9031 0.9200 1277 85 137\n", + "3 Bach (Fugue_bwv_848) 0.9494 0.9171 0.9329 1294 69 117\n", + "4 Bach (Fugue_bwv_848) 0.9370 0.9083 0.9224 1278 86 129\n", + "5 Bach (Fugue_bwv_848) 0.9470 0.9190 0.9328 1304 73 115\n", + "6 Bach (Fugue_bwv_848) 0.9585 0.9294 0.9437 1316 57 100\n", + "7 Bach (Fugue_bwv_848) 0.9453 0.9071 0.9258 1279 74 131\n", + "8 Bach (Fugue_bwv_848) 0.9563 0.9273 0.9416 1313 60 103\n", + "9 Bach (Fugue_bwv_848) 0.9541 0.9251 0.9394 1310 63 106" ] }, "metadata": {}, @@ -699,9 +775,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Mean Precision: 0.9238\n", - "Mean Recall : 0.8876\n", - "Mean F1 : 0.9051\n" + "Mean Precision: 0.941\n", + "Mean Recall : 0.9044\n", + "Mean F1 : 0.9221\n" ] } ], @@ -714,7 +790,8 @@ " ASAP_PATH,\n", " sample[\"metadata_row\"],\n", " result[\"event_details\"],\n", - " sample[\"response\"][\"notes\"], \n", + " result[\"response_events_normalized\"],\n", + " result[\"response_onset_offset\"],\n", " )\n", " if metrics is not None:\n", " eval_rows.append({\n", From 253f08b0855dacf120dbdc72c1797d8b03845707 Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Sat, 4 Jul 2026 21:13:35 +0100 Subject: [PATCH 6/7] fix the evaluation metrics design part, polish notebook --- notebooks/Alignment_on_ASAPdataset.ipynb | 234 +++++++++++++++++------ 1 file changed, 175 insertions(+), 59 deletions(-) diff --git a/notebooks/Alignment_on_ASAPdataset.ipynb b/notebooks/Alignment_on_ASAPdataset.ipynb index 09b76cb..d7b5b34 100644 --- a/notebooks/Alignment_on_ASAPdataset.ipynb +++ b/notebooks/Alignment_on_ASAPdataset.ipynb @@ -35,7 +35,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Download the ASAP Dataset" + "## Download the (n)ASAP Dataset" ] }, { @@ -68,21 +68,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Load the ground truth\n", + "## Load the Ground Truth (GT)\n", "\n", - "The `note_alignment.tsv` file in each performance contains the official note-level alignment annotations. Each row in the TSV represents one note pair. \n", - "\n", - "The `label` has three possible values:\n", - "- `match`: a score note (reference) and a performance note (response) were successfully aligned\n", - "- `deletion`: the score note was not played in the performance\n", - "- `insertion`: an extra note was played that has no counterpart in the score\n", + "The `note_alignment.tsv` file in each performance contains the official note-level alignment annotations. Each row represents one note, with columns: `xml_id`, `midi_id`, `track`, `channel`, `pitch`, `onset`.\n", + "- If `midi_id == \"deletion\"`: the score note was not played → label `deletion`\n", + "- If `xml_id == \"insertion\"`: an extra note was played → label `insertion`\n", + "- Otherwise: a matched pair → label `paired`\n", "\n", "For `match` and `insertion` rows, the TSV provides `onset` (onset time in seconds) and `pitch` (MIDI pitch) for the performance note." ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -92,11 +90,6 @@ " \"\"\"\n", " Read a note_alignments TSV file from the CPJKU ASAP dataset.\n", "\n", - " The TSV columns are: xml_id, midi_id, track, channel, pitch, onset\n", - " - if xml_id == 'insertion': extra note played with no score counterpart\n", - " - if midi_id == 'deletion': score note not played in the performance\n", - " - otherwise: matched pair, onset and pitch refer to the performance note\n", - " \n", " Args:\n", " tsv_path: str, path to the note_alignments.tsv file\n", "\n", @@ -109,6 +102,7 @@ " rows = []\n", " with open(tsv_path, \"r\", encoding=\"utf-8\") as f:\n", " reader = csv.DictReader(f, delimiter=\"\\t\")\n", + " \n", " for row in reader:\n", " xml_id = row[\"xml_id\"].strip()\n", " midi_id = row[\"midi_id\"].strip()\n", @@ -134,7 +128,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Load Score (reference)/Performance (response) pairs from ASAP" + "## Load Score (reference)/Performance (response) pairs" ] }, { @@ -146,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -176,6 +170,7 @@ " \"start\": round(note.start, 3),\n", " \"duration\": round(note.end - note.start, 3),\n", " })\n", + " # Sort by onset time, then by pitch (ensures consistent ordering for chords)\n", " all_notes.sort(key=lambda n: (n[\"start\"], n[\"pitch\"]))\n", " return all_notes\n", "\n", @@ -199,6 +194,8 @@ " score_notes = midi_file_to_notes(ref_path)\n", " perf_notes = midi_file_to_notes(response_path)\n", "\n", + " # If either MIDI file has no notes, return None to indicate that this sample\n", + " # should be skipped (e.g., some ASAP pieces have empty score or performance).\n", " if not score_notes or not perf_notes:\n", " return None\n", "\n", @@ -215,12 +212,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For the ASAP dataset, the `metadata.csv` lists every score/performance pair in the dataset, check that both MIDI files exist on disk." + "For the (n)ASAP dataset, the `metadata.csv` lists every score/performance pair in the dataset, check that both MIDI files exist on disk." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -286,11 +283,7 @@ "\n", "samples = load_samples(ASAP_PATH, \"Bach\")\n", "\n", - "print(\"Loaded\", len(samples), \"score (reference) /performance (response) pairs.\")\n", - "for i, sample in enumerate(samples[:10]):\n", - " print(f\"{str(i + 1)}. {sample['composer']}({sample['title']}) |\"\n", - " f\" ref notes: {len(sample['reference']['notes'])} |\"\n", - " f\" response notes: {len(sample['response']['notes'])}\")" + "print(\"Loaded\", len(samples), \"score (reference) /performance (response) pairs.\")" ] }, { @@ -302,12 +295,13 @@ "Passing each score/performance pair through `compare_performance_ED`, where score MIDI is the reference and the performance MIDI is the response.\n", "\n", "**Why do we need the normalised events? - to fix the onset mismatch bug**\n", - "- Inside `compare_performance_ED`, `normalize_start_times` shifts the first note to t = 0, and `group_notes_into_events` groups simultaneous notes into chords. The `response_index` values stored in `event_details` refer to positions in these grouped events, not in the original flat note list. Therefore, need to reproduce these two steps here so that the correct onset and pitch for each event during ground truth comparison can be looked up." + "- Inside `compare_performance_ED`, `normalize_start_times` shifts the first note to t = 0, and `group_notes_into_events` groups simultaneous notes into chords. The `response_index` values stored in `event_details` refer to positions in these grouped events, not in the original flat note list. Therefore, need to reproduce these two steps here so that the correct onset and pitch for each event during ground truth comparison can be looked up.\n", + "- The `response_onset_offset` (the original first-note onset before normalisation) will also be recorded, and will be added back when comparing normalised pipeline onsets against the original (absolute) times stored in the ground truth TSV." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -330,7 +324,7 @@ " \"\"\"\n", " Run compare_performance_ED on every sample and collect results.\n", "\n", - " We also store:\n", + " In addition to the pipeline result, we also store:\n", " - response_events_normalized: the grouped event list after start-time\n", " normalisation, needed to look up correct onset/pitch during evaluation.\n", " - response_onset_offset: the original first-note onset time, needed to\n", @@ -390,22 +384,37 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Compares our alignment output against the ASAP ground truth to compute standard information retrieval metrics:\n", + "**our pipeline vs GT**\n", + "\n", + "The GT has three different labels, while our pipeline has four operations:\n", + "\n", + "| Pipeline operation | GT label | Explanation |\n", + "|---|---|---|\n", + "| `match` | `paired` | Same pitch — GT still calls this `paired` |\n", + "| `replacement` | `paired` | Different pitch — GT still calls this `paired`, because the score note *was* played at approximately the right time |\n", + "| `missing` | `deletion` | Score note not found in response |\n", + "| `extra` | `insertion` | Response note has no score counterpart |\n", "\n", - "- **Precision**: TP / (TP + FP) \n", - " - i.e. of all note pairs we called a match, how many the ground truth also call a match\n", - "- **Recall**: TP / (TP + FN)\n", - " - i.e of all note pairs the ground truth called a match, how many we find\n", - "- **F1**: harmonic mean of precision and recall\n", + "**Evaluation metrics design**\n", "\n", - "A predicted match is a True Positive only if both pitch and onset time (rounded to 1 decimal place) agree with the ground truth.\n", + "GT does not distinguish `match` from `replacement` because the alignment methods in GT are based on the temporal information. However, the cost function in our edit distance algorithm is based on the pitch correctness. \n", "\n", - "**fix the index problem caused by the grouping in the pipeline**: look up pitches and onsets from `response_events_normalized` (the grouped events) instead of the flat note list. Also, need to add back `response_onset_offset` to every onset so that the normalised times match the original times stored in the ground truth TSV" + "Following Peter et al. (2023, TISMIR), label-level F1 would be applied as the main metric where TP, FP, and FN are computed separately for each label type and then summed:\n", + "\n", + "| Label | Matching method | Rationale |\n", + "|---|---|---|\n", + "| `paired` | Onset key only (rounded to 1 d.p.) | GT does not check pitch correctness for alignment |\n", + "| `deletion` | Count-based: TP = min(GT count, predicted count) | No onset or pitch available for deletions |\n", + "| `insertion` | `(pitch, onset)` key | Both sides have onset and pitch info for extra notes |\n", + "\n", + "- Precision: TP / (TP + FP), i.e. of all labels the pipeline predicted, how many were correct \n", + "- Recall: TP / (TP + FN), i.e. of all GT labels, how many the pipeline found \n", + "- F1: the harmonic mean of precision and recall " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -413,17 +422,17 @@ " \"\"\"\n", " Convert event_details from compare_performance_ED into a list of\n", " dicts with keys: onset, pitch, label.\n", - " Labels follow the TSV convention:\n", - " 'paired': correctly aligned note pair (pitch matches)\n", - " 'deletion' : score note not found in response (missing)\n", - " 'insertion': response note has no score counterpart (extra)\n", + " Label mapping from pipeline operations to GT labels:\n", + " match or replacement -> 'paired'\n", + " missing -> 'deletion'\n", + " extra -> 'insertion'\n", "\n", " Chord events are expanded note-by-note using correct_pitches /\n", " missing_pitches / extra_pitches from event_level_feedback.\n", "\n", " Args:\n", " event_details: list of dicts from compare_performance_ED\n", - " response_events_normalized: list of EVENT dicts produced by\n", + " response_events_normalized: list of event dicts produced by\n", " group_notes_into_events on the normalised\n", " response notes — NOT the flat note list.\n", "\n", @@ -508,55 +517,158 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "def compute_metrics(ground_truth, my_pairs, onset_offset=0.0):\n", + "def count_paired_matches(ground_truth, my_pairs, onset_offset):\n", " \"\"\"\n", - " Compare pipeline alignment against ASAP ground truth.\n", + " Count TP, FP, FN for 'paired' labels.\n", "\n", - " A predicted match is a True Positive only if both pitch and\n", - " onset time (rounded to 1 decimal place) agree with the GT.\n", + " Matching key: onset only (rounded to 1 d.p.).\n", + " Pitch is NOT used here because the GT 'paired' label means the score note\n", + " was aligned to a performance note regardless of whether the pitch is correct.\n", + " Both pipeline 'match' and 'replacement' operations map to 'paired'.\n", "\n", " Args:\n", " ground_truth: list of dicts from load_ground_truth()\n", " my_pairs: list of dicts from convert_pipeline_output()\n", - " onset_offset: float, the value that was subtracted from all response\n", - " onsets by normalize_start_times. Added back here so that\n", - " our normalised onsets match the absolute GT onsets.\n", + " onset_offset: float, added back to normalised onsets before comparison\n", "\n", " Returns:\n", - " dict with keys: precision, recall, f1, tp, fp, fn\n", + " tp, fp, fn: ints\n", " \"\"\"\n", - " # Build a set of GT matched (pitch, onset) pairs\n", - " gt_matches = set()\n", + " # Collect GT 'paired' onset keys\n", + " gt_paired_set = set()\n", " for row in ground_truth:\n", " if row[\"label\"] == \"paired\":\n", - " gt_matches.add((int(row[\"pitch\"]), round(float(row[\"onset\"]), 1)))\n", + " gt_paired_set.add(round(float(row[\"onset\"]), 1))\n", "\n", - " # Build a set of our matched (pitch, onset) pairs.\n", - " # Add the onset_offset back so our normalised times match GT absolute times.\n", - " my_matches = set()\n", + " # Collect predicted 'paired' onset keys, restoring absolute time\n", + " my_paired_set = set()\n", " for row in my_pairs:\n", " if row[\"label\"] == \"paired\" and row[\"onset\"] is not None:\n", " absolute_onset = row[\"onset\"] + onset_offset\n", - " my_matches.add((int(row[\"pitch\"]), round(float(absolute_onset), 1)))\n", + " my_paired_set.add(round(float(absolute_onset), 1))\n", + "\n", + " tp = len(my_paired_set & gt_paired_set) # correctly predicted\n", + " fp = len(my_paired_set - gt_paired_set) # predicted but not in GT\n", + " fn = len(gt_paired_set - my_paired_set) # in GT but not predicted\n", + "\n", + " return tp, fp, fn\n", + "\n", + "\n", + "def count_deletion_matches(ground_truth, my_pairs):\n", + " \"\"\"\n", + " Count TP, FP, FN for 'deletion' labels.\n", + "\n", + " Deletions have no onset or pitch (the note was never played), so\n", + " identity-based matching is not possible. Count-based matching is used:\n", + " TP = min(GT deletion count, predicted deletion count)\n", + " FP = max(0, predicted count - GT count)\n", + " FN = max(0, GT count - predicted count)\n", + "\n", + " Args:\n", + " ground_truth: list of dicts from load_ground_truth()\n", + " my_pairs: list of dicts from convert_pipeline_output()\n", + "\n", + " Returns:\n", + " tp, fp, fn: ints\n", + " \"\"\"\n", + " gt_deletion_count = sum(1 for row in ground_truth if row[\"label\"] == \"deletion\")\n", + " my_deletion_count = sum(1 for row in my_pairs if row[\"label\"] == \"deletion\")\n", + "\n", + " tp = min(gt_deletion_count, my_deletion_count)\n", + " fp = max(0, my_deletion_count - gt_deletion_count)\n", + " fn = max(0, gt_deletion_count - my_deletion_count)\n", + "\n", + " return tp, fp, fn\n", + "\n", + "\n", + "def count_insertion_matches(ground_truth, my_pairs, onset_offset):\n", + " \"\"\"\n", + " Count TP, FP, FN for 'insertion' labels.\n", + "\n", + " Matching key: (pitch, onset) -- pitch IS included here because GT\n", + " insertion records carry a pitch value, and we want to verify that\n", + " the correct extra note was identified.\n", + "\n", + " Args:\n", + " ground_truth: list of dicts from load_ground_truth()\n", + " my_pairs: list of dicts from convert_pipeline_output()\n", + " onset_offset: float, added back to normalised onsets before comparison\n", + "\n", + " Returns:\n", + " tp, fp, fn: ints\n", + " \"\"\"\n", + " # Collect GT 'insertion' (pitch, onset) keys\n", + " gt_insertion_set = set()\n", + " for row in ground_truth:\n", + " if row[\"label\"] == \"insertion\":\n", + " key = (int(row[\"pitch\"]), round(float(row[\"onset\"]), 1))\n", + " gt_insertion_set.add(key)\n", + "\n", + " # Collect predicted 'insertion' (pitch, onset) keys, restoring absolute time\n", + " my_insertion_set = set()\n", + " for row in my_pairs:\n", + " if row[\"label\"] == \"insertion\" and row[\"onset\"] is not None:\n", + " absolute_onset = row[\"onset\"] + onset_offset\n", + " key = (int(row[\"pitch\"]), round(float(absolute_onset), 1))\n", + " my_insertion_set.add(key)\n", + "\n", + " tp = len(my_insertion_set & gt_insertion_set)\n", + " fp = len(my_insertion_set - gt_insertion_set)\n", + " fn = len(gt_insertion_set - my_insertion_set)\n", + "\n", + " return tp, fp, fn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def compute_metrics_label_level(ground_truth, my_pairs, onset_offset=0.0):\n", + " \"\"\"\n", + " Compute label-level precision, recall, and F1 following Peter et al. (2023).\n", + " Args:\n", + " ground_truth: list of dicts from load_ground_truth()\n", + " my_pairs: list of dicts from convert_pipeline_output()\n", + " onset_offset: float, the first-note onset before normalisation.\n", + " Added back to convert normalised onsets to absolute time.\n", + "\n", + " Returns:\n", + " dict with keys:\n", + " precision, recall, f1 -> floats (rounded to 4 d.p.)\n", + " tp, fp, fn -> ints (totals across all labels)\n", + " tp_paired, fp_paired, fn_paired\n", + " tp_deletion, fp_deletion, fn_deletion\n", + " tp_insertion, fp_insertion, fn_insertion\n", + " \"\"\"\n", + " # Count TP/FP/FN separately for each label type\n", + " tp_p, fp_p, fn_p = count_paired_matches(ground_truth, my_pairs, onset_offset)\n", + " tp_d, fp_d, fn_d = count_deletion_matches(ground_truth, my_pairs)\n", + " tp_i, fp_i, fn_i = count_insertion_matches(ground_truth, my_pairs, onset_offset)\n", "\n", - " tp = len(my_matches & gt_matches)\n", - " fp = len(my_matches - gt_matches)\n", - " fn = len(gt_matches - my_matches)\n", + " # Sum across all label types for overall metrics\n", + " tp = tp_p + tp_d + tp_i\n", + " fp = fp_p + fp_d + fp_i\n", + " fn = fn_p + fn_d + fn_i\n", "\n", + " # Precision: of all labels predicted, how many are correct\n", " if tp + fp > 0:\n", " precision = tp / (tp + fp)\n", " else:\n", " precision = 0.0\n", "\n", + " # Recall: of all GT labels, how many the pipeline found\n", " if tp + fn > 0:\n", " recall = tp / (tp + fn)\n", " else:\n", " recall = 0.0\n", "\n", + " # F1: harmonic mean of precision and recall\n", " if precision + recall > 0:\n", " f1 = 2 * precision * recall / (precision + recall)\n", " else:\n", @@ -567,6 +679,10 @@ " \"recall\": round(recall, 4),\n", " \"f1\": round(f1, 4),\n", " \"tp\": tp, \"fp\": fp, \"fn\": fn,\n", + " # Per-label breakdown -- useful for diagnosing which label type is weakest\n", + " \"tp_paired\": tp_p, \"fp_paired\": fp_p, \"fn_paired\": fn_p,\n", + " \"tp_deletion\": tp_d, \"fp_deletion\": fp_d, \"fn_deletion\": fn_d,\n", + " \"tp_insertion\": tp_i, \"fp_insertion\": fp_i, \"fn_insertion\": fn_i,\n", " }" ] }, From 324a2cdf1a282c1f5c2737bb4b580ac293c835b5 Mon Sep 17 00:00:00 2001 From: ada-3e212e610b Date: Sun, 5 Jul 2026 22:32:05 +0100 Subject: [PATCH 7/7] change file path --- evaluation_function/evaluation_test.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/evaluation_function/evaluation_test.py b/evaluation_function/evaluation_test.py index 0a2e10a..b7d4480 100755 --- a/evaluation_function/evaluation_test.py +++ b/evaluation_function/evaluation_test.py @@ -565,17 +565,14 @@ def test_custom_chord_onset_window_affects_grouping(self): # correct; one of a pair missing; extra double-struck note # single mid-phrase pause; pause combined with missing note # long break with no notes lost; long break with a skip - -# Load realistic scenario test cases from the JSON file. -# Path is relative to this test file, not the current working directory, -# so the tests work no matter where pytest is run from. import json import os import pytest -cwd = os.getcwd() -dir = os.path.dirname(cwd) -path = os.path.join(dir, "data", "longMIDIsequence.json") +this_file = os.path.abspath(__file__) +this_dir = os.path.dirname(this_file) # evaluation_function/ +root_dir = os.path.dirname(this_dir) # compareMusic/ +path = os.path.join(root_dir, "data", "longMIDIsequence.json") with open(path, "r") as json_file: REALISTIC_TEST_DATA = json.load(json_file)