From 1ad09d87d428b52a1027f59c15fe15018dcfde9d Mon Sep 17 00:00:00 2001 From: Fredrik Bagge Carlson Date: Fri, 5 Jun 2026 07:16:24 +0200 Subject: [PATCH] test: drop flaky joint-vs-single assertion in LPV multi-dataset test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final assertion `e1 < e2_only1` was both apples-to-oranges (joint-fit training error on d1 vs single-d1-fit held-out error on d2) and, even after fixing the names to `e2 < e2_only1`, would still be flaky: with σy = 0.005 the noise floor is ~2.5e-5, and both quantities saturate there because the true LPV system is identical across d1 and d2 and a 2-state, 22-parameter model fit to 1200 samples already recovers θ to within noise. The "joint beats single" signal does not exist in this regime — the relative ordering is just BFGS noise. The remaining assertions (shape of x0_mat, per-dataset MSE bounds) still validate that the multi-dataset path runs to completion, gives each dataset its own initial state, and explains both datasets well. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/test_lpv.jl | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/test_lpv.jl b/test/test_lpv.jl index 08a7d62d..a48f6e59 100644 --- a/test/test_lpv.jl +++ b/test/test_lpv.jl @@ -131,18 +131,6 @@ end e2 = mean(abs2, d2.y .- ControlSystemIdentification.predict(sys_joint, d2, λ2; x0 = x0_mat[:, 2])) @test e1 < 0.05 @test e2 < 0.05 - - # Joint fit must beat a single-dataset fit on the OTHER experiment, since - # the single fit overfits its own dataset's input/noise realization. - res1 = lpv_pem(d1, λ1, 2; basis, - K0 = 1e-6 .* ones(2, 1), - show_trace = false, store_trace = false, - iterations = 200, time_limit = 60) - sys_only1, x0_only1, _ = res1 - # Held-out prediction using sys_only1 starts from zero state, which is fair - # since it never saw d2. - e2_only1 = mean(abs2, d2.y .- ControlSystemIdentification.predict(sys_only1, d2, λ2)) - @test e1 < e2_only1 end @testset "LPV PEM basis-of-length-1 ≈ LTI" begin