diff --git a/ALGORITHMS.md b/ALGORITHMS.md index afff0ff..6534cd1 100644 --- a/ALGORITHMS.md +++ b/ALGORITHMS.md @@ -61,7 +61,7 @@ Grouped by family (subdirectory under `lib/src/onehz/`). File paths are relative | `illnessCusum` | `clinical/illness_cusum.dart` | Online CUSUM state machine (green/yellow/red) over RHR — "NightSignal" | Alavi et al. 2022; Mishra et al. 2020 | | `readinessLnRmssd` | `clinical/readiness_lnrmssd.dart` | ln(RMSSD) z-scored against a rolling prior-nights baseline | Plews et al. 2013 | | `cosinor` | `clinical/cosinor.dart` | Cosinor rhythmometry (MESOR/amplitude/acrophase) | Halberg & Nelson 1979 | -| `banisterTrimp` / `edwardsTrimp` | `clinical/load_trimp.dart` | Training impulse from HR-reserve | Banister 1991; Edwards 1993 | +| `banisterTrimp` / `StrainScorer.edwardsTRIMP` | `clinical/load_trimp.dart` | Training impulse from HR-reserve | Banister 1991; Edwards 1993 | | `strainScoreMetric` | `clinical/load_trimp.dart` | log-squash of TRIMP onto a 0-21 scale | — | | `trimpStrain` | `clinical/load_trimp.dart` | TRIMP → 0-100 strain, honesty-wrapped (absent without real HRmax/RHR anchors) | — | | `ctlAtlTsb` | `clinical/load_trimp.dart` | Fitness-Fatigue-Form: EWMA CTL (42d) / ATL (7d) / TSB = CTL-ATL | Banister impulse-response model | diff --git a/lib/onehz.dart b/lib/onehz.dart index 00c7a8f..b518e09 100644 --- a/lib/onehz.dart +++ b/lib/onehz.dart @@ -13,10 +13,11 @@ library onehz; // Layer: input types & math. export 'src/onehz/types.dart'; export 'src/onehz/util.dart'; +// Per-device dispatch seam: unknown family refuses, never falls back to gen4. +export 'src/onehz/device.dart'; // Layer 0: foundations. export 'src/onehz/foundations/rr_correction.dart'; -export 'src/onehz/foundations/ppg_sqi.dart'; export 'src/onehz/foundations/baseline.dart'; export 'src/onehz/foundations/ewma_baselines.dart'; export 'src/onehz/foundations/fusion.dart'; diff --git a/lib/src/onehz/clinical/hrv_freq.dart b/lib/src/onehz/clinical/hrv_freq.dart index b9ebf09..164360d 100644 --- a/lib/src/onehz/clinical/hrv_freq.dart +++ b/lib/src/onehz/clinical/hrv_freq.dart @@ -6,9 +6,16 @@ // ULF < 0.003 Hz | VLF 0.003–0.04 | LF 0.04–0.15 | HF 0.15–0.40 | total // Normalized units: nu_lf = LF/(LF+HF)·100, nu_hf = HF/(LF+HF)·100. // +// Band powers are WELCH-AVERAGED (Welch 1967): the record is split into +// segments short enough that a modest frequency grid resolves them, each +// segment's Lomb-Scargle PSD is integrated over the band on its own +// resolution-matched grid, and the per-segment powers are averaged. See +// [_welchBandPower] for why a single whole-night grid cannot work. +// // HONESTY: HF is the band most corrupted by 1 Hz timing quantization and by // artifacts — we GATE HF (and LF/HF, nu) on the artifact fraction and report -// reduced confidence. ULF needs a 24-h record; null on short reads. +// reduced confidence. Every band is null unless the record is long enough to +// RESOLVE it (ULF genuinely needs 24 h) — never a leakage-filled 0.0. import '../types.dart'; import '../util.dart'; @@ -19,6 +26,11 @@ class HrvFreq { final double? lf; final double? hf; final double? total; + + /// Which bands [total] is the sum of. Null exactly when [total] is null. A + /// band the record could not resolve is ABSENT from this list, never a 0 in + /// the sum — so "total" always says what it totalled. + final List? totalBands; final double? lfhf; final double? nuLf; final double? nuHf; @@ -29,6 +41,7 @@ class HrvFreq { this.lf, this.hf, this.total, + this.totalBands, this.lfhf, this.nuLf, this.nuHf, @@ -40,6 +53,7 @@ class HrvFreq { if (lf != null) 'lf': round6(lf!), if (hf != null) 'hf': round6(hf!), if (total != null) 'total': round6(total!), + if (totalBands != null) 'total_bands': totalBands, if (lfhf != null) 'lf_hf': round6(lfhf!), if (nuLf != null) 'nu_lf': round6(nuLf!), if (nuHf != null) 'nu_hf': round6(nuHf!), @@ -57,7 +71,7 @@ Metric hrvFreq( List nnTimesMs, { required double artifactFraction, double hfArtifactGate = 0.15, - int gridPoints = 600, + double oversample = 4.0, }) { const inputs = ['rr_cleaned', 'beat_times']; if (nnMs.length < 16 || nnTimesMs.length != nnMs.length) { @@ -79,28 +93,28 @@ Metric hrvFreq( ); } - // Frequency grid: from ~1/span up to the HF ceiling (0.4 Hz). - final loHz = (1.0 / spanSec).clamp(0.0005, 0.04); - final ls = lombScargle(tSec, nnMs, freqGrid(loHz, 0.4, gridPoints)); - if (ls == null) { + // Each band gets its own segment length and its own resolution-matched grid. + // A band whose lowest frequency the record is too short to resolve returns + // null — absent, not a leakage-filled 0.0 (ULF used to be emitted as exactly + // 0.0 for any session over 333 s, which is not a 24-h record by any reading). + final ulf = + _welchBandPower(tSec, nnMs, 0.0003, 0.003, oversample: oversample); + final vlf = _welchBandPower(tSec, nnMs, 0.003, 0.04, oversample: oversample); + final lf = _welchBandPower(tSec, nnMs, 0.04, 0.15, oversample: oversample); + final hfRaw = _welchBandPower(tSec, nnMs, 0.15, 0.40, oversample: oversample); + if (lf == null && hfRaw == null) { return const Metric.absent( tier: Tier.high, inputs_used: inputs, - note: 'spectrum undefined', + note: 'record too short to resolve any HRV band', ); } - // Only report ULF/VLF if the record is long enough to resolve them. - final ulf = spanSec >= 1.0 / 0.003 ? ls.bandPower(0, 0.003) : null; - final vlf = spanSec >= 1.0 / 0.04 ? ls.bandPower(0.003, 0.04) : null; - final lf = ls.bandPower(0.04, 0.15); - final hfRaw = ls.bandPower(0.15, 0.40); - final hfGated = artifactFraction > hfArtifactGate; final hf = hfGated ? null : hfRaw; double? lfhf, nuLf, nuHf; - if (hf != null && (lf + hf) > 0) { + if (lf != null && hf != null && (lf + hf) > 0) { lfhf = hf == 0 ? null : lf / hf; nuLf = 100.0 * lf / (lf + hf); nuHf = 100.0 * hf / (lf + hf); @@ -112,7 +126,28 @@ Metric hrvFreq( // out bit-identical), and dropping HF from the sum would republish a // different quantity under the same name. Either way it would be dishonest — // so total is WITHHELD alongside HF. - final total = hfGated ? null : (ulf ?? 0) + (vlf ?? 0) + lf + hfRaw; + // + // A band the record could not RESOLVE is a different case, and it used to be + // handled with `?? 0` — which is exactly the absence-as-zero this package + // forbids, and it is not hypothetical: ULF needs 33 333 s of span (10 cycles + // at 0.0003 Hz) and a night is ~28 700 s, so ULF resolved on precisely no + // nights and the published total was silently the ULF-less sum. Withholding + // total instead would delete a rendered number on every night forever. So it + // is published with its composition NAMED in [totalBands]: the sum is over + // those bands and no others. + final bands = { + 'ulf': ulf, + 'vlf': vlf, + 'lf': lf, + 'hf': hfRaw + }; + final resolved = [ + for (final e in bands.entries) + if (e.value != null) e.key + ]; + final total = (hfGated || lf == null || hfRaw == null) + ? null + : [for (final b in resolved) bands[b]!].reduce((a, b) => a + b); // Confidence: penalize artifacts heavily; low-band-only reads still HIGH-ish. final conf = clamp((1 - artifactFraction) * (hfGated ? 0.6 : 0.9), 0.2, 0.9); @@ -123,6 +158,7 @@ Metric hrvFreq( lf: lf, hf: hf, total: total, + totalBands: total == null ? null : resolved, lfhf: lfhf, nuLf: nuLf, nuHf: nuHf, @@ -137,3 +173,71 @@ Metric hrvFreq( : 'PRV spectrum; HF band quantization-limited at 1 Hz', ); } + +/// Welch-averaged Lomb-Scargle power in [loHz, hiHz), or null when the record +/// is too short to RESOLVE that band. +/// +/// Why not one grid over the whole night: an 8 h record's periodogram has peaks +/// ~1/28800 Hz wide, so a rectangular sum over a grid coarser than that is a +/// lucky sample of the peaks, not an integral. The shipped 600-point grid put +/// `lf_hf` at 0.095 on a synthetic whose converged value is 2.243 — a factor of +/// 20+, on the one spectral number that is charted, stored and fed to the coach. +/// The resolution-matched grid for a whole night is ~50k points and measured +/// 18 s per night on this hardware; 600 points measured 1.2 s. Neither is +/// acceptable: one is wrong, the other unshippable. +/// +/// So, Welch 1967: split the record into segments SHORT enough that a modest +/// grid resolves them, integrate each segment's PSD over the band on its own +/// resolution-matched grid, and average. Segment length is [cyclesPerSegment] +/// periods of the band's LOWEST frequency, so the band is resolved by +/// construction; grid spacing is the segment's resolution (1/segment) divided +/// by [oversample]. Cost is O(beats x gridPoints) per band and independent of +/// how many segments the record splits into — measured ~1.4 s for a full night, +/// i.e. no worse than the wrong version it replaces, with the periodogram's +/// large variance averaged down as a bonus. +/// +/// Averaging per-segment powers means the estimate excludes variance slower +/// than one segment — correct by definition for a band whose lowest frequency +/// sets the segment length, and the reason each band is segmented separately. +double? _welchBandPower( + List tSec, + List y, + double loHz, + double hiHz, { + double cyclesPerSegment = 10.0, + double oversample = 4.0, + int minPointsPerSegment = 16, +}) { + if (loHz <= 0 || hiHz <= loHz || tSec.length < minPointsPerSegment) + return null; + final span = tSec.last - tSec.first; + final segSec = cyclesPerSegment / loHz; + if (span < segSec) return null; // band not resolvable in this record + + final df = 1.0 / segSec / oversample; + final nGrid = ((hiHz - loHz) / df).ceil() + 1; + final grid = freqGrid(loHz, hiHz, nGrid); + + var sum = 0.0; + var k = 0; + final step = segSec / 2; // 50 % overlap, the Welch default + for (var start = tSec.first; start + segSec <= tSec.last; start += step) { + final end = start + segSec; + final ts = []; + final ys = []; + for (var i = 0; i < tSec.length; i++) { + if (tSec[i] < start) continue; + if (tSec[i] >= end) break; + ts.add(tSec[i]); + ys.add(y[i]); + } + if (ts.length < minPointsPerSegment) continue; + final ls = lombScargle(ts, ys, grid); + if (ls == null) continue; + final p = ls.bandPower(loHz, hiHz); + if (!p.isFinite) continue; + sum += p; + k++; + } + return k == 0 ? null : sum / k; +} diff --git a/lib/src/onehz/clinical/hrv_time.dart b/lib/src/onehz/clinical/hrv_time.dart index 897fd92..3293021 100644 --- a/lib/src/onehz/clinical/hrv_time.dart +++ b/lib/src/onehz/clinical/hrv_time.dart @@ -9,11 +9,81 @@ // HONESTY: this is PRV (pulse-rate variability), not ECG HRV. RMSSD and pNNx // are the metrics most biased by the 1 Hz beat-time quantization (successive- // difference inflation) — flagged in `note`. Lead with SDNN / SDANN. +// +// That warning used to be advice only: every RMSSD in this file shipped, at +// confidence 0.95, however much of it was beat-timing jitter. [kNnDiffAcf1Floor] +// makes it behaviour — see that constant for the measurement and the threshold. import 'dart:math' as math; import '../types.dart'; import '../util.dart'; +/// Lag-1 ACF floor for the NN successive-difference series, below which RMSSD +/// and pNN50 are REFUSED (SDNN / SDANN survive it and are the honest lead). +/// +/// Differencing a smooth tachogram leaves ACF1 near 0; differencing white noise +/// leaves exactly −0.5. So ACF1 measures, per night and from the series already +/// in hand, how much of the "variability" is beat-timing jitter rather than +/// physiology. It is deliberately the gate INSTEAD of a per-family constant +/// (`device.dart`): the sensor difference is real and large, but it reaches us +/// as something measurable, not as a label — a strap that starts reporting +/// cleaner beats is believed the night it does so, and an unknown strap is +/// judged on its own signal rather than refused for its badge. +/// +/// MEASURED over the 13-night audit corpus: gen4 −0.057..−0.324, +/// MG −0.426..−0.456, WHOOP 5 −0.428..−0.517. −0.35 keeps every gen4 night and +/// refuses every gen5/MG night. OURS, not a published threshold — there is no +/// literature constant for this, and it is calibration, so it is a knob. +const double kNnDiffAcf1Floor = -0.35; + +/// Fewest successive differences [nnDiffAcf1] will judge a series on. Below it +/// the ACF1 estimate is noisier than what it is meant to screen out, so it +/// returns null and NOTHING is gated — thin windows are already handled by the +/// beat-count term in confidence. +const int _acf1MinDiffs = 30; + +/// Lag-1 autocorrelation of the successive-difference series, pooled over +/// CONTIGUOUS runs. +/// +/// Each entry of [diffRuns] must be differences between beats that are adjacent +/// in time; a dropped run / sensor hole ends one run and starts the next, so no +/// lag-1 pair is ever formed across a seam. Null when there is too little to +/// judge or the series is constant. +double? nnDiffAcf1(List> diffRuns) { + var n = 0; + var sum = 0.0; + for (final r in diffRuns) { + for (final d in r) { + sum += d; + n++; + } + } + if (n < _acf1MinDiffs) return null; + final m = sum / n; + var cov = 0.0; + var varSum = 0.0; + for (final r in diffRuns) { + for (var i = 0; i < r.length; i++) { + final a = r[i] - m; + varSum += a * a; + if (i > 0) cov += (r[i - 1] - m) * a; + } + } + return varSum > 0 ? cov / varSum : null; +} + +/// Confidence multiplier for a measured [acf1]: 1.0 on a smooth tachogram, +/// falling linearly to 0 at [kNnDiffAcf1Floor] so confidence bottoms out +/// exactly where RMSSD is refused. 1.0 when ACF1 could not be measured. +double _acf1Quality(double? acf1) => + acf1 == null ? 1.0 : clamp(1 - acf1 / kNnDiffAcf1Floor, 0.0, 1.0); + +String _jitterNote(double acf1) => + 'rmssd_refused:acf1=${acf1.toStringAsFixed(3)} — the NN successive ' + 'differences are essentially differenced white noise (−0.5 = pure, floor ' + '$kNnDiffAcf1Floor), so RMSSD/pNN50 would measure beat-timing jitter, not ' + 'vagal tone'; + class HrvTime { final double? rmssd; // ms final double? sdnn; // ms @@ -21,6 +91,7 @@ class HrvTime { final double? sdnnIndex; // ms (24-h: mean of 5-min SDs) final double? pnn50; // % final int nBeats; + final double? diffAcf1; // lag-1 ACF of the NN successive differences const HrvTime({ this.rmssd, this.sdnn, @@ -28,6 +99,7 @@ class HrvTime { this.sdnnIndex, this.pnn50, required this.nBeats, + this.diffAcf1, }); Map toJson() => { if (rmssd != null) 'rmssd_ms': round6(rmssd!), @@ -36,15 +108,25 @@ class HrvTime { if (sdnnIndex != null) 'sdnn_index_ms': round6(sdnnIndex!), if (pnn50 != null) 'pnn50_pct': round6(pnn50!), 'n_beats': nBeats, + if (diffAcf1 != null) 'diff_acf1': round6(diffAcf1!), }; } /// Short-window time-domain HRV on a cleaned NN series (ms). /// -/// [nnMs] cleaned NN intervals. [nnTimesMs] beat times for SDANN/SDNN-index -/// segmentation (optional; if absent, SDANN/SDNN-index are null). Returns an -/// absent Metric when there are too few beats. -Metric hrvTime(List nnMs, {List? nnTimesMs}) { +/// [nnMs] cleaned NN intervals. [nnTimesMs] beat times, used both for +/// SDANN/SDNN-index segmentation and to skip successive-difference pairs that +/// straddle a dropped run (optional; without it SDANN/SDNN-index are null and +/// RMSSD/pNN50 include the seams). [artifactFraction] is the fraction of beats +/// the upstream corrector rejected (0..1), folded into confidence exactly as +/// `hrvFreq` and `irregularBeatScreen` already do. Returns an absent Metric when +/// there are too few beats; RMSSD/pNN50 alone go null when the successive +/// differences fail [kNnDiffAcf1Floor]. +Metric hrvTime( + List nnMs, { + List? nnTimesMs, + double artifactFraction = 0.0, +}) { const inputs = ['rr_cleaned']; if (nnMs.length < 2) { return const Metric.absent( @@ -54,20 +136,54 @@ Metric hrvTime(List nnMs, {List? nnTimesMs}) { ); } - // RMSSD: root mean square of successive differences. + // RMSSD / pNN50: root mean square of SUCCESSIVE differences — successive in + // TIME, not merely adjacent in the compacted list. correctRr drops multi-beat + // artifact runs while advancing its clock across them, so nn[i-1] and nn[i] + // can sit either side of a seconds-long hole; differencing straight down the + // list manufactured one large difference per dropped run. Same `keep`-mask + // treatment irregular_rhythm.dart already applies. A pair is contiguous iff + // the elapsed time between the two beat times is the interval itself. + // + // The differences are kept as contiguous RUNS (a seam ends a run) so the same + // pass feeds [nnDiffAcf1] without ever forming a lag-1 pair across a hole. + final gapAware = nnTimesMs != null && nnTimesMs.length == nnMs.length; + final runs = >[]; + var run = []; + for (var i = 1; i < nnMs.length; i++) { + if (gapAware && nnTimesMs[i] - nnTimesMs[i - 1] > nnMs[i] + 0.5) { + if (run.isNotEmpty) { + runs.add(run); + run = []; + } + continue; + } + run.add(nnMs[i] - nnMs[i - 1]); + } + if (run.isNotEmpty) runs.add(run); + var ssd = 0.0; var nn50 = 0; - for (var i = 1; i < nnMs.length; i++) { - final d = nnMs[i] - nnMs[i - 1]; - ssd += d * d; - if (d.abs() > 50) nn50++; + var pairs = 0; + for (final r in runs) { + for (final d in r) { + ssd += d * d; + if (d.abs() > 50) nn50++; + pairs++; + } } - final rmssd = math.sqrt(ssd / (nnMs.length - 1)); - final pnn50 = 100.0 * nn50 / (nnMs.length - 1); + // RMSSD and pNN50 are the two outputs made of successive differences, so they + // are the two the jitter floor refuses. SDNN/SDANN are made of the levels and + // are far less contaminated (jitter share 1–27 % against RMSSD's 11–100 % on + // the audit corpus) — they keep publishing, which is what the header has + // always advised. + final acf1 = nnDiffAcf1(runs); + final jittery = acf1 != null && acf1 < kNnDiffAcf1Floor; + final rmssd = (pairs > 0 && !jittery) ? math.sqrt(ssd / pairs) : null; + final pnn50 = (pairs > 0 && !jittery) ? 100.0 * nn50 / pairs : null; final sdnn = stddev(nnMs); double? sdann, sdnnIndex; - if (nnTimesMs != null && nnTimesMs.length == nnMs.length) { + if (gapAware) { final seg = _fiveMinSegments(nnMs, nnTimesMs); if (seg.length >= 2) { final means = [for (final s in seg) mean(s)!]; @@ -77,8 +193,21 @@ Metric hrvTime(List nnMs, {List? nnTimesMs}) { } } - // Confidence scales with beat count (ultra-short reads are less reliable). - final conf = clamp(nnMs.length / 250.0, 0.3, 0.95); // ~250 beats ≈ 5 min + // Confidence scales with beat count (ultra-short reads are less reliable), + // with the artifact fraction we were handed, and with the measured jitter + // level. It used to be beat count alone, which published 0.95 on all 13 nights + // of the audit corpus — including a 15.3 %-artifact night whose differences + // were ~pure noise. The beat-count term is capped BEFORE the quality terms + // multiply it; multiplying first let an all-night beat count (n/250 ≈ 100) + // swallow any penalty and re-clamp to 0.95 regardless. + final conf = clamp( + clamp(nnMs.length / 250.0, 0.0, 1.0) // ~250 beats ≈ 5 min + * + _acf1Quality(acf1) * + (1 - artifactFraction), + 0.3, + 0.95, + ); return Metric( value: HrvTime( rmssd: rmssd, @@ -87,12 +216,16 @@ Metric hrvTime(List nnMs, {List? nnTimesMs}) { sdnnIndex: sdnnIndex, pnn50: pnn50, nBeats: nnMs.length, + diffAcf1: acf1, ), confidence: conf, tier: Tier.high, inputs_used: inputs, - note: 'PRV not ECG-HRV; RMSSD/pNN50 are quantization-sensitive at 1 Hz ' - '— lead with SDNN/SDANN', + note: jittery + ? '${_jitterNote(acf1)}. SDNN/SDANN survive it and are the lead here. ' + 'PRV not ECG-HRV.' + : 'PRV not ECG-HRV; RMSSD/pNN50 are quantization-sensitive at 1 Hz ' + '— lead with SDNN/SDANN', ); } @@ -113,7 +246,10 @@ Metric hrvTime(List nnMs, {List? nnTimesMs}) { /// true at the window's MIDPOINT second. /// /// Returns a Metric whose value is the median-of-windows RMSSD (ms). Keeps the -/// PRV-not-ECG honesty note. Absent when there are too few usable windows. +/// PRV-not-ECG honesty note. Absent when there are too few usable windows, or +/// when the night's successive differences fail [kNnDiffAcf1Floor]. A window +/// contributes only if it holds [minBeatsPerWindow] differences between beats +/// that are ADJACENT IN TIME, not merely adjacent in the compacted NN list. Metric nocturnalRmssd( List nnMs, List nnTimesMs, { @@ -130,15 +266,23 @@ Metric nocturnalRmssd( ); } final t0 = nnTimesMs.first; - // Bucket NN values by window index, carrying each window's time span so we can - // mask it (the successive difference uses consecutive NN within the window). - final buckets = >{}; + // Bucket beat INDICES by window index, so each window keeps its beat times and + // the successive differences below can skip the ones that straddle a dropped + // run — the same seam rule `hrvTime` applies. + final buckets = >{}; for (var i = 0; i < nnMs.length; i++) { final idx = ((nnTimesMs[i] - t0) / windowMs).floor(); - (buckets[idx] ??= []).add(nnMs[i]); + (buckets[idx] ??= []).add(i); } - // Compute per-window RMSSD over the windows we keep. + // Compute per-window RMSSD over the windows we keep. Each window's difference + // series is also kept as one contiguous run for the jitter floor below. final rmssds = []; + // The jitter floor is judged over the WHOLE night, pooled across windows, not + // per window: at 5 min a window holds a few hundred differences and its ACF1 + // is noisy enough that dropping only the windows that fail keeps the ones that + // passed by luck — measured, that let WHOOP 5 publish 109-116 ms from its + // calmest-looking windows while the night pooled to −0.43/−0.51. + final runs = >[]; final indices = buckets.keys.toList()..sort(); for (final idx in indices) { if (stageMaskPerSec != null) { @@ -150,12 +294,42 @@ Metric nocturnalRmssd( } final seg = buckets[idx]!; if (seg.length < minBeatsPerWindow + 1) continue; + // Contiguous runs inside the window: a pair whose beat times are further + // apart than the interval itself sits either side of a dropped run, and + // differencing across it manufactures one large difference per hole. + final winRuns = >[]; + var run = []; + for (var k = 1; k < seg.length; k++) { + final i = seg[k], p = seg[k - 1]; + if (nnTimesMs[i] - nnTimesMs[p] > nnMs[i] + 0.5) { + if (run.isNotEmpty) { + winRuns.add(run); + run = []; + } + continue; + } + run.add(nnMs[i] - nnMs[p]); + } + if (run.isNotEmpty) winRuns.add(run); var ssd = 0.0; - for (var i = 1; i < seg.length; i++) { - final d = seg[i] - seg[i - 1]; - ssd += d * d; + var nd = 0; + for (final r in winRuns) { + for (final d in r) { + ssd += d * d; + nd++; + } } - rmssds.add(math.sqrt(ssd / (seg.length - 1))); + if (nd < minBeatsPerWindow) continue; + runs.addAll(winRuns); + rmssds.add(math.sqrt(ssd / nd)); + } + final acf1 = nnDiffAcf1(runs); + if (acf1 != null && acf1 < kNnDiffAcf1Floor) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: _jitterNote(acf1), + ); } if (rmssds.isEmpty) { return const Metric.absent( @@ -165,8 +339,13 @@ Metric nocturnalRmssd( ); } final robust = median(rmssds)!; - // Confidence scales with how many windows we could median over. - final conf = clamp(rmssds.length / 12.0, 0.3, 0.95); // ~1 h ≈ 12 windows + // Confidence scales with how many windows we could median over, and with the + // measured jitter level (see [kNnDiffAcf1Floor]). + final conf = clamp( + clamp(rmssds.length / 12.0, 0.0, 1.0) * _acf1Quality(acf1), // 12 ≈ 1 h + 0.3, + 0.95, + ); return Metric( value: robust, confidence: conf, @@ -188,6 +367,10 @@ Metric nocturnalRmssd( /// distinct from [nocturnalRmssd], which uses cleaned NN + /// median-of-windows robustness. /// +/// This is the nightly HEADLINE (→ `ln_rmssd` → readiness), so it refuses +/// rather than approximates: absent when the successive differences fail +/// [kNnDiffAcf1Floor]. +/// /// [rrMs]/[rrTsMs] are the raw RR intervals and their beat-end epoch times in /// milliseconds. [startSec]/[endSec] bound the chosen sleep session in epoch /// seconds. The implementation is one-pass over the time-sorted RR stream: @@ -229,14 +412,37 @@ Metric sleepSessionWindowedRmssd( } final rmssds = []; + final runs = >[]; // pooled jitter floor — see [nocturnalRmssd] final indices = buckets.keys.toList()..sort(); for (final idx in indices) { - final cleaned = _cleanWindowRr(buckets[idx]!); - if (cleaned.length < 2) continue; - final rmssd = _rmssdRaw(cleaned); - if (rmssd != null) rmssds.add(rmssd); + final diffRuns = [ + for (final r in _cleanWindowRuns(buckets[idx]!)) + if (r.length >= 2) [for (var i = 1; i < r.length; i++) r[i] - r[i - 1]] + ]; + var ssd = 0.0; + var nd = 0; + for (final r in diffRuns) { + for (final d in r) { + ssd += d * d; + nd++; + } + } + if (nd == 0) continue; + runs.addAll(diffRuns); + rmssds.add(math.sqrt(ssd / nd)); } + // THE HEADLINE nightly RMSSD (→ ln_rmssd → readiness). When the differences + // are noise, the honest output is no headline, not a plausible one — the + // readiness composite already treats a null HRV driver as absent. + final acf1 = nnDiffAcf1(runs); + if (acf1 != null && acf1 < kNnDiffAcf1Floor) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: _jitterNote(acf1), + ); + } if (rmssds.isEmpty) { return const Metric.absent( tier: Tier.high, @@ -246,7 +452,11 @@ Metric sleepSessionWindowedRmssd( } final meanRmssd = mean(rmssds)!; - final conf = clamp(rmssds.length / 12.0, 0.3, 0.95); + final conf = clamp( + clamp(rmssds.length / 12.0, 0.0, 1.0) * _acf1Quality(acf1), + 0.3, + 0.95, + ); return Metric( value: meanRmssd, confidence: conf, @@ -278,42 +488,59 @@ List> _fiveMinSegments(List nn, List times) { return out; } -List _cleanWindowRr(List rr) => - _rejectWindowEctopic([for (final v in rr) if (v >= 300 && v <= 2000) v]); - -List _rejectWindowEctopic(List nn) { +/// Range-filter [300, 2000] ms + Malik-style ectopic rejection against a local +/// median, returned as CONTIGUOUS RUNS of kept intervals. +/// +/// Runs, not one compacted list: differencing straight down a compacted list +/// manufactures exactly one difference per rejected beat, spanning it — the same +/// defect `hrvTime` refuses at dropped runs and `irregularBeatScreen` refuses +/// with its keep-mask. This was the last producer in the file still doing it, +/// and it is the one feeding the nightly headline. MEASURED over the 13-night +/// audit corpus: it inflated the headline by 2–13 % on gen4 (57.2 → 52.3 ms at +/// worst) and by 51–102 % on MG (87.7 → 58.2, 82.9 → 40.9, 76.9 → 40.2 ms) — +/// i.e. most of the "gen5 reads 2× gen4" gap was this, not physiology. +List> _cleanWindowRuns(List rr) { const radius = 2; const threshold = 0.20; - if (nn.length <= radius) return nn; - final kept = []; + // Range filter first, keeping each survivor's position in [rr] — BOTH filters + // break a run, so neither one's compaction can manufacture a difference. + final nn = []; + final at = []; + for (var i = 0; i < rr.length; i++) { + if (rr[i] >= 300 && rr[i] <= 2000) { + nn.add(rr[i]); + at.add(i); + } + } + final runs = >[]; + var run = []; + var lastKept = -2; for (var i = 0; i < nn.length; i++) { - final lo = math.max(0, i - radius); - final hi = math.min(nn.length - 1, i + radius); - final neighbors = []; - for (var j = lo; j <= hi; j++) { - if (j != i) neighbors.add(nn[j]); + var keep = true; + if (nn.length > radius) { + final lo = math.max(0, i - radius); + final hi = math.min(nn.length - 1, i + radius); + final neighbors = []; + for (var j = lo; j <= hi; j++) { + if (j != i) neighbors.add(nn[j]); + } + final med = neighbors.length < 2 ? null : median(neighbors); + if (med != null && med > 0) keep = (nn[i] - med).abs() / med <= threshold; } - if (neighbors.length < 2) { - kept.add(nn[i]); + if (!keep) { + if (run.isNotEmpty) { + runs.add(run); + run = []; + } continue; } - final med = median(neighbors); - if (med == null || med <= 0) { - kept.add(nn[i]); - continue; + if (run.isNotEmpty && at[i] != lastKept + 1) { + runs.add(run); + run = []; } - final deviation = (nn[i] - med).abs() / med; - if (deviation <= threshold) kept.add(nn[i]); - } - return kept; -} - -double? _rmssdRaw(List nn) { - if (nn.length < 2) return null; - var sumSq = 0.0; - for (var i = 1; i < nn.length; i++) { - final d = nn[i] - nn[i - 1]; - sumSq += d * d; + run.add(nn[i]); + lastKept = at[i]; } - return math.sqrt(sumSq / (nn.length - 1)); + if (run.isNotEmpty) runs.add(run); + return runs; } diff --git a/lib/src/onehz/clinical/illness_cusum.dart b/lib/src/onehz/clinical/illness_cusum.dart index 3b115b0..cbd53e5 100644 --- a/lib/src/onehz/clinical/illness_cusum.dart +++ b/lib/src/onehz/clinical/illness_cusum.dart @@ -1,10 +1,23 @@ -// CLINICAL TIER-1 — NightSignal / CUSUM illness flag on nightly RHR. +// CLINICAL TIER-1 — one-sided Page CUSUM illness flag on nightly RHR. +// +// WHAT THIS IS, AND WHAT IT IS NOT. This is a textbook one-sided cumulative-sum +// chart [Page, *Biometrika* 1954;41:100-115; Hawkins & Olwell, *Cumulative Sum +// Charts and Charting for Quality Improvement*, 1998] run on the standardized +// nightly RHR deviation, with a NightSignal-flavoured recovery clause bolted on. +// +// IT IS NOT NightSignal. Alavi et al., *Nat Med* 2022;28:175-184 (methods at +// PMC8240687) is a six-state FSM on ABSOLUTE bpm symbols (A < M+3, = M+3, ≥ M+4) +// against a streaming median — no standardization, no accumulator — and that +// paper benchmarks NightSignal AGAINST CuSum as a separate method. This file +// cited Alavi 2022 and Mishra 2020 and implemented neither; the citations are +// corrected rather than the algorithm because the shipped gate measures out +// fine (below). If anyone reinstates the NightSignal citation, implement the +// FSM: measured nightly-RHR MAD on real gen4 data is 2.86 bpm, so the published +// 3 bpm threshold transfers to this stream without rescaling. // -// Alavi 2022 (NightSignal) + Mishra 2020 (RHR-AD CUSUM). A deterministic finite -// state machine on the nightly resting-HR series: // * 28-day ROBUST baseline (median + MAD) over the trailing window. // * One-sided upper CUSUM accumulator on the standardized RHR deviation with -// a slack k and a decision threshold h (designed for a target ARL). +// a slack k and a decision threshold h. // * State ladder: green -> yellow (CUSUM crosses h) -> red (yellow persists // ≥ persistDays). Recovers to green when CUSUM resets to 0. // @@ -54,7 +67,13 @@ class IllnessDay { /// [dates] display labels, [rhr] nightly resting HR (bpm), same length. /// [baselineDays] trailing robust-baseline window (default 28). /// [k] CUSUM slack in z-units (reference value, default 0.5). -/// [h] CUSUM decision threshold (default 4.0 — conservative, low false-alarm). +/// [h] CUSUM decision threshold, in z-units (default 4.0). ITS FALSE-ALARM RATE +/// IS MEASURED, not asserted: replaying this FSM over 200 simulated years of +/// round(N(54, 2.56)) — σ taken from real gen4 nightly RHR — gives **7.4 red +/// nights and 4.2 yellow per stable year** at h = 4, and 2.9 red at h = 6. For +/// scale, NightSignal's own published false-positive rate is ≈30 red alerts/yr, +/// so this is roughly 4× more specific than the method it used to cite. Say the +/// number, not "conservative": red drives a medical-class notification. /// [persistDays] yellow nights required before escalating to red. /// /// Returns a per-night list. Nights before [minBaseline] valid history are @@ -72,15 +91,21 @@ List illnessCusum( }) { final n = rhr.length; final out = []; + // CALENDAR days, not rows. The caller passes only the days that produced a + // derived row, so a positional window silently stretched a "28-day baseline" + // over months after a wear gap, and a positional persistDays let an elevated + // Monday and an elevated night three weeks later read as "sustained". + final day = calendarDays(dates); var cusum = 0.0; var yellowRun = 0; var normalRun = 0; + var lastScoredDay = -1 << 20; for (var i = 0; i < n; i++) { final r = rhr[i]; // Robust baseline from the trailing window (valid nights only). - final lo = math.max(0, i - baselineDays); final window = []; - for (var j = lo; j < i; j++) { + for (var j = i - 1; j >= 0; j--) { + if (day[i] - day[j] > baselineDays) break; final v = rhr[j]; if (v != null) window.add(v); } @@ -114,11 +139,29 @@ List illnessCusum( need: degenerateBaselineNote)); continue; } + // A break in the calendar breaks every "N nights running" counter — the + // runs below mean CONSECUTIVE NIGHTS, not consecutive rows — and that + // includes the ACCUMULATOR. It is loop-external and only bleeds off via + // max(0, …) or the two-normal-nights recovery clause, so an episode's + // charge used to survive an arbitrarily long wear gap: 5 illness nights, + // 70 days off-wrist, then the first scorable night back fired yellow at + // cusum 28.55 on a night measured z = −0.12 BELOW baseline, and the gap + // guard had just zeroed normalRun so recovery could not clear it either. + // A CUSUM is evidence accumulated over consecutive observations; there are + // no observations across a gap, so there is no evidence to carry. + if (day[i] - lastScoredDay > 1) { + yellowRun = 0; + normalRun = 0; + cusum = 0; + } + lastScoredDay = day[i]; + final z = (r - med) / scale; // One-sided upper CUSUM on elevation (RHR up = potential illness). cusum = math.max(0, cusum + (z - k)); - // NightSignal-style recovery: once the RHR is back within the normal band + // Recovery clause — OURS, not from any cited paper: once the RHR is back + // within the normal band // (z below returnZ) for [recoverDays] consecutive nights, clear the // accumulator. This stops a brief spike from latching the alarm "red" for // weeks (the bare one-sided CUSUM only bleeds off at rate k) while keeping diff --git a/lib/src/onehz/clinical/irregular_rhythm.dart b/lib/src/onehz/clinical/irregular_rhythm.dart index 06353db..c4fee0b 100644 --- a/lib/src/onehz/clinical/irregular_rhythm.dart +++ b/lib/src/onehz/clinical/irregular_rhythm.dart @@ -64,7 +64,14 @@ Metric irregularBeatScreen( double maxArtifact = 0.30, }) { const inputs = ['rr_cleaned']; - final nn = [for (final v in rrMs) if (v >= 300 && v <= 2000) v]; + // The defensive [300, 2000] filter COMPACTS the series. Keep the mask too, so + // successive differences below are taken only between beats that were both + // kept AND adjacent in the input — otherwise every filtered beat manufactured + // one spurious difference spanning the gap, which counted toward pNNx and + // inflated sdsd/sd1, pushing both flag conditions toward a false "sustained + // irregularity" screen positive. + final keep = [for (final v in rrMs) v >= 300 && v <= 2000]; + final nn = [for (var i = 0; i < rrMs.length; i++) if (keep[i]) rrMs[i]]; if (nn.length < minBeats) { return const Metric.absent( tier: Tier.estimate, @@ -81,14 +88,35 @@ Metric irregularBeatScreen( ); } - // Poincaré descriptors. - final diffs = [for (var i = 1; i < nn.length; i++) nn[i] - nn[i - 1]]; - final sdsd = stddev(diffs) ?? 0.0; - final sdnn = stddev(nn) ?? 0.0; + // Poincaré descriptors — successive beats only (see [keep]). + final diffs = [ + for (var i = 1; i < rrMs.length; i++) + if (keep[i] && keep[i - 1]) rrMs[i] - rrMs[i - 1] + ]; + final sdsd = stddev(diffs); + final sdnn = stddev(nn); + if (sdsd == null || sdnn == null) { + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'no successive clean beats to build a Poincare plot from', + ); + } final sd1 = sdsd / math.sqrt2; final v = 2 * sdnn * sdnn - sd1 * sd1; final sd2 = v > 0 ? math.sqrt(v) : 0.0; - final ratio = sd2 > 0 ? sd1 / sd2 : 0.0; + if (sd2 <= 0) { + // SD1/SD2 is undefined without long-term variability to divide by; emitting + // ratio 0.0 with sd1 = sd2 = 0 published "perfectly regular" as a + // measurement of a degenerate series. + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'no long-term variability (SD2 = 0) — the SD1/SD2 ratio is ' + 'undefined, not "perfectly regular"', + ); + } + final ratio = sd1 / sd2; // pNNx — irregularly-irregular fraction. var over = 0; @@ -98,8 +126,11 @@ Metric irregularBeatScreen( final pnnPct = diffs.isEmpty ? 0.0 : 100.0 * over / diffs.length; final flag = ratio >= sd1sd2Flag && pnnPct >= pnnFlagPct; - // Confidence scales with beat count; ~5000 beats ≈ a full strong night. - final conf = clamp(nn.length / 5000.0, 0.3, 0.9); + // Confidence scales with beat count (~5000 beats ≈ a full strong night) AND + // with the artifact fraction we were handed — it used to ignore it entirely, + // so a barely-passing 29 %-artifact night published at the same confidence as + // a clean one. + final conf = clamp(nn.length / 5000.0 * (1 - artifactFraction), 0.2, 0.9); return Metric( value: IrregularRhythm( sd1: sd1, diff --git a/lib/src/onehz/clinical/load_trimp.dart b/lib/src/onehz/clinical/load_trimp.dart index 0509e9e..726e672 100644 --- a/lib/src/onehz/clinical/load_trimp.dart +++ b/lib/src/onehz/clinical/load_trimp.dart @@ -1,9 +1,20 @@ // CLINICAL TIER-1 — training load: TRIMP + CTL/ATL/TSB. // -// Edwards 1993 zone-sum TRIMP and Banister exponential TRIMP (Morton 1990). +// Banister exponential TRIMP (Morton 1990) — the ONE TRIMP in this package. // CTL (Chronic Training Load, 42-day EWMA of daily TRIMP), ATL (Acute, 7-day // EWMA), TSB = CTL − ATL (Training Stress Balance / "form"). // +// EDWARDS IS GONE (audit MOT-06, deleted 2026-08-17). `StrainScorer` used to +// carry a second, zone-sum TRIMP applying Edwards 1993's 50/60/70/80/90 +// cut-offs to %HRR, where Edwards defines them on %HRmax. At RHR 55 / HRmax +// 187, 50 % HRR = 121 bpm = 64.7 % HRmax, so every minute banded about one zone +// low and the two published-looking 0–100 numbers disagreed by 2.8×–184× on the +// same stream (measured on whoop-4.db). Nothing shipped it — `strain_effort` +// was a permanently-null key removed from edge at v68 — so rather than re-derive +// it on the right denominator, the whole second scale is deleted. The app's +// display zones (`hr_zones.dart`) already do %HRmax correctly, with a separately +// labelled Karvonen variant. Do not reintroduce a second 0–100 strain. +// // Banister: TRIMP = Σ Δt(min) · ΔHRr · y, where ΔHRr = (HR−RHR)/(HRmax−RHR) // and y = e^(b·ΔHRr), b = 1.92 (male) / 1.67 (female). Needs measured HRmax+RHR. // @@ -85,10 +96,33 @@ const double maximalNetTrimp = 400.0; /// Curvature of the 0–21 map; higher gives more resolution at the low end. /// -/// Calibrated together with [maximalNetTrimp] so that, for a representative -/// profile (RHR 60, HRmax 187, 16 h awake), a day scores: -/// inactive → ~0 · rest + a walk → 2–4 · 45 min moderate run → 8–11 · -/// 90 min hard session → 14–17 · 5 h at 160 bpm → 21. +/// CALIBRATION ANCHORS — regenerated 2026-08-17 (audit MOT-04), and they only +/// mean anything with the profile and the convention printed next to them: +/// +/// PROFILE RHR 60, HRmax 187 (Tanaka @30), male constants, 960 waking +/// minutes, every NON-SESSION minute sitting at exactly +/// [quietWakingHrr] (= 85 bpm on this profile). +/// +/// inactive 16 h TRIMP 176.5 → 0.00 +/// + 60 min walk @105 bpm TRIMP 192.3 → 2.70 +/// + 45 min moderate run @145 bpm TRIMP 237.9 → 8.55 +/// + 90 min hard session @165 bpm TRIMP 392.9 → 16.54 +/// + 5 h @160 bpm TRIMP 806.9 → 21.00 (SATURATED) +/// 135 min wear, no activity TRIMP 24.8 → 0.00 +/// +/// The old table's "5 h at 160 bpm → 21" was loose, not wrong: with the +/// baseline subtraction included the scale SATURATES AT ≈3.25 h at 160 bpm +/// (195 min) and ≈4.25 h at 150 bpm, so everything above that is one number. +/// +/// AND THE ANCHORS ARE NOT WHAT REAL DAYS SCORE. The convention above puts +/// quiet waking at exactly 0.20 HRR; this user's measured wake minutes sit at +/// p50 0.274 / p75 0.332 HRR (whoop-4.db, 9 full-wear days, RHR 55 / HRmax +/// 187). At that real quiet level a day containing only 2–6 minutes above 50 % +/// HRR scores 6.93 / 11.17 / 11.38 / 11.97 / 12.14 — i.e. a NOTHING-DAY lands +/// in the band this table calls "90 min hard session". That gap is +/// [quietWakingHrr] being too low for this user, not the map: it is audit +/// MOT-03, it is not fixed here, and every number in this table moves when it +/// is. Regenerate the table with the constant, never separately. const double strainCurvature = 15.0; /// The TRIMP that [wakeMinutes] of ordinary waking accrues on its own. @@ -96,6 +130,20 @@ const double strainCurvature = 15.0; /// Scales with the wake window ACTUALLY observed, so a partial-wear day is not /// charged a full day's overhead (a 2 h inactive wear window would otherwise /// come out negative and clamp, while a 16 h one read as real effort). +/// +/// WHY THIS IS STILL SUBTRACTIVE (audit MOT-05, measured 2026-08-17 and NOT +/// adopted). MOT-05 asks for the domain bound to move into the accumulator — +/// sum `x·y(x)` only over minutes above [quietWakingHrr] and drop this +/// subtraction — on the argument that a gated sum depends on data rather than +/// on a fourth-decimal constant. Replayed on all three real corpora at the +/// SHIPPED constant it is strictly worse, because 0.20 HRR is below where quiet +/// waking actually sits: gen4's five nothing-days go 11.2/11.4/12.0/12.1/13.1 → +/// 17.6/17.7/17.8/18.2/21.0, and MG's three genuinely quiet days go 0.00 → +/// 10.72/12.72/0.06. Crediting a quiet minute in FULL the moment it clears the +/// gate is what does it. (Σ(x−Q)·y(x) over the same minutes tracks the current +/// form within ~1.5 points on gen4 but still breaks MG's honest zeros: 0.00 → +/// 4.05/4.83.) The gated form is only defensible once [quietWakingHrr] is this +/// user's own quiet level — MOT-03 — so it is blocked on that, not on taste. double baselineTrimp(double wakeMinutes, {bool female = false}) => wakeMinutes * quietWakingHrr * @@ -151,47 +199,25 @@ Metric strainScoreMetric( ); } -/// Edwards zone-sum TRIMP. [zoneMinutes] minutes in each of the 5 HR zones -/// (50–60/60–70/70–80/80–90/90–100 %HRmax); weights 1..5. -Metric edwardsTrimp(List zoneMinutes) { - const inputs = ['zone_minutes']; - if (zoneMinutes.length != 5) { - return const Metric.absent( - tier: Tier.estimate, - inputs_used: inputs, - note: 'Edwards TRIMP needs 5 zone minutes', - ); - } - var t = 0.0; - for (var i = 0; i < 5; i++) { - t += zoneMinutes[i] * (i + 1); - } - return Metric( - value: t, - confidence: 0.6, - tier: Tier.estimate, - inputs_used: inputs, - note: 'Edwards zone-sum TRIMP', - ); -} +// Two other TRIMPs used to sit here: a top-level `edwardsTrimp` binning minutes +// on %HRmax, and `StrainScorer`'s zone sum binning the SAME cut-offs on %HRR. +// Both are deleted (audit MOT-06) — see the file header. Banister is the one +// TRIMP. // ════════════════════════════════════════════════════════════════════════════ -// StrainScorer — Edwards/Banister TRIMP → 0–100 strain ("Effort"). +// StrainScorer — Banister TRIMP → 0–100 strain ("Effort"). // Implementation of published exercise-physiology methods. // // 1. Heart-Rate Reserve (Karvonen): HRR = HRmax − RHR. // 2. Per-sample intensity %HRR = (HR − RHR) / HRR × 100, clamped 0..100. // 3. TRIMP over the window (each sample carries its OWN duration, measured // from the real timestamps and capped at the stream's median cadence so a -// gap is never counted as effort): -// a. Edwards 5-zone (default): sample contributes its zone weight (1..5 at -// 50/60/70/80/90 %HRR cut-offs) × that sample's duration (min). -// b. Banister exponential: sample contributes dur × x × y(x), with -// y = 0.64·e^(1.92x) (men) / 0.86·e^(1.67x) (women). -// 4. strain = 100 × ln(TRIMP + 1) / ln(D), D = 7201 (TRIMP 7200 ≈ max). +// gap is never counted as effort): sample contributes dur × x × y(x), with +// y = 0.64·e^(1.92x) (men) / 0.86·e^(1.67x) (women). +// 4. strain = 100 × ln(TRIMP + 1) / ln(D), D = 7201. // -// References: Karvonen 1957; Edwards 1993; Banister 1991 (y = 0.64·e^(1.92x) -// men / 0.86·e^(1.67x) women); Tanaka 2001 (HRmax = 208 − 0.7·age). +// References: Karvonen 1957; Banister 1991 (y = 0.64·e^(1.92x) men / +// 0.86·e^(1.67x) women); Tanaka 2001 (HRmax = 208 − 0.7·age). // // NOTE (steps/active-energy floor): strain is PURELY HR-derived // (Edwards/Banister TRIMP → log map). Steps and active calories are computed as @@ -214,8 +240,13 @@ class StrainScorer { /// Top of the strain ("Effort") scale (rescaled 21.0 → 100.0). static const double maxStrain = 100.0; - /// Logarithmic-map denominator D = 7200 + 1: Edwards daily ceiling - /// (top weight 5 sustained 24h = 7200) maps to exactly maxStrain. + /// Logarithmic-map denominator D = 7200 + 1. + /// + /// It was set from the Edwards daily ceiling (top weight 5 sustained 24 h = + /// 7200), and Edwards is gone (MOT-06). Kept unchanged anyway: the Banister + /// ceiling over the same 24 h is 1440 × 1 × y(1) = 6,293, within 15 % of it, + /// and this is a DISPLAY map — rescaling it would rescore every stored day for + /// no gain in truth. static const double strainDenominator = 7201.0; /// Fallback per-sample duration (minutes) — 1 s at 1 Hz. @@ -251,22 +282,17 @@ class StrainScorer { (female ? banisterScaleWomen : banisterScaleMen) * math.exp((female ? banisterBWomen : banisterBMen) * x); - /// Edwards zone cut-offs as (%HRR threshold, weight), highest-first. - static const List> edwardsZones = [ - [90.0, 5], - [80.0, 4], - [70.0, 3], - [60.0, 2], - [50.0, 1], - ]; - // ── HRmax helpers ─────────────────────────────────────────────────────────── /// Tanaka (2001): HRmax = 208 − 0.7 × age. static double tanakaHRmax(double age) => 208.0 - 0.7 * age; - /// Classic 220 − age. Last-resort fallback only. - static int defaultMaxHR([int age = defaultAge]) => 220 - age; + // `defaultMaxHR([age]) => 220 - age` used to sit here as StrainScorer's + // "last-resort fallback". It had exactly one caller — `strain` — and its only + // job there was to keep producing a number when the person's ceiling was + // unknown (MOT-11). Deleted with that fallback. `AutoWorkoutDetector` keeps + // its own 190 for the "did you work out?" prompt, which is not a published + // number. /// Linear-interpolated percentile of an ALREADY-SORTED sequence (numpy-style). static double _percentileSorted(List sortedValues, double pct) { @@ -296,7 +322,7 @@ class StrainScorer { return (0.0, 'unknown'); } - // ── Karvonen %HRR and Edwards zone weight ────────────────────────────────── + // ── Karvonen %HRR ────────────────────────────────────────────────────────── /// Karvonen %HRR, clamped [0, 100]. static double pctHRR(double bpm, double restingHR, double hrReserve) { @@ -306,15 +332,6 @@ class StrainScorer { return pct; } - /// Edwards 5-zone weight (0–5) from %HRR (unclamped). - static int zoneWeight(double bpm, double restingHR, double hrReserve) { - final pct = (bpm - restingHR) / hrReserve * 100.0; - for (final z in edwardsZones) { - if (pct >= z[0]) return z[1].toInt(); - } - return 0; - } - // ── TRIMP accumulation ────────────────────────────────────────────────────── /// Median inter-sample interval (seconds) of a time-ordered stream, ignoring @@ -358,19 +375,6 @@ class StrainScorer { return out; } - /// Edwards 5-zone TRIMP: Σ zoneWeight(sample) × that sample's duration (min). - static double edwardsTRIMP(List bpm, double restingHR, - double hrReserve, List durationsMin) { - var acc = 0.0; - for (var i = 0; i < bpm.length; i++) { - final dur = i < durationsMin.length - ? durationsMin[i] - : (durationsMin.isEmpty ? fallbackSampleMin : durationsMin.last); - acc += zoneWeight(bpm[i], restingHR, hrReserve) * dur; - } - return acc; - } - /// Banister exponential TRIMP: Σ duration(min) × x × y(x), y per [banisterY]. static double banisterTRIMP(List bpm, double restingHR, double hrReserve, List durationsMin, {bool female = false}) { @@ -404,18 +408,25 @@ class StrainScorer { /// /// [bpm] per-sample HR; [tsSec] their timestamps in SECONDS (same length). /// Returns null when there isn't enough data to trust the number (fewer than - /// [minReadings] AND less than [minSpanSeconds] coverage), or when maxHR ≤ - /// restingHR (invalid HRR). [edwards] true → Edwards (default); false → Banister. + /// [minReadings] AND less than [minSpanSeconds] coverage), when [maxHR] is + /// missing, or when maxHR ≤ restingHR (invalid HRR). + /// + /// [maxHR] IS REQUIRED-AND-NULLABLE, and null means NO NUMBER (audit MOT-11). + /// It used to fall back to `defaultMaxHR()` = 190, a 220−age ceiling for a 30 + /// y/o applied to whoever's wrist arrived — a fabricated anchor inside the + /// value, caught (if at all) by the caller. Honesty belongs at the source, so + /// that helper is gone too. `AutoWorkoutDetector` keeps its own 190 for the + /// "did you work out?" gate, which is a prompt, not a published number. static double? strain( List bpm, List tsSec, { - double? maxHR, + required double? maxHR, double restingHR = defaultRestingHR, - bool edwards = true, bool female = false, double denominator = strainDenominator, }) { - final effMax = maxHR ?? defaultMaxHR().toDouble(); + if (maxHR == null) return null; + final effMax = maxHR; final bool enoughData; if (bpm.length >= minReadings) { enoughData = true; @@ -436,32 +447,24 @@ class StrainScorer { if (!enoughData || effMax <= restingHR) return null; final durations = sampleDurationsMinutes(tsSec); - final hrReserve = effMax - restingHR; - - final double trimp; - if (edwards) { - trimp = edwardsTRIMP(bpm, restingHR, hrReserve, durations); - } else { - trimp = banisterTRIMP(bpm, restingHR, hrReserve, durations, - female: female); - } + final trimp = banisterTRIMP(bpm, restingHR, effMax - restingHR, durations, + female: female); return trimpToStrain(trimp, denominator: denominator); } } -/// Edwards/Banister TRIMP strain ("Effort", 0–100) as a Metric, with the -/// honesty envelope: absent when the gates fail (never fabricated). +/// Banister TRIMP strain ("Effort", 0–100) as a Metric, with the honesty +/// envelope: absent when the gates fail (never fabricated). /// /// [bpm] per-sample HR (bpm). [tsSec] timestamps (s), same length. [maxHr] / /// [restingHr] the personal anchors (HRmax resolved by the caller via -/// [StrainScorer.estimateHRmax] / Tanaka). [method] 'edwards' (default) or -/// 'banister'. [sex] selects the Banister coefficient. +/// [StrainScorer.estimateHRmax] / Tanaka). [sex] selects the Banister +/// coefficient. The `method` parameter is gone with the Edwards path (MOT-06). Metric trimpStrain( List bpm, List tsSec, { double? maxHr, double? restingHr, - String method = 'edwards', Sex sex = Sex.male, }) { const inputs = ['hr_series', 'resting_hr', 'max_hr']; @@ -483,7 +486,6 @@ Metric trimpStrain( tsSec, maxHR: maxHr, restingHR: restingHr, - edwards: method != 'banister', female: sex == Sex.female, ); if (s == null) { @@ -498,7 +500,7 @@ Metric trimpStrain( confidence: 0.6, tier: Tier.estimate, inputs_used: inputs, - note: '${method == "banister" ? "Banister" : "Edwards"} TRIMP → 0–100 strain ' + note: 'Banister TRIMP → 0–100 strain ' '(100·ln(TRIMP+1)/ln(7201)); wrist-HR ESTIMATE, not clinical', ); } diff --git a/lib/src/onehz/clinical/prsa.dart b/lib/src/onehz/clinical/prsa.dart index 65cf415..5d4d460 100644 --- a/lib/src/onehz/clinical/prsa.dart +++ b/lib/src/onehz/clinical/prsa.dart @@ -6,9 +6,17 @@ // // Algorithm: // 1. Anchor selection: for DC, anchor i where RR(i) > RR(i-1) (a -// deceleration); for AC, RR(i) < RR(i-1). (Optionally bounded by a ratio -// threshold T to suppress artifacts — default unbounded as in the -// canonical DC.) +// deceleration); for AC, RR(i) < RR(i-1). Bounded by a ratio threshold T +// to suppress artifacts, default 0.05. +// +// THE 0.05 IS OURS. PMC6299532 gives the DCorg formula and L = 64 and +// states NO percentage anchor-exclusion threshold; the header used to call +// 0.05 "Bauer's own" and "the canonical setting", and no primary source for +// that could be found. DC scales strongly with it — at RMSSD 80 the capped +// estimate is 10.3 ms against 35.1 ms unbounded — so it is a calibration +// knob that governs the number, and it must not be changed on the strength +// of a comment in either direction. (The header before that one claimed +// "default unbounded"; it never was.) // 2. For each anchor, take a window of length 2L centered at the anchor. // Drop anchors too close to the series ends. // 3. Phase-aligned averaging: X(k) = mean over anchors of RR(anchor+k), @@ -16,8 +24,19 @@ // 4. Quantify with the Haar-wavelet-like contrast (s=2): // DC = [X(0) + X(1) − X(−1) − X(−2)] / 4. (ms; positive = healthier) // -// Risk tiers (Bauer 2006, post-MI): DC ≤2.5 high | 2.6–4.5 intermediate | -// >4.5 low risk. +// NO RISK TIER. This used to emit Bauer's post-MI mortality tier (≤2.5 high / +// 2.6–4.5 intermediate / >4.5 low). Those cut-offs come from 24-h Holter ECG in +// 1,455 post-MI patients (Lancet 2006;367:1674-81); healthy 24-h DC is +// 13.43 ± 5.60 ms, so every wrist-PPG night sits below the whole table and the +// string read "low risk" or worse for a healthy user forever. Measured on one +// subject across three straps inside nine days: gen4 DC 7.61–9.90 → 'low' every +// night, MG 4.06/4.87/5.96 → 'intermediate'/'low'/'low', W5 1.70 → 'high'. The +// tier was decided by which strap he wore that week. Detection, never diagnosis. +// `capacity` stays as a RELATIVE trended number against the user's own baseline. +// A tier may come back only with per-family calibration (`device.dart`) plus a +// validation on a comparable signal — the one wrist-PPG DC validation that +// exists (Sci Rep 2026, doi 10.1038/s41598-026-52700-7, ρ = 0.95) is 5 min 30 s +// of continuous beat detection in sinus rhythm, not an all-night PPG tachogram. import '../types.dart'; import '../util.dart'; @@ -27,28 +46,19 @@ class PrsaResult { final List profile; // averaged X(k), k=-L..L-1 final int anchors; // number of anchors averaged final String kind; // 'DC' or 'AC' - final String? riskTier; // only meaningful for DC const PrsaResult({ required this.capacity, required this.profile, required this.anchors, required this.kind, - this.riskTier, }); Map toJson() => { 'capacity_ms': round6(capacity), 'anchors': anchors, 'kind': kind, - if (riskTier != null) 'risk_tier': riskTier, }; } -String _dcRisk(double dc) { - if (dc <= 2.5) return 'high'; - if (dc <= 4.5) return 'intermediate'; - return 'low'; -} - /// PRSA deceleration capacity. Metric decelerationCapacity(List nnMs, {int l = 2, double anchorRatioCap = 0.05}) => @@ -134,7 +144,6 @@ Metric _prsa( profile: profile, anchors: anchors.length, kind: kind, - riskTier: deceleration ? _dcRisk(capacity) : null, ), confidence: conf, tier: Tier.high, diff --git a/lib/src/onehz/device.dart b/lib/src/onehz/device.dart new file mode 100644 index 0000000..5173a3f --- /dev/null +++ b/lib/src/onehz/device.dart @@ -0,0 +1,71 @@ +// device.dart — the per-device dispatch seam. +// +// A "device family" is the sensor package a row was MEASURED BY, stamped at +// ingest from the link that produced it (edge: `decoded_onehz.device_family`). +// It is not a model name and it is never inferred from the data afterwards. +// +// Why it exists: the same column means different things on different straps — +// gen4 reports skin temperature as a raw ADC count, gen5 as centi-°C; the HR +// ceiling, the accel scale and the staging inputs all differ too. A universal +// constant across families is a fabricated number for whichever family it was +// not calibrated on. +// +// THE CONTRACT: unknown family ⇒ REFUSE. Every historical row predates this +// column and carries NULL, and a strap we have not calibrated for is not gen4 +// with a different badge. A metric that cannot name its own constants for the +// family in front of it returns an ABSENT Metric with [unknownFamilyNote] — +// never gen4's numbers as a fallback. +// +// HOW A METRIC USES IT — the whole mechanism, no framework: +// +// const _ceiling = {DeviceFamily.gen4: 230, DeviceFamily.gen5: 210}; +// +// Metric foo(..., {String? deviceFamily}) { +// final hi = calibrationFor(_ceiling, deviceFamily); +// if (hi == null) { +// return Metric.absent( +// tier: Tier.high, +// inputs_used: const ['hr'], +// note: unknownFamilyNote(deviceFamily), +// ); +// } +// ... +// } +// +// The constants live NEXT TO the metric that uses them. There is deliberately +// no registry, no plugin table and no central constants file: a shared table +// is how one family's number quietly becomes another's. + +/// The sensor packages we have calibrated algorithms for. +/// +/// `gen5` covers the whole fd4b family (WHOOP 5, MG) — they share one sensor +/// package. Add a value here only alongside the calibration work; an enum entry +/// with no constants behind it just moves the refusal one step later. +enum DeviceFamily { gen4, gen5 } + +/// Parse the ingest-stamped family id (`decoded_onehz.device_family`). +/// +/// Returns null for NULL, for empty, and for any id this build does not know — +/// all three mean "unknown provenance", which is its own case, not gen4. +DeviceFamily? deviceFamilyOf(String? id) => switch (id) { + 'gen4' => DeviceFamily.gen4, + 'gen5' => DeviceFamily.gen5, + _ => null, + }; + +/// The id a [DeviceFamily] is stamped as. Inverse of [deviceFamilyOf]. +String deviceFamilyId(DeviceFamily f) => f.name; + +/// This metric's own constants for [family], or null when it must REFUSE — +/// either the family is unknown or this metric has nothing calibrated for it. +/// Both are refusals: never substitute another family's constants. +T? calibrationFor(Map byFamily, String? family) { + final f = deviceFamilyOf(family); + return f == null ? null : byFamily[f]; +} + +/// MACHINE-READABLE refusal note, same convention as `need_baseline:`. +/// +/// unknown_device_family:id= +String unknownFamilyNote(String? id) => + 'unknown_device_family:id=${(id == null || id.isEmpty) ? 'none' : id}'; diff --git a/lib/src/onehz/foundations/baseline.dart b/lib/src/onehz/foundations/baseline.dart index adc0343..bbbff11 100644 --- a/lib/src/onehz/foundations/baseline.dart +++ b/lib/src/onehz/foundations/baseline.dart @@ -1,10 +1,17 @@ // FOUNDATION — robust personal baseline. // -// median + MAD (Leys 2013; Iglewicz-Hoaglin modified-z, flag |M|>3.5), -// clamped gap-aware EWMA (Roberts 1959; λ ↔ half-life), coverage-gate -// (Plews 2014: require ≥minValid of window), and a SWC/MDC gate so we only -// surface a change beyond the metric's minimal detectable change (Hopkins -// 2000). MAD=0 is guarded for quantized data. +// median + MAD (Leys 2013; Iglewicz-Hoaglin modified-z, flag |M|>3.5), a +// MINIMUM-COUNT gate, and an MDC gate so we only surface a change beyond the +// metric's minimal detectable change (Hopkins 2000). MAD=0 is guarded for +// quantized data. +// +// NOT A COVERAGE GATE, and it used to cite Plews 2014 as if it were. Plews' +// compliance result is about ≥3 valid days PER WEEK — a fraction, with a +// denominator. [robustBaseline] is handed a list of values that are all valid +// by construction, so it has no denominator to divide by: `sufficient` can only +// ever mean "the caller passed at least minValid values". A caller that DOES +// know its window length can say so by building [RobustBaseline] itself with a +// larger [RobustBaseline.nWindow]; nothing in this file can infer it. // // HONESTY: insufficient coverage => baseline absent (null), never an // optimistic guess. @@ -16,8 +23,14 @@ class RobustBaseline { final double? center; // median of the window (null if no data) final double? scale; // MAD (scaled to σ); null if undefined final int nValid; + + /// Days the window SPANNED, when the caller knows it. [robustBaseline] cannot + /// — it receives only valid values — and sets it equal to [nValid]; a caller + /// that constructs this directly may pass the real span. final int nWindow; - final bool sufficient; // passed the coverage gate + + /// [nValid] ≥ minValid. A minimum-COUNT gate, not a coverage fraction. + final bool sufficient; const RobustBaseline({ required this.center, required this.scale, @@ -40,8 +53,9 @@ class RobustBaseline { } /// Build a robust baseline from a window of values (already filtered to the -/// metric of interest). [minValid] coverage gate (e.g. 3 for a 7-day window per -/// Plews 2014). Values are taken as-is; pass only valid samples. +/// metric of interest). [minValid] is a MINIMUM-COUNT gate on those values — +/// see the file header for why it is not a coverage gate. Values are taken +/// as-is; pass only valid samples. RobustBaseline robustBaseline(List window, {int minValid = 3}) { final n = window.length; if (n == 0) { @@ -59,75 +73,6 @@ RobustBaseline robustBaseline(List window, {int minValid = 3}) { ); } -/// Convert an EWMA half-life (in samples) to the smoothing factor λ. -/// λ = 1 - 2^(-1/halfLife). -double lambdaFromHalfLife(double halfLifeSamples) { - if (halfLifeSamples <= 0) return 1; - return 1 - math.pow(2, -1 / halfLifeSamples).toDouble(); -} - -class EwmaPoint { - final double value; // smoothed estimate at this step - final bool gap; // true if this step was a (clamped) gap fill - const EwmaPoint(this.value, this.gap); -} - -/// Gap-aware, clamped EWMA over a possibly-irregular series. -/// -/// [series] (time-ordered) values with their times (ms). On a gap larger than -/// [maxGapMs], the update is CLAMPED: λ is not allowed to fully reset the -/// estimate (we cap the effective weight) and the point is flagged as a gap — -/// we never invent intervening data. λ derived from [halfLifeMs]. -List gapAwareEwma( - List timesMs, - List values, { - required double halfLifeMs, - double maxGapMs = 0, -}) { - final n = values.length; - if (n == 0 || timesMs.length != n) return const []; - final out = []; - double? est; - double? lastT; - for (var i = 0; i < n; i++) { - if (est == null) { - est = values[i]; - out.add(EwmaPoint(est, false)); - lastT = timesMs[i]; - continue; - } - final dt = timesMs[i] - lastT!; - if (dt <= 0 || halfLifeMs <= 0 || !dt.isFinite) { - // Duplicate or non-monotonic timestamp: NO time has elapsed, so the - // estimate earns no new weight (λ = 0). Letting λ = 1 − 2^(−dt/H) go - // negative here EXTRAPOLATES the estimate outside the data (Roberts 1959 - // requires λ ∈ [0,1]). We also keep lastT at the latest time already - // seen so an out-of-order sample cannot corrupt the next dt. - out.add(EwmaPoint(est, false)); - continue; - } - // Time-aware λ: half-life expressed in ms => decay over the elapsed dt. - var lambda = 1 - math.pow(2, -dt / halfLifeMs).toDouble(); - final isGap = maxGapMs > 0 && dt > maxGapMs; - if (isGap) { - // Clamp: a long gap should not let one new sample dominate. Cap λ at 0.5. - lambda = math.min(lambda, 0.5); - } - est = lambda * values[i] + (1 - lambda) * est; - out.add(EwmaPoint(est, isGap)); - lastT = timesMs[i]; - } - return out; -} - -/// Smallest Worthwhile Change (Hopkins 2000): 0.2 × between-subject SD. For an -/// n-of-1 context we use the personal baseline scale (SD-equivalent) as the -/// dispersion, so SWC = swcMultiplier × scale. -double? swc(RobustBaseline baseline, {double swcMultiplier = 0.2}) { - if (baseline.scale == null) return null; - return swcMultiplier * baseline.scale!; -} - /// Minimal Detectable Change: MDC = 1.96 × √2 × typical-error. /// We approximate the typical error by the baseline scale unless a measured /// [typicalError] is supplied. Returns null if no dispersion is known. @@ -138,12 +83,3 @@ double? mdc(RobustBaseline baseline, {double? typicalError}) { if (te == null || te <= 0) return null; return 1.96 * math.sqrt2 * te; } - -/// Gate a candidate change: surface it only if |Δ| exceeds the MDC (or, when -/// no MDC is known, never claim a change). Returns true => report the change. -bool changeExceedsMdc(double delta, RobustBaseline baseline, - {double? typicalError}) { - final m = mdc(baseline, typicalError: typicalError); - if (m == null) return false; // can't justify a claim => stay silent - return delta.abs() > m; -} diff --git a/lib/src/onehz/foundations/ewma_baselines.dart b/lib/src/onehz/foundations/ewma_baselines.dart index 36c1846..fe5d2fa 100644 --- a/lib/src/onehz/foundations/ewma_baselines.dart +++ b/lib/src/onehz/foundations/ewma_baselines.dart @@ -7,15 +7,13 @@ // gating, early-life anti-anchoring (fast adapt + suspended hard-outlier gate + // inflated Winsor band), hard-outlier rejection, and Winsor clamping. // -// Two paths: -// 1. Winsorized EWMA (production): [baselineUpdate] / [baselineFoldHistory]. -// 2. Trailing-window mean/SD (auditable): [baselineRollingMeanSD]. +// [Baselines.update] folds one night, [Baselines.foldHistory] replays a series, +// [Baselines.deviation] computes z / delta / ratio / in-normal-range. // -// Both produce a [BaselineState] so a recovery scorer can consume either -// uniformly. [baselineDeviation] computes z / delta / ratio / in-normal-range. -// -// HONESTY: insufficient nights => status `calibrating`; the engine never -// fabricates a "trusted" baseline. +// HONESTY: insufficient nights => status `calibrating`; NO nights => no state +// at all (null), never a baseline invented from the midpoint of the metric's +// physiological bounds. The engine never fabricates a baseline, trusted or +// otherwise. import 'dart:math' as math; @@ -27,7 +25,11 @@ class MetricCfg { /// Physiological upper bound (hard reject above). final double maxVal; - /// σ_floor: minimum dispersion. + /// Minimum [BaselineState.spread] — i.e. a floor in ABS-DEVIATION units, not + /// in σ. (Multiply by 1.253 for the σ it implies.) The dead trailing-mean/SD + /// path applied the same constant as a σ floor, which put the two paths 25 % + /// apart on identical history; it has been deleted, so only this reading + /// exists now. final double floorSpread; /// Baseline-center half-life (nights). @@ -145,13 +147,29 @@ class Baselines { /// Default per-metric configurations (HRV, resting HR, respiration, skin temp). static const Map metricCfg = { 'hrv': MetricCfg( - minVal: 5.0, maxVal: 250.0, floorSpread: 5.0, halfLifeB: 14.0, halfLifeS: 21.0), + minVal: 5.0, + maxVal: 250.0, + floorSpread: 5.0, + halfLifeB: 14.0, + halfLifeS: 21.0), 'resting_hr': MetricCfg( - minVal: 30.0, maxVal: 120.0, floorSpread: 2.0, halfLifeB: 14.0, halfLifeS: 21.0), + minVal: 30.0, + maxVal: 120.0, + floorSpread: 2.0, + halfLifeB: 14.0, + halfLifeS: 21.0), 'resp': MetricCfg( - minVal: 4.0, maxVal: 40.0, floorSpread: 0.5, halfLifeB: 14.0, halfLifeS: 21.0), + minVal: 4.0, + maxVal: 40.0, + floorSpread: 0.5, + halfLifeB: 14.0, + halfLifeS: 21.0), 'skin_temp': MetricCfg( - minVal: 20.0, maxVal: 42.0, floorSpread: 0.3, halfLifeB: 14.0, halfLifeS: 21.0), + minVal: 20.0, + maxVal: 42.0, + floorSpread: 0.3, + halfLifeB: 14.0, + halfLifeS: 21.0), }; /// Convenience accessors for the standard configs. @@ -161,7 +179,8 @@ class Baselines { static MetricCfg get skinTempCfg => metricCfg['skin_temp']!; /// Convert a half-life in nights to an EWMA smoothing factor. - static double lambda(double halfLife) => 1.0 - math.pow(0.5, 1.0 / halfLife).toDouble(); + static double lambda(double halfLife) => + 1.0 - math.pow(0.5, 1.0 / halfLife).toDouble(); static BaselineStatus computeStatus(int nValid, int nightsSinceUpdate) { if (nightsSinceUpdate > staleDays && nValid >= minNightsSeed) { @@ -176,11 +195,14 @@ class Baselines { /// Incorporate one new nightly value into the baseline state. /// - /// - `state == null`: seed the first night. + /// - `state == null`: seed the first night — or return NULL when that night + /// carries no usable value, because nothing has been observed yet and a + /// baseline for nothing does not exist. /// - `value == null` or out-of-range: skip-and-hold (carry forward). /// - hard outlier (> HARD_OUTLIER_K × spread): seen but not folded. /// - otherwise: Winsorized EWMA center + EWMA-abs-dev spread update. - static BaselineState update(BaselineState? state, double? value, MetricCfg cfg) { + static BaselineState? update( + BaselineState? state, double? value, MetricCfg cfg) { final lb = lambda(cfg.halfLifeB); final ls = lambda(cfg.halfLifeS); @@ -194,13 +216,11 @@ class Baselines { nightsSinceUpdate: 0, status: BaselineStatus.calibrating); } - final seed = (cfg.minVal + cfg.maxVal) / 2.0; - return BaselineState( - baseline: seed, - spread: cfg.floorSpread, - nValid: 0, - nightsSinceUpdate: 1, - status: BaselineStatus.calibrating); + // No usable observation yet. This used to return a state whose baseline + // was `(minVal + maxVal) / 2` — a number nobody measured — and + // [deviation] then dutifully reported z = −13 against it. There is no + // baseline; say so. + return null; } // Missing night: skip-and-hold. @@ -231,36 +251,43 @@ class Baselines { if (state.nValid >= minNightsSeed && !isYoung) { final dev = (value - state.baseline).abs(); if (dev > hardOutlierK * state.spread) { + // [nightsSinceUpdate] counts nights the baseline DID NOT MOVE — that is + // the only reading `computeStatus` uses it for (stale after 14). A + // rejected night moved nothing, so it increments, exactly like the + // missing-night and out-of-range holds above. It used to reset to 0 + // here, so the same counter meant "nights since we heard from the + // sensor" on one branch and "nights since the baseline moved" on the + // other, and 14 straight rejected nights read as trusted. + final m = state.nightsSinceUpdate + 1; return BaselineState( baseline: state.baseline, spread: state.spread, nValid: state.nValid, - nightsSinceUpdate: 0, - status: computeStatus(state.nValid, 0)); + nightsSinceUpdate: m, + status: computeStatus(state.nValid, m)); } } - // First real value after a None-placeholder seed: treat as clean first night. - if (state.nValid == 0) { - return BaselineState( - baseline: value, - spread: cfg.floorSpread, - nValid: 1, - nightsSinceUpdate: 0, - status: BaselineStatus.calibrating); - } - // Step 1: Winsorized EWMA update. - final effSpread = isYoung ? state.spread * earlySpreadInflate : state.spread; + final effSpread = + isYoung ? state.spread * earlySpreadInflate : state.spread; final effLb = isYoung ? lambda(earlyHalfLifeB) : lb; final lo = state.baseline - winsorK * effSpread; final hi = state.baseline + winsorK * effSpread; final clamped = math.max(lo, math.min(hi, value)); final newBaseline = effLb * clamped + (1.0 - effLb) * state.baseline; - // Spread uses the UNCLAMPED value so true deviations are tracked. - final absDev = (value - newBaseline).abs(); - final newSpread = math.max(cfg.floorSpread, ls * absDev + (1.0 - ls) * state.spread); + // Spread uses the UNCLAMPED value so true deviations are tracked — and it + // is measured against the baseline the night ARRIVED at, not the one the + // night has already pulled toward itself. Measuring against `newBaseline` + // (what shipped) makes the deviation identically (1 − λ_B)·(value − + // baseline) for anything inside the Winsor band, so the spread tracker was + // fed a systematically shrunken deviation: −4.83 % at halfLifeB = 14 and + // −20.63 % while young (earlyHalfLifeB = 3). A too-tight spread inflates + // every z downstream, which means MORE alerts, not fewer. + final absDev = (value - state.baseline).abs(); + final newSpread = + math.max(cfg.floorSpread, ls * absDev + (1.0 - ls) * state.spread); final newN = state.nValid + 1; return BaselineState( @@ -273,78 +300,33 @@ class Baselines { /// Replay an ordered sequence of nightly values (oldest first) to build state. /// `null` entries are treated as missing nights (skip-and-hold). - static BaselineState foldHistory(List values, MetricCfg cfg) { + /// + /// NULL when no night in [values] was usable: with nothing observed there is + /// no personal baseline, and the caller must withhold z / delta / ratio / + /// in-normal-range rather than score today against an invented centre. + static BaselineState? foldHistory(List values, MetricCfg cfg) { BaselineState? state; for (final v in values) { state = update(state, v, cfg); } - if (state != null) return state; - final seed = (cfg.minVal + cfg.maxVal) / 2.0; - return BaselineState( - baseline: seed, - spread: cfg.floorSpread, - nValid: 0, - nightsSinceUpdate: 0, - status: BaselineStatus.calibrating); + return state; } // ── Deviation ─────────────────────────────────────────────────────────────── /// Compute z / delta / ratio / in-normal-range for a value vs a baseline. /// z uses (value − baseline) / (1.253 × spread). - static Deviation deviation(double value, BaselineState state) { + /// + /// NULL when the state rests on zero valid nights — there is nothing to + /// deviate FROM. (The engine can no longer produce such a state; the guard + /// holds the contract for a hand-built or rehydrated one.) + static Deviation? deviation(double value, BaselineState state) { + if (state.nValid == 0) return null; final sigma = math.max(1.253 * state.spread, 1e-9); final z = (value - state.baseline) / sigma; final delta = value - state.baseline; final ratio = state.baseline != 0 ? (value / state.baseline - 1.0) : 0.0; - return Deviation(z: z, delta: delta, ratio: ratio, inNormalRange: z.abs() <= 1.0); - } - - // ── Trailing-window mean/SD (simple, auditable) ───────────────────────────── - - /// Rolling personal baseline from the trailing [window] valid nights, as a - /// plain mean and sample SD (ddof=1). The spread returned is in the SAME - /// internal abs-dev units the Winsor EWMA uses (SD / 1.253). - static BaselineState rollingMeanSD(List values, MetricCfg cfg, - {int window = 30}) { - final valid = [ - for (final v in values) - if (v != null && cfg.minVal <= v && v <= cfg.maxVal) v - ]; - if (valid.isEmpty) { - final seed = (cfg.minVal + cfg.maxVal) / 2.0; - return BaselineState( - baseline: seed, - spread: cfg.floorSpread, - nValid: 0, - nightsSinceUpdate: 0, - status: BaselineStatus.calibrating); - } - final trailing = - valid.length > window ? valid.sublist(valid.length - window) : valid; - final n = trailing.length; - final mean = trailing.reduce((a, b) => a + b) / n; - - double sd; - if (n >= 2) { - var ss = 0.0; - for (final v in trailing) { - final d = v - mean; - ss += d * d; - } - sd = math.sqrt(ss / (n - 1)); - } else { - sd = cfg.floorSpread * 1.253; - } - - final sigmaFloored = math.max(cfg.floorSpread, sd); - final spreadInternal = sigmaFloored / 1.253; - - return BaselineState( - baseline: mean, - spread: spreadInternal, - nValid: n, - nightsSinceUpdate: 0, - status: computeStatus(n, 0)); + return Deviation( + z: z, delta: delta, ratio: ratio, inNormalRange: z.abs() <= 1.0); } } diff --git a/lib/src/onehz/foundations/fusion.dart b/lib/src/onehz/foundations/fusion.dart index 2a87b8e..e783060 100644 --- a/lib/src/onehz/foundations/fusion.dart +++ b/lib/src/onehz/foundations/fusion.dart @@ -2,8 +2,7 @@ // // Inverse-variance fusion (Aitken 1935): the minimum-variance unbiased linear // combination of independent estimates weights each by 1/σ²; the fused -// variance is 1/Σ(1/σ²). Simple GUM (JCGM 100:2008) uncertainty propagation for -// a weighted sum gives the combined standard uncertainty. +// variance is 1/Σ(1/σ²). // // Biased motion-artifact inputs are gated OUT, not down-weighted: a caller // passes `trusted=false` for any channel the SQI gate rejected, and those are @@ -45,7 +44,13 @@ FusionResult inverseVarianceFuse(List inputs) { var wsum = 0.0; // Σ 1/σ² var vwsum = 0.0; // Σ value/σ² for (final inp in inputs) { - if (!inp.trusted || inp.variance <= 0 || inp.value.isNaN) { + // NOTE: `variance <= 0` is FALSE for a NaN variance, so test finiteness + // explicitly — otherwise a NaN variance sails past the guard and poisons + // `wsum`, and the absent branch below can never fire. + if (!inp.trusted || + !inp.variance.isFinite || + inp.variance <= 0 || + !inp.value.isFinite) { dropped.add(inp.label); continue; } @@ -72,25 +77,3 @@ FusionResult inverseVarianceFuse(List inputs) { dropped: dropped, ); } - -/// Map a per-input SNR (or contact-quality 0..1) to a variance, so the SQI -/// channel directly drives fusion weight. Higher quality => lower variance. -/// variance = baseVariance / max(quality, floor). -double varianceFromQuality(double quality, double baseVariance, - {double floor = 0.05}) { - final q = quality < floor ? floor : (quality > 1 ? 1 : quality); - return baseVariance / q; -} - -/// GUM combined standard uncertainty for a weighted sum y = Σ wᵢ·xᵢ with -/// independent inputs: u_c = √(Σ (wᵢ·uᵢ)²). Returns null on length mismatch. -double? gumWeightedSumUncertainty( - List weights, List stdUncertainties) { - if (weights.length != stdUncertainties.length || weights.isEmpty) return null; - var s = 0.0; - for (var i = 0; i < weights.length; i++) { - final term = weights[i] * stdUncertainties[i]; - s += term * term; - } - return math.sqrt(s); -} diff --git a/lib/src/onehz/foundations/ppg_sqi.dart b/lib/src/onehz/foundations/ppg_sqi.dart deleted file mode 100644 index 7e8efbf..0000000 --- a/lib/src/onehz/foundations/ppg_sqi.dart +++ /dev/null @@ -1,87 +0,0 @@ -// FOUNDATION — PPG signal-quality gate. -// -// Elgendi 2016 skewness-SQI (the best single cheap PPG quality index; works on -// a short window of 1 Hz-or-faster samples) + Orphanidou 2015 physiological- -// range rules (plausible HR 40–180 bpm; plausible RR; not-too-variable HR). -// -// Template-matching SQI is foreground-only (needs the 419 Hz waveform) and is -// deliberately NOT implemented here — see the catalog. We expose only what a -// 1 Hz substrate honestly supports: a per-window TRUST flag. - -import 'dart:math' as math; -import '../util.dart'; - -class SqiResult { - final bool trusted; - final double skewness; // Elgendi skewness-SQI of the window - final List reasons; // why it failed, if it did - const SqiResult({ - required this.trusted, - required this.skewness, - required this.reasons, - }); -} - -/// Skewness of a window. Positive skew correlates with good PPG (systolic -/// upstroke). Returns 0 for degenerate windows. -double skewnessSqi(List window) { - if (window.length < 3) return 0; - final m = mean(window)!; - final sd = stddevPop(window)!; - if (sd == 0) return 0; - var s = 0.0; - for (final x in window) { - final d = (x - m) / sd; - s += d * d * d; - } - return s / window.length; -} - -/// Per-window PPG trust gate. -/// -/// [ppgWindow] raw green-ADC samples for the window (skewness-SQI input). -/// [hrBpm] the window's HR (bpm) for the physiological-range rule. -/// [skewMin] minimum acceptable skewness (Elgendi suggests >0 good quality). -SqiResult ppgTrust( - List ppgWindow, - double hrBpm, { - double skewMin = -0.1, -}) { - final reasons = []; - final sk = skewnessSqi(ppgWindow); - - // Orphanidou physiological-range rules. - if (hrBpm <= 0) { - reasons.add('off-skin'); - } else if (hrBpm < 40 || hrBpm > 180) { - reasons.add('hr-out-of-range'); - } - if (ppgWindow.isEmpty) { - reasons.add('no-ppg'); - } else if (sk < skewMin) { - reasons.add('low-skewness'); - } - // Flatline check (no perfusion variability). - if (ppgWindow.isNotEmpty) { - final sd = stddevPop(ppgWindow)!; - final m = mean(ppgWindow)!.abs(); - if (sd == 0 || (m > 0 && sd / m < 1e-4)) reasons.add('flatline'); - } - - return SqiResult(trusted: reasons.isEmpty, skewness: sk, reasons: reasons); -} - -/// Orphanidou physiological RR-range rule on a beat series (ms): every NN in -/// [300,2000] and no successive ratio > 3 (a missed/extra beat signature). -bool rrPhysiologicallyPlausible(List rrMs) { - if (rrMs.isEmpty) return false; - for (var i = 0; i < rrMs.length; i++) { - if (rrMs[i] < 300 || rrMs[i] > 2000) return false; - if (i > 0) { - final ratio = - math.max(rrMs[i], rrMs[i - 1]) / math.min(rrMs[i], rrMs[i - 1]); - if (ratio > 3) return false; - } - } - return true; -} diff --git a/lib/src/onehz/foundations/rr_correction.dart b/lib/src/onehz/foundations/rr_correction.dart index c76f609..8984013 100644 --- a/lib/src/onehz/foundations/rr_correction.dart +++ b/lib/src/onehz/foundations/rr_correction.dart @@ -24,7 +24,14 @@ class RrCorrectionResult { /// runs dropped (length may differ from the input). final List nn; - /// Beat-time (ms) for each NN interval, cumulative from t0. + /// Beat-time (ms) for each NN interval, relative to the START of the first + /// input beat (so `nnTimesMs.first == rrMs.first` when the first beat is + /// kept, and adding `rrTsMs.first - rrMs.first` puts it back on the epoch). + /// + /// Built by cumulative-summing RR *within a contiguous run*, and RE-ANCHORED + /// to the beat's real timestamp at every sensor dropout — see [correctRr]'s + /// `rrTsMs`. Without `rrTsMs` it is a pure cumsum and the dropouts are + /// spliced out; pass the timestamps. final List nnTimesMs; /// Per-input-beat classification (same length as the input rr). @@ -54,13 +61,31 @@ class RrCorrectionResult { /// [rrMs] the raw RR series (ms). [alpha] scales the time-varying threshold /// (paper default 5.2 on the QD estimate). [windowBeats] sliding window for the /// local dispersion estimate. +/// +/// [rrTsMs] the beat timestamps that came with [rrMs] (same length, sorted, +/// epoch ms). PASS THEM. Without them the output clock is Σrr, which splices +/// every sensor dropout out of the record: the intervals the band never +/// reported are not in [rrMs] at all, so no amount of book-keeping inside this +/// function can recover them. Measured on 13 real nights, the cumsum span was +/// 0.13–0.87 of the true rec_ts span, and the Lomb-Scargle spectrum built on it +/// moved LF/HF across the sympatho-vagal line (0.65 → 1.53 on one MG night). +/// With them, the clock cumsums RR *inside* a contiguous run — the RR values +/// are the only sub-second information that exists, since `rr_ts_ms` is +/// `rec_ts*1000` and therefore whole-second — and re-anchors to the real +/// timestamp whenever the wall step exceeds the interval by more than +/// [reanchorGapMs], i.e. at a dropout. RrCorrectionResult correctRr( List rrMs, { + List? rrTsMs, double alpha = 5.2, int windowBeats = 91, double minThresholdMs = 100, + double reanchorGapMs = 1000, }) { final n = rrMs.length; + // Beat-end time for EVERY input beat, kept or not. One array, built once, so + // every branch below reads a clock instead of advancing its own. + final tAll = _beatTimes(rrMs, rrTsMs, reanchorGapMs); if (n == 0) { return const RrCorrectionResult( nn: [], @@ -80,12 +105,10 @@ RrCorrectionResult correctRr( ]; final nn = []; final times = []; - var t = 0.0; for (var i = 0; i < n; i++) { if (classes[i] == BeatClass.normal) { - t += rrMs[i]; nn.add(rrMs[i]); - times.add(t); + times.add(tAll[i]); } } return RrCorrectionResult( @@ -166,15 +189,13 @@ RrCorrectionResult correctRr( final isArtifact = [for (final c in classes) c != BeatClass.normal]; final nn = []; final times = []; - var t = 0.0; var dropped = 0; var corrected = 0; var i = 0; while (i < n) { if (!isArtifact[i]) { - t += rrMs[i]; nn.add(rrMs[i]); - times.add(t); + times.add(tAll[i]); i++; continue; } @@ -188,15 +209,26 @@ RrCorrectionResult correctRr( // Isolated -> spline-correct from surrounding normals. final corr = _splineCorrect(rrMs, isArtifact, i); if (corr != null) { - t += corr; + // The beat keeps its REAL time, not the interpolated one. `corr` is our + // best guess at what the NN *should* have been (a missed beat splits one + // ~2000 ms interval into two ~1000 ms ones), but the 2000 ms still + // elapsed. Advancing by `corr` deleted ~1 s of record per corrected beat + // — 1.6 % of a night at one missed beat in sixty. nn.add(corr); - times.add(t); + times.add(tAll[i]); corrected++; } else { - dropped++; // no anchors -> honest drop + dropped++; // no anchors -> honest drop; the clock already elapsed } } else { - dropped += runLen; // multi-beat run: NEVER interpolate + // Multi-beat run: NEVER interpolate. The clock is not touched here at all + // — `tAll` already carries the run's elapsed time, so the two sides of a + // dropped run are not spliced together. Not doing this compacted + // `nnTimesMs` (299.0 s of real record emitted as a 294.0 s span) and + // inflated everything keyed off it: cvhr_per_hour's analyzedHours, the + // Lomb-Scargle axis, spanSec — on exactly the noisy nights that needed + // the correction most. + dropped += runLen; } i = j; } @@ -212,6 +244,55 @@ RrCorrectionResult correctRr( ); } +/// Beat-end time (ms, relative to the start of beat 0) for every input beat. +/// +/// RR intervals tile the timeline, so the clock advances for EVERY interval, +/// kept or dropped — but only for intervals the band actually REPORTED. A +/// sensor dropout is not an interval at all: it is absent from [rrMs], and +/// summing across it splices the two sides of the hole together. [rrTsMs] is +/// the only witness that the hole existed, so when the wall step between two +/// consecutive reported beats exceeds that beat's own interval by more than +/// [reanchorGapMs], the clock is RE-ANCHORED to the real timestamp instead of +/// accumulated. Inside a contiguous run the RR cumsum is kept, because +/// `rr_ts_ms = rec_ts*1000` is whole-second and the RR values are the only +/// sub-second information in the record. +/// +/// The slack has to be ≥ the max plausible RR (2,400 ms saturation) minus a +/// normal beat, or ordinary quantisation would re-anchor constantly; 1,000 ms +/// is the audit's figure and is ~1 whole quantisation step. +List _beatTimes( + List rrMs, List? rrTsMs, double reanchorGapMs) { + final n = rrMs.length; + final t = List.filled(n, 0); + if (n == 0) return t; + final wall = rrTsMs != null && rrTsMs.length == n; + final origin = wall ? rrTsMs[0] - rrMs[0] : 0.0; + var cur = 0.0; + for (var i = 0; i < n; i++) { + // The test is LOCAL — this beat's wall step against this beat's own + // interval — so a run's accumulated cumsum-vs-wall drift never triggers it. + // + // That drift is real and NOT resolved here. Inside contiguous runs of ≥200 + // beats, Σrr / rec_ts-span measures 0.9631 on gen4 against 0.9988 (W5) and + // 1.0013 (MG). Which clock is wrong is not decidable from the exports: the + // band's `hr` field is computed from the same beat detector as `rr_ms`, so + // it is not independent evidence, and a low rate of beats the band never + // reports produces the same deficit with a perfectly correct clock. So + // there is deliberately no `rrClockScale` — applying 0.963 to gen4's RR + // values would be actively wrong under that second explanation. The + // consequence to carry: **gen4 and gen5/MG nights are not comparable on any + // frequency-domain or per-hour quantity** until it is resolved. + final dropout = + wall && i > 0 && (rrTsMs[i] - rrTsMs[i - 1]) - rrMs[i] > reanchorGapMs; + final anchored = wall ? rrTsMs[i] - origin : 0.0; + // `> cur` keeps the clock monotonic even if timestamps arrive out of order + // or the cumsum has run ahead of the wall — a backwards jump is never taken. + cur = (dropout && anchored > cur) ? anchored : cur + rrMs[i]; + t[i] = cur; + } + return t; +} + /// Time-varying threshold: alpha × QD of the SIGNED series x in a sliding /// window, where QD = (Q3 − Q1)/2 (Lipponen & Tarvainen 2019, eq. for th1/th2: /// "th = α · quartile deviation of dRR over the 91-beat window", α = 5.2). diff --git a/lib/src/onehz/human/alertness_forecast.dart b/lib/src/onehz/human/alertness_forecast.dart new file mode 100644 index 0000000..7220a78 --- /dev/null +++ b/lib/src/onehz/human/alertness_forecast.dart @@ -0,0 +1,288 @@ +// HUMAN — when today is likely to get hard (MIND-11). +// +// Åkerstedt & Folkard's Three-Process Model of alertness regulation: a +// homeostatic process S that falls exponentially with time awake and recovers +// exponentially across sleep, a circadian process C anchored to clock time, and +// a sleep-inertia term W that decays over the first hour after waking. +// Alertness = S + C − W. +// +// WHAT THIS IS. A MODEL PREDICTION, not a measurement. Nothing on this band +// measures alertness — there is no reaction time, no KSS, no eye tracking, no +// EEG. The Three-Process parameters are fitted to GROUP data from laboratory +// sleep-restriction studies, so what comes out is *the average person's* +// alertness given *your* sleep timing. Individual differences in sleep need and +// circadian amplitude are large, and this model cannot see yours. +// +// THEREFORE: NO NUMERIC SCORE, ANYWHERE. The output is a SHAPE — a unitless +// curve normalised inside its own day, good for drawing and for nothing else — +// plus a NAMED trough window. A KSS-like number is the thing that gets +// screenshotted and quoted back at us, so there isn't one, and there must never +// be one bolted on: not a 0-100, not a letter, not "alertness 42". +// +// TWO GATES, and they are the feature: +// 1. It ABSTAINS on a missing or unjudged night. It never assumes eight +// hours — assuming the input is how a model prediction becomes a +// fabrication with a chart around it. +// 2. The SAFETY REFUSAL is explicit and belongs on the card: this is not a +// fitness-to-drive assessment, not a shift-safety tool, and it never says +// you are impaired. + +import 'dart:math' as math; + +import '../types.dart'; +import '../util.dart'; + +/// A daytime sleep, in local clock hours since midnight of the wake day. +class DaytimeSleepWindow { + final double startHour; + final double endHour; + const DaytimeSleepWindow(this.startHour, this.endHour); +} + +/// The forecast: a drawable shape and the window it bottoms out in. +class AlertnessForecast { + /// Local clock hour of the first point of [shape]. + final double startHour; + + /// Hours between consecutive [shape] points. + final double stepHours; + + /// UNITLESS, normalised to its own min/max within this day. It is a picture + /// of a shape and it is NOT a score: do not print a value from it, do not + /// compare one day's to another's, do not put a number on an axis. + final List shape; + + /// The lowest window of the day, in local clock hours. + final double troughStartHour; + final double troughEndHour; + + /// A plain name for that window ("early afternoon"), so the card can say + /// when without saying how much. + final String troughLabel; + + const AlertnessForecast({ + required this.startHour, + required this.stepHours, + required this.shape, + required this.troughStartHour, + required this.troughEndHour, + required this.troughLabel, + }); + + Map toJson() => { + 'start_hour': round6(startHour), + 'step_hours': round6(stepHours), + 'shape': [for (final v in shape) round6(v)], + 'trough_start_hour': round6(troughStartHour), + 'trough_end_hour': round6(troughEndHour), + 'trough_label': troughLabel, + }; +} + +// Published Three-Process parameters, quoted verbatim from Ingre / Van Leeuwen +// et al., PLOS ONE 2014;9(10):e108679 (PMC4203690 §1.1–1.8), which restates +// Åkerstedt & Folkard's fitted values. They are group values on the model's own +// 1-16 alertness scale; we only ever use the SHAPE they produce, which is why no +// constant here is exposed or printed. The rate constants are published as +// per-hour RATES — written as 1/rate here so the paper's own numbers are on the +// page and the next sweep can re-check without re-finding it. +// +// THE TWO TIME CONSTANTS WERE WRONG UNTIL 2026-08-17: τ_wake was 2.6 h (the +// published τ_SLEEP, written into the wake slot) and τ_sleep 4.2 h. S therefore +// hit its lower asymptote by mid-morning and the drawn curve was process C alone +// for two thirds of the day. Do not "simplify" these back to round numbers. +const double _sUpper = 14.3; // ha — asymptote S recovers toward during sleep +const double _sLower = 2.4; // la — asymptote S decays toward while awake +const double _tauWake = 1 / 0.0353; // h — 28.33, wake decay (published rate) +const double _tauSleep = 1 / 0.3813; // h — 2.62, sleep recovery (published rate) +const double _cAmplitude = 2.5; // Ca — 24 h circadian amplitude +const double _uAmplitude = 0.5; // Ua — 12 h ultradian amplitude +const double _uMean = -0.5; // Um — ultradian offset +const double _wInertia = 5.72; // Wc — sleep-inertia magnitude at wake +const double _tauInertia = 0.67; // Wd — h, inertia decay + +// NOT IMPLEMENTED, deliberately: the sleep brake (bl 12.2), which slows S +// recovery late in a sleep episode. It only bends recovery inside a long sleep, +// and every sleep this function models is either last night (entered through +// [sleepDurationHours], not simulated) or a nap far shorter than the brake's +// reach. Named here so the omission is a decision on the page, not a gap. + +/// Population circadian peak (p = 16.8 h), used only when the user's own +/// acrophase is not available. Disclosed in the note, because it is an +/// assumption about *this* user's body clock made from other people's. +const double _defaultAcrophaseHours = 16.8; + +const double _stepHours = 0.25; + +/// The two phase disclosures. Which one ships is the difference between "we +/// guessed your body clock from other people" and "we fitted a proxy for it on +/// your own rhythm" — never "we measured it", because nothing here does. +const String _assumedPhaseNote = + 'Circadian phase ASSUMED from the population (peak ~16:48), not measured ' + 'on you. '; +const String _fittedPhaseNote = + 'Circadian phase taken from a rhythm fitted on your own data — a PROXY for ' + 'body-clock phase, which nothing on this band measures. '; + +/// Forecast the SHAPE of today's alertness from last night's sleep. +/// +/// [wakeLocalHour] local clock hour the night ended; [sleepDurationHours] how +/// long that night's sleep actually was. BOTH are required to be present — a +/// null in either is an abstention, not a default. [naps] are daytime sleeps in +/// the same local-hour frame. [circadianAcrophaseHours] is the user's own +/// cosinor acrophase when we have one — a fitted PROXY for body-clock phase (the +/// caller today fits it on hourly HR, which tracks activity), never a measured +/// alertness phase, and the note says which of the two produced the curve. +/// +/// The curve runs from wake to the projected end of the waking day +/// (24 h − [sleepDurationHours]), capped at [horizonHours]. +/// +/// Local clock hours throughout, never epochs: mixing the two is how a day +/// label ends up an hour out and a "trough" lands in the wrong afternoon. +Metric alertnessForecast({ + required double? wakeLocalHour, + required double? sleepDurationHours, + List naps = const [], + double? circadianAcrophaseHours, + double horizonHours = 18, +}) { + const inputs = [ + 'sleep_offset', + 'sleep_duration', + 'naps', + 'circadian_acrophase?' + ]; + const safety = 'MODEL PREDICTION from group data, not a measurement of you — ' + 'nothing here measures alertness. NOT a fitness-to-drive or shift-safety ' + 'assessment, and it never says you are impaired.'; + + if (wakeLocalHour == null || + sleepDurationHours == null || + !wakeLocalHour.isFinite || + !sleepDurationHours.isFinite || + sleepDurationHours <= 0) { + // The night is missing or the window was never judged. There is no eight + // hours to fall back on. This abstention is the gate; do not remove it. + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'no_judged_night: alertness is not forecast without last night. ' + '$safety', + ); + } + + // S at wake: recovery across last night from wherever the previous day left + // it. Starting the recovery from the lower asymptote is the conservative + // choice — it makes a short night matter and a long one saturate. + final s0 = + _sUpper - (_sUpper - _sLower) * math.exp(-sleepDurationHours / _tauSleep); + + final phase = + (circadianAcrophaseHours != null && circadianAcrophaseHours.isFinite) + ? circadianAcrophaseHours + : _defaultAcrophaseHours; + + // THE CURVE ENDS WHEN THE DAY DOES. Until 2026-08-17 it ran a fixed 18 h and + // the trough search below could only look at windows that fitted inside that + // cut — so the "lowest two hours" was the LAST admissible window, wake + 16.25 + // h, in 24 of 24 replayed wake × sleep combinations, and sleep duration moved + // it by exactly 0.00 h. The horizon, not the model, was the answer. The waking + // day is 24 h minus last night's sleep, so the input the two documented gates + // exist to protect now actually moves the output. [horizonHours] caps it; the + // 3 h floor only guards absurd input (a claimed 22 h night). + final wakingHours = + math.max(3.0, math.min(horizonHours, 24.0 - sleepDurationHours)); + final steps = (wakingHours / _stepHours).round(); + final raw = []; + var s = s0; + for (var i = 0; i <= steps; i++) { + final hour = wakeLocalHour + i * _stepHours; + final clock = hour % 24; + final asleep = naps.any((n) => clock >= n.startHour && clock < n.endHour); + if (i > 0) { + s = asleep + ? _sUpper - (_sUpper - s) * math.exp(-_stepHours / _tauSleep) + : _sLower + (s - _sLower) * math.exp(-_stepHours / _tauWake); + } + final c = _cAmplitude * math.cos(2 * math.pi * (clock - phase) / 24.0); + // Process U — the model's 12 h ultradian harmonic, phase-locked to the same + // acrophase. Missing until 2026-08-17, which flattened the mid-day wobble. + final u = + _uAmplitude * math.cos(2 * math.pi * (clock - phase) / 12.0) + _uMean; + final w = _wInertia * math.exp(-(i * _stepHours) / _tauInertia); + raw.add(s + c + u - w); + } + + // Normalise inside the day. The result is a picture, not a quantity. + final lo = raw.reduce(math.min); + final hi = raw.reduce(math.max); + final span = hi - lo; + if (!(span > 0)) { + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'degenerate forecast (flat curve); nothing to show. $safety', + ); + } + final shape = [for (final v in raw) (v - lo) / span]; + + // The trough is the lowest TWO-HOUR window, not the single lowest point: + // "around mid-afternoon" is the resolution this model supports, and a + // to-the-minute low would imply a precision it does not have. The first hour + // is excluded — that dip is sleep inertia, which is over, not a forecast. + // + // READ THE NOTE BEFORE TRUSTING THIS WINDOW. Under this model alertness mostly + // falls across the waking day, so the lowest window is usually its last hours + // (an early riser with a long night is the case where the morning wins). Where + // it lands is arithmetic on wake time and sleep length — nothing here measures + // a dip on you — which is why the note says so and the label is a part-of-day, + // never a time. + final w2 = (2.0 / _stepHours).round(); + final skip = (1.0 / _stepHours).round(); + var bestStart = skip; + var bestSum = double.infinity; + for (var i = skip; i + w2 <= shape.length; i++) { + var sum = 0.0; + for (var k = 0; k < w2; k++) { + sum += shape[i + k]; + } + if (sum < bestSum) { + bestSum = sum; + bestStart = i; + } + } + final tStart = (wakeLocalHour + bestStart * _stepHours) % 24; + final tEnd = (tStart + 2.0) % 24; + + return Metric( + value: AlertnessForecast( + startHour: wakeLocalHour % 24, + stepHours: _stepHours, + shape: shape, + troughStartHour: tStart, + troughEndHour: tEnd, + troughLabel: _partOfDay(tStart), + ), + // Deliberately modest and flat: the confidence of a group model applied to + // one person does not improve with more of that person's data. + confidence: 0.35, + tier: Tier.estimate, + inputs_used: inputs, + note: 'Three-Process Model shape only — NO SCORE. ' + '${circadianAcrophaseHours == null ? _assumedPhaseNote : _fittedPhaseNote}' + 'The low window is where this model bottoms out over your waking day — ' + 'usually its last hours. It follows from when you woke and how long you ' + 'slept, and is not a dip measured on you. ' + '$safety', + ); +} + +String _partOfDay(double hour) { + if (hour < 5) return 'the small hours'; + if (hour < 9) return 'early morning'; + if (hour < 12) return 'mid-morning'; + if (hour < 15) return 'early afternoon'; + if (hour < 18) return 'late afternoon'; + if (hour < 21) return 'evening'; + return 'late evening'; +} diff --git a/lib/src/onehz/human/circadian_lifestyle.dart b/lib/src/onehz/human/circadian_lifestyle.dart index 2a8425a..a10d340 100644 --- a/lib/src/onehz/human/circadian_lifestyle.dart +++ b/lib/src/onehz/human/circadian_lifestyle.dart @@ -191,13 +191,23 @@ Metric chronotype( int totalDaysObserved = 0, }) { const inputs = ['mid_sleep_free', 'sleep_duration']; - if (freeMidSleepH.length < minFreeDays || - freeSleepDurH.length != freeMidSleepH.length || - totalDaysObserved < minTotalDays) { - return const Metric.absent( + // The two lists are index-paired free nights. A length mismatch is a CALLER + // bug (one list gated on the sleep window, the other on the accounting block), + // not a coverage problem — saying "you need 14 days" to someone who has 40 is + // a false statement about their data, so it gets its own note. + if (freeSleepDurH.length != freeMidSleepH.length) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'free-day mid-sleep and sleep-duration nights are not paired ' + '(${freeMidSleepH.length} vs ${freeSleepDurH.length})', + ); + } + if (freeMidSleepH.length < minFreeDays || totalDaysObserved < minTotalDays) { + return Metric.absent( tier: Tier.high, inputs_used: inputs, - note: 'chronotype needs ≥14 days with ≥2 free days', + note: 'chronotype needs ≥$minTotalDays days with ≥$minFreeDays free days', ); } // CIRCULAR median (see the helpers at the top): a free-day mid-sleep set @@ -242,6 +252,8 @@ Metric chronotype( stability = percentileOfYou( band, [for (final h in history) _unwrapAround(_mctqBandAnchorH, _wrap24(h))], + // A chronotype has no better end — an early type is not a good type. + better: Better.neither, minN: 14, ); } diff --git a/lib/src/onehz/human/coaching.dart b/lib/src/onehz/human/coaching.dart index cfe4ac2..d99b46b 100644 --- a/lib/src/onehz/human/coaching.dart +++ b/lib/src/onehz/human/coaching.dart @@ -1,8 +1,7 @@ import 'dart:math' as math; import '../types.dart'; -import '../util.dart' show mean, theilSen; - +import '../util.dart' show averageRanks, benjaminiHochberg, mean, theilSen; class SleepNeed { final double needSec; @@ -59,14 +58,21 @@ class BedtimeRec { Map toJson() => {'bedtime_min_of_day': bedtimeMinOfDay}; } +/// Time in bed (minutes) that delivers [needSec] of SLEEP at the user's own +/// efficiency. Shared by [recommendedBedtime] and [recommendedWake] so the two +/// ends of the same night can never be built from different durations again. +double _inBedMin(double needSec, double typicalEfficiencyPct) { + final eff = (typicalEfficiencyPct / 100.0).clamp(0.75, 0.99); + return needSec / eff / 60.0; +} + Metric recommendedBedtime({ required double needSec, required double typicalWakeMinOfDay, required double typicalEfficiencyPct, }) { - final eff = (typicalEfficiencyPct / 100.0).clamp(0.75, 0.99); - final inBedSec = needSec / eff; - final bedMin = (typicalWakeMinOfDay - inBedSec / 60.0) % 1440.0; + final bedMin = + (typicalWakeMinOfDay - _inBedMin(needSec, typicalEfficiencyPct)) % 1440.0; return Metric( value: BedtimeRec(bedMin < 0 ? bedMin + 1440.0 : bedMin), confidence: 0.6, @@ -81,19 +87,31 @@ class WakeRec { Map toJson() => {'wake_min_of_day': wakeMinOfDay}; } +/// Target wake = the recommended bedtime plus the SAME time-in-bed the bedtime +/// was backed off from. +/// +/// It used to add `round(need / 90) × 90` — sleep minutes, cycle-quantized — +/// onto a bedtime built from an IN-BED duration, so the span was always short +/// of the night the card promised: the efficiency gap is need × 0.136 (65 min +/// at an 8 h need) while the largest possible round-UP is 45 min. The pair +/// described a night delivering less than the need printed beside it, target +/// wake landed before the user's own typical wake, and a 0.5 h need increase +/// could push target wake 55 min LATER by crossing a cycle boundary. Cycle +/// alignment is gone with it: quantizing TIME IN BED to 90-minute sleep cycles +/// was never the thing Kleitman's cycles describe. Metric recommendedWake({ required double bedtimeMinOfDay, required double needSec, + required double typicalEfficiencyPct, }) { - final sleepMin = needSec / 60.0; - final cycles = math.max(1, (sleepMin / 90.0).round()); - final wake = (bedtimeMinOfDay + cycles * 90.0) % 1440.0; + final wake = + (bedtimeMinOfDay + _inBedMin(needSec, typicalEfficiencyPct)) % 1440.0; return Metric( - value: WakeRec(wake), + value: WakeRec(wake < 0 ? wake + 1440.0 : wake), confidence: 0.55, tier: Tier.estimate, - inputs_used: const ['sleep_need', 'bedtime'], - note: '90-minute cycle-aligned wake estimate', + inputs_used: const ['sleep_need', 'bedtime', 'efficiency'], + note: 'wake = bedtime + the time in bed the need and your efficiency imply', ); } @@ -116,6 +134,22 @@ class StrainTarget { }; } +/// A BAND, NOT AN INSTRUCTION — and it must stay that way. +/// +/// nocturnal RMSSD and RHR are the best-validated things this band produces +/// (CCC 0.94, MAPE ~8% vs ECG over 536 nights), which is precisely why turning +/// them into "do not train hard today" is tempting and why the request keeps +/// arriving. the step from "today's ln-RMSSD is below your 7-day rolling mean" +/// to a training prescription requires an effect on training OUTCOME that has +/// not been established outside small athlete cohorts under daily supervision. +/// the inputs validate; the instruction does not. +/// +/// so this returns a range and a `rationale` that describes drivers, and it +/// emits no verb. never "skip your session", never "train hard today", never a +/// session type attached to a recovery state, never a rest day. anything in the +/// imperative mood does not ship. two facts and no verb is the only honest +/// extension. `glassBoxReadiness` sits under the same rule for the same reason +/// — nothing to build there either, it already complies. Metric strainTarget({ required double? recovery0to100, required double? ctl, @@ -184,119 +218,6 @@ Metric strainTarget({ ); } -Metric vo2maxEstimate({ - required double? restingHr, - required double? maxHr, - required Sex sex, - required double? age, -}) { - // Uth-Sørensen-Overgaard-Pedersen 2004: VO2max ≈ 15.3 · (HRmax / HRrest). - // The ratio is only defined for a STRICTLY POSITIVE resting HR — a 0 (the - // package's off-skin sentinel, see types.dart HrSample) divides to Infinity, - // which Metric.toJson emits raw and jsonEncode then throws on. `maxHr <= - // restingHr` does not catch it, so guard the denominator explicitly and - // abstain. Non-finite inputs abstain for the same reason. - if (restingHr == null || - maxHr == null || - !restingHr.isFinite || - !maxHr.isFinite || - restingHr <= 0 || - maxHr <= restingHr) { - return const Metric.absent( - tier: Tier.estimate, - inputs_used: ['resting_hr', 'max_hr'], - note: 'VO2max needs a positive resting HR below HRmax — "—" (never imputed)', - ); - } - final vo2 = 15.3 * (maxHr / restingHr); - return Metric( - value: vo2, - confidence: 0.45, - tier: Tier.estimate, - inputs_used: const ['resting_hr', 'max_hr'], - note: 'Uth-style resting VO2max estimate from HRmax:RHR', - ); -} - -class PhysioAge { - final double physioAge; - final double deltaYears; - const PhysioAge({required this.physioAge, required this.deltaYears}); - Map toJson() => { - 'physio_age': physioAge, - 'delta_years': deltaYears, - }; -} - -Metric physiologicalAge({ - required double chronologicalAge, - required Sex sex, - required double? vo2max, - required double? restingHr, - required double? rmssd, - required double? sleepDurationH, - required double? sleepEfficiency, - required double? dailySteps, -}) { - // ABSTAIN when NOTHING physiological was supplied. `score` starts at the - // chronological age and only the blocks below move it, so with every - // physiological input null this used to return a PRESENT metric reading - // "physioAge == your age, delta 0" — a fabricated result — while claiming six - // inputs it never saw. A physiological age with no physiology in it is not an - // estimate, it is the birth date restated. - final used = [ - if (vo2max != null) 'vo2max', - if (restingHr != null) 'resting_hr', - if (rmssd != null) 'rmssd', - if (sleepDurationH != null) 'sleep_duration', - if (sleepEfficiency != null) 'sleep_efficiency', - if (dailySteps != null) 'steps', - ]; - if (used.isEmpty) { - return const Metric.absent( - tier: Tier.estimate, - inputs_used: ['profile'], - note: 'no physiological input present — "—" (never imputed; ' - 'chronological age alone is not a physiological age)', - ); - } - - var score = chronologicalAge; - if (vo2max != null) { - score -= ((vo2max - 35.0) / 5.0).clamp(-8.0, 8.0); - } - if (restingHr != null) { - score += ((restingHr - 60.0) / 6.0).clamp(-5.0, 8.0); - } - if (rmssd != null) { - score -= ((rmssd - 35.0) / 12.0).clamp(-4.0, 6.0); - } - if (sleepDurationH != null) { - // Deviation from the ~7.5 h optimum ages you in BOTH directions — under- and - // over-sleep both associate with worse outcomes. (Was `(7.5 - h)`, which - // wrongly made oversleep look biologically YOUNGER.) - score += (7.5 - sleepDurationH).abs().clamp(0.0, 3.0); - } - if (sleepEfficiency != null) { - score += ((88.0 - sleepEfficiency) / 6.0).clamp(-2.0, 3.0); - } - if (dailySteps != null) { - score -= ((dailySteps - 7000.0) / 3000.0).clamp(-3.0, 3.0); - } - score = score.clamp(18.0, 95.0); - return Metric( - value: PhysioAge(physioAge: score, deltaYears: score - chronologicalAge), - // Confidence tracks how much physiology actually went in: one input is a - // hint, all six is the intended estimate. - confidence: (0.15 + 0.035 * used.length).clamp(0.15, 0.35), - tier: Tier.estimate, - // inputs_used reports what was ACTUALLY used, never the full menu. - inputs_used: ['profile', ...used], - note: 'directional physiological-age estimate from ${used.length}/6 ' - 'physiological inputs', - ); -} - /// Unbiased (n−1) sample variance about a known mean. 0 for n < 2. double _sampleVar(List xs, double m) { if (xs.length < 2) return 0.0; @@ -332,6 +253,13 @@ class JournalEffect { /// Pooled within-group SD used for [cohensD]; null when not computed. final double? pooledSd; + + /// Two-sided permutation p for the difference of means, and the + /// Benjamini-Hochberg q over the whole (tag × outcome) grid this call + /// returned. Null when the cell was not tested. [meaningful] is gated on q, + /// never on p — see the doc comment. + final double? p; + final double? q; const JournalEffect({ required this.outcome, required this.delta, @@ -343,6 +271,8 @@ class JournalEffect { required this.meaningful, this.cohensD, this.pooledSd, + this.p, + this.q, }); } @@ -358,13 +288,26 @@ class JournalTagCorrelation { /// series of a different length cannot be attributed to dates at all, so it is /// reported as insufficient rather than silently truncated or index-crashed. /// -/// [minEffectPct] and [minCohensD] set the "meaningful" bar. A percentage -/// difference of means alone is NOT evidence: with 2 days per side, two -/// noisy series routinely differ by several percent. The verdict therefore also -/// requires a standardized effect size (Cohen's d = delta / pooled SD ≥ 0.5, -/// Cohen's conventional "medium" effect) so within-group spread is accounted -/// for. When both sides are exactly constant (pooled SD = 0) d is undefined and -/// we require [minNForZeroSpread] observations per side before calling it. +/// [minEffectPct] and [minCohensD] are REPORTING FLOORS, not the verdict. A +/// percentage difference of means alone is NOT evidence: with 2 days per side, +/// two noisy series routinely differ by several percent. A standardized effect +/// size (Cohen's d = delta / pooled SD ≥ 0.5) is barely better, because at n = 2 +/// the sample d is the same noisy Δ over an equally noisy denominator. Simulated +/// under the pre-2026-08-17 rule (`|Δ%| ≥ 3 AND |d| ≥ 0.5`, outcome ~ N(60, 8), +/// no effect at all): a cell was called meaningful **65.5 %** of the time at 2 +/// vs 2 and **42.9 %** at 3 vs 20 — about 14 falsely-meaningful cells on a +/// routine 8-tag × 4-outcome screen. +/// +/// SO THE VERDICT IS A TEST, and the same one the numeric sibling uses: a +/// two-sided permutation p on the difference of means ([permutations] shuffles, +/// seeded so the same journal always gives the same answer), then +/// Benjamini-Hochberg across the WHOLE (tag × outcome) grid this call returns, +/// with `meaningful` gated on q ≤ [fdrAlpha]. At 2 vs 2 there are only six label +/// assignments, so the smallest reachable p is ~0.33 and no such cell can ever +/// be called — which is the point. +/// +/// When both sides are exactly constant (pooled SD = 0) d is undefined and we +/// require [minNForZeroSpread] observations per side before the floor passes. List journalCorrelations({ required List journal, required List dates, @@ -372,54 +315,79 @@ List journalCorrelations({ double minEffectPct = 3.0, double minCohensD = 0.5, int minNForZeroSpread = 3, + double fdrAlpha = 0.10, + int permutations = 999, + int permutationSeed = 20260817, }) { - final allTags = {for (final j in journal) ...j.tags}; + final allTags = {for (final j in journal) ...j.tags}.toList()..sort(); final tagByDate = {for (final j in journal) j.date: j.tags}; - final out = []; + // One row per (tag, outcome) in emission order. Built first, gated second: + // the FDR correction needs the whole family before any verdict can be given. + final rows = <({ + String tag, + String outcome, + double delta, + double? pct, + String higherSide, + int nTagged, + int nUntagged, + bool insufficient, + double? cohensD, + double? pooledSd, + bool floorOk, + double? p, + })>[]; for (final tag in allTags) { - final effects = []; for (final entry in outcomes.entries) { // LENGTH GUARD: `entry.value[i]` used to be indexed by dates.length with // no check, so any outcome list shorter than `dates` threw RangeError. // A misaligned series is not partially usable — we cannot know which // dates the values belong to — so abstain for this outcome. if (entry.value.length != dates.length) { - effects.add( - JournalEffect( - outcome: entry.key, - delta: 0, - pctChange: null, - higherSide: 'neither', - nTagged: 0, - nUntagged: 0, - insufficient: true, - meaningful: false, - ), - ); + rows.add(( + tag: tag, + outcome: entry.key, + delta: 0, + pct: null, + higherSide: 'neither', + nTagged: 0, + nUntagged: 0, + insufficient: true, + cohensD: null, + pooledSd: null, + floorOk: false, + p: null, + )); continue; } final tagged = []; final untagged = []; + final vals = []; + final inGroup = []; for (var i = 0; i < dates.length; i++) { final v = entry.value[i]; if (v == null) continue; final hasTag = tagByDate[dates[i]]?.contains(tag) == true; (hasTag ? tagged : untagged).add(v); + vals.add(v); + inGroup.add(hasTag); } final insufficient = tagged.length < 2 || untagged.length < 2; if (insufficient) { - effects.add( - JournalEffect( - outcome: entry.key, - delta: 0, - pctChange: null, - higherSide: 'neither', - nTagged: tagged.length, - nUntagged: untagged.length, - insufficient: true, - meaningful: false, - ), - ); + rows.add(( + tag: tag, + outcome: entry.key, + delta: 0, + pct: null, + higherSide: 'neither', + nTagged: tagged.length, + nUntagged: untagged.length, + insufficient: true, + cohensD: null, + pooledSd: null, + floorOk: false, + p: null, + )); continue; } final taggedMean = tagged.reduce((a, b) => a + b) / tagged.length; @@ -452,25 +420,49 @@ List journalCorrelations({ tagged.length >= minNForZeroSpread && untagged.length >= minNForZeroSpread); - effects.add( - JournalEffect( - outcome: entry.key, - delta: delta, - pctChange: pct, - higherSide: delta >= 0 ? 'tagged' : 'untagged', - nTagged: tagged.length, - nUntagged: untagged.length, - insufficient: false, - meaningful: bigEnough && separated, - cohensD: d, - pooledSd: pooledSd, - ), - ); + rows.add(( + tag: tag, + outcome: entry.key, + delta: delta, + pct: pct, + higherSide: delta >= 0 ? 'tagged' : 'untagged', + nTagged: tagged.length, + nUntagged: untagged.length, + insufficient: false, + cohensD: d, + pooledSd: pooledSd, + floorOk: bigEnough && separated, + p: _permTwoSampleP(vals, inGroup, permutations, permutationSeed), + )); } - out.add(JournalTagCorrelation(tag, effects)); } - out.sort((a, b) => a.tag.compareTo(b.tag)); - return out; + + final qs = benjaminiHochberg([for (final r in rows) r.p]); + final byTag = >{}; + for (var i = 0; i < rows.length; i++) { + final r = rows[i]; + final q = qs[i]; + byTag.putIfAbsent(r.tag, () => []).add( + JournalEffect( + outcome: r.outcome, + delta: r.delta, + pctChange: r.pct, + higherSide: r.higherSide, + nTagged: r.nTagged, + nUntagged: r.nUntagged, + insufficient: r.insufficient, + meaningful: r.floorOk && q != null && q <= fdrAlpha, + cohensD: r.cohensD, + pooledSd: r.pooledSd, + p: r.p, + q: q, + ), + ); + } + return [ + for (final tag in allTags) + JournalTagCorrelation(tag, byTag[tag] ?? const []) + ]; } // --------------------------------------------------------------------------- @@ -530,11 +522,60 @@ class JournalNumericEffect { /// way. Distinct from a computed-but-weak relationship. final bool insufficient; - /// Strong enough AND separated from zero to be worth showing: |rho| clears - /// the floor and the confidence interval excludes 0. A rho alone is not - /// evidence — over ten days, |rho| ≈ 0.5 arises constantly from noise. + /// Strong enough AND surviving the multiplicity correction to be worth + /// showing: the effect size clears its floor (|rho| ≥ minAbsRho on the dose + /// path, |d| ≥ minCohensD on the 0/1 path) AND [q] ≤ the FDR level. A rho + /// alone is not evidence — over ten days, |rho| ≈ 0.5 arises constantly from + /// noise — and a per-test p is not evidence either once 36 of them run at + /// once. final bool meaningful; + /// This field only ever took the values 0 and 1 on the paired days, so it was + /// routed to a DIFFERENCE OF MEANS, not to a rank correlation. A tick-box is + /// a group membership, not a dose: rho and "slope per unit" on it are a group + /// difference wearing the wrong clothes. On this path [rho], [slopePerUnit] + /// and the rho interval are null and [delta]/[cohensD] carry the answer. + final bool binary; + + /// 0/1 path only: mean(outcome | field = 1) − mean(outcome | field = 0), in + /// the outcome's own units. Null on the dose path. + final double? delta; + + /// 0/1 path only: Cohen's d = [delta] / pooled within-group SD. Null on the + /// dose path, and null when both sides are exactly constant. + final double? cohensD; + + /// 0/1 path only: days with the field at 1, and at 0. Both null otherwise. + final int? nWith; + final int? nWithout; + + /// Two-sided PERMUTATION p for this single test, before any correction — + /// labels reshuffled on the 0/1 path, ranks on the dose path. Exact under H₀ + /// for any marginal and any tie structure, which is what tie-heavy ordinal + /// journal fields need. Null when no test could be run. NOT a publication + /// gate on its own — see [q]. Note it is NOT the p implied by + /// [rhoLow]/[rhoHigh]: that interval is Bonett & Wright, for display. + final double? p; + + /// Benjamini-Hochberg (1995) FDR-adjusted p over THE WHOLE GRID returned by + /// one call — every field × every outcome. This is the number the "meaningful" + /// verdict is gated on. Null when [p] is null. + /// + /// Why it is not optional: 9 built-in numeric fields × 4 outcomes is up to 36 + /// simultaneous tests. At a per-test 0.05 gate that is ~2 findings from pure + /// noise for every user, every load — a machine for manufacturing confident + /// nonsense out of a journal. The ACTUAL family size is 4 × the fields this + /// user really journals (nulls are excluded from m), so it is commonly 4 — + /// which is also the multiplier the discreteness floor is checked against. + final double? q; + + /// MACHINE-READABLE reason when [insufficient] is set for a reason the day + /// count alone does not explain — currently only + /// `need_history:have=N,need=M`, meaning the permutation test's own + /// discreteness floor could not clear the multiplicity correction at N days + /// however strong the effect was, and M days would. Null otherwise. + final String? note; + const JournalNumericEffect({ required this.outcome, required this.rho, @@ -544,38 +585,80 @@ class JournalNumericEffect { required this.n, required this.insufficient, required this.meaningful, + this.binary = false, + this.delta, + this.cohensD, + this.nWith, + this.nWithout, + this.p, + this.q, + this.note, }); } class JournalNumericCorrelation { final String field; final List effects; - const JournalNumericCorrelation(this.field, this.effects); + + /// Days between the journal entry and the outcome it was matched against + /// (MIND-02). 0 = the same day label, +1 = the following morning's outcome. + /// Surfaced so a screen can say WHICH night it means — "yesterday's coffee + /// against last night's HRV" is a different sentence from "today's mood + /// against last night's HRV", and a user who has already seen a finding is + /// entitled to know the alignment changed. + final int lagDays; + const JournalNumericCorrelation(this.field, this.effects, {this.lagDays = 0}); } -/// Average ranks, 1-based, ties sharing their mean rank. +/// PER-FIELD outcome lag, in days (MIND-02). /// -/// Tie handling is not a detail here: journal fields are full of ties (mood is -/// 1–5, most people log the same 2 coffees most days), and ranking ties -/// arbitrarily would invent an ordering the user never reported. -List _averageRanks(List xs) { - final idx = List.generate(xs.length, (i) => i) - ..sort((a, b) => xs[a].compareTo(xs[b])); - final ranks = List.filled(xs.length, 0); - var i = 0; - while (i < idx.length) { - var j = i; - while (j + 1 < idx.length && xs[idx[j + 1]] == xs[idx[i]]) { - j++; - } - // Ranks are 1-based, so positions i..j map to ranks i+1..j+1. - final shared = (i + 1 + j + 1) / 2.0; - for (var k = i; k <= j; k++) { - ranks[idx[k]] = shared; - } - i = j + 1; - } - return ranks; +/// The bug this fixes: `_targetDayWindow` searches back 12 h from midnight, so +/// readiness / rmssd / efficiency labelled date D come from the night ENDING on +/// the morning of D. The journal row is written at bedtime of D and describes +/// the DAYTIME of D. Correlating them at lag 0 compares today's coffee against +/// a night that was over before it was drunk. +/// +/// It is per-field and NEVER a global shift, because the fields point in +/// opposite directions. Mood, sleep quality, soreness and stress are +/// RETROSPECTIVE — they describe the state the finished night produced, and +/// lag 0 is already correct for them; a blanket +1 would break the ones that +/// work today. Caffeine, alcohol, water and steps are BEHAVIOUR during the day +/// and land on the night that follows, so they take +1. +/// +/// A field not listed here keeps lag 0 — a custom field could be either kind, +/// and quietly re-aligning something we cannot classify is the same mistake in +/// the other direction. +/// +/// NO LAG SCAN. Trying {0,+1,+2} triples the grid and multiplies exactly the +/// multiplicity problem the Benjamini-Hochberg correction above just paid to +/// fix. One fixed constant, disclosed. +const Map journalFieldLagDays = { + 'caffeine': 1, + 'caffeine_at_min': 1, + 'alcohol': 1, + 'water': 1, + 'steps': 1, + 'nicotine': 1, + 'late_meal': 1, + 'screen_time': 1, + 'mood': 0, + 'sleep_quality': 0, + 'soreness': 0, + 'stress': 0, + 'energy': 0, +}; + +/// The day label [days] after [label] (`YYYY-MM-DD`), or null when the label is +/// not a date we can shift. UTC arithmetic: a DST boundary must not turn +1 day +/// into +23 h and round to 0. +String? shiftDayLabel(String label, int days) { + if (days == 0) return label; + final d = DateTime.tryParse('${label}T00:00:00Z'); + if (d == null) return null; + final s = d.add(Duration(days: days)); + return '${s.year.toString().padLeft(4, '0')}-' + '${s.month.toString().padLeft(2, '0')}-' + '${s.day.toString().padLeft(2, '0')}'; } /// Pearson correlation. Null when either side has no spread. @@ -597,7 +680,138 @@ double? _pearson(List a, List b) { /// Spearman's rho — Pearson on average ranks, so ties are handled correctly. double? spearmanRho(List a, List b) => - _pearson(_averageRanks(a), _averageRanks(b)); + _pearson(averageRanks(a), averageRanks(b)); + +/// Per-field relationship between numeric journal entries and each outcome. +/// +/// [outcomes] values must be POSITIONALLY ALIGNED to [dates], exactly as in +/// [journalCorrelations]; a series of a different length is reported as +/// insufficient rather than silently truncated. +/// +/// Two-sided permutation p for a difference of means between two labelled +/// groups: reshuffle the labels [b] times and count how often the shuffled +/// |difference| reaches the observed one. +/// +/// Distribution-free, which is the point — the alternative at these sample +/// sizes is a normal approximation to Welch's t over eight days, which is +/// anticonservative exactly where the answer matters. Seeded, so the same +/// journal always yields the same p; the +1s are the standard bias correction +/// that keeps p away from a fabricated 0. +double _permTwoSampleP(List ys, List inGroup, int b, int seed) { + double diff(List g) { + var s1 = 0.0, s0 = 0.0; + var n1 = 0, n0 = 0; + for (var i = 0; i < ys.length; i++) { + if (g[i]) { + s1 += ys[i]; + n1++; + } else { + s0 += ys[i]; + n0++; + } + } + if (n1 == 0 || n0 == 0) return 0.0; + return s1 / n1 - s0 / n0; + } + + final obs = diff(inGroup).abs(); + final rng = math.Random(seed); + final shuffled = [...inGroup]; + var ge = 0; + for (var k = 0; k < b; k++) { + shuffled.shuffle(rng); + if (diff(shuffled).abs() >= obs - 1e-12) ge++; + } + return (ge + 1) / (b + 1); +} + +/// Two-sided permutation p for Spearman's rho: shuffle one rank vector [b] +/// times and count how often |rho| reaches the observed one. +/// +/// This is the GATE, while the Bonett & Wright interval below is the DISPLAY. +/// They are deliberately two different things. The interval's standard error is +/// a simulation result on CONTINUOUS data (Ruscio 2008, J Mod Appl Stat Methods +/// 7(2):416-434, found poorer coverage on ordinal data and bootstrap as good or +/// better) — and journal fields are ordinal by construction: mood is 1–5, +/// soreness is 1–5, coffees are small integers, every one of them tie-heavy. +/// It was also being read as a null-hypothesis p while evaluated at the +/// OBSERVED r, which is the interval SE, not the H₀ SE. Both deviations point +/// the safe way, so this costs power, not honesty — but a permutation p is +/// exact under H₀ for any marginal and any tie structure, so there is no reason +/// to pay for either. Seeded, like [_permTwoSampleP], so the same journal always +/// yields the same p. +double _permSpearmanP(List xs, List ys, int b, int seed) { + final rx = averageRanks(xs); + final ry = averageRanks(ys); + final obs = (_pearson(rx, ry) ?? 0.0).abs(); + final rng = math.Random(seed); + final shuffled = [...ry]; + var ge = 0; + for (var k = 0; k < b; k++) { + shuffled.shuffle(rng); + final r = _pearson(rx, shuffled); + if (r != null && r.abs() >= obs - 1e-12) ge++; + } + return (ge + 1) / (b + 1); +} + +/// n! as a double, +inf on overflow. Only ever compared against a threshold. +double _factorial(int n) { + var r = 1.0; + for (var i = 2; i <= n; i++) { + r *= i; + if (!r.isFinite) return double.infinity; + } + return r; +} + +/// C(n, k) as a double, +inf on overflow. Only ever compared against a +/// threshold, so the saturation is harmless. +double _binom(int n, int k) { + if (k < 0 || k > n) return 0; + var r = 1.0; + for (var i = 1; i <= k; i++) { + r = r * (n - k + i) / i; + if (!r.isFinite) return double.infinity; + } + return r; +} + +/// The SMALLEST p a permutation test on this sample could ever return. +/// +/// A permutation test's null distribution is DISCRETE. For the two-group +/// difference the label vector has only C(n, n₁) distinct assignments; the +/// observed one always reaches its own |difference|, and its complement does too +/// WHEN the split is balanced (with n₁ ≠ n − n₁ no complement exists inside the +/// fixed group size). So the two-sided floor is 1/C(n, n₁), or 2/C(n, n₁) at +/// n₁ = n/2 — not something more shuffles can lower. Drawing only [b] shuffles +/// adds its own 1/(b+1) floor on top. +/// +/// Why it matters: at the n = 8 minimum with a 3/5 split the floor is +/// 1/56 = 0.0179, and after a 4-test BH correction that is q = 0.071 — a binary +/// field at exactly 8 days CANNOT be published however real the effect. At n = 9 +/// it can (1/126 × 4 = 0.032). A test that cannot pass is not a "no", and the +/// silence with no reason attached is the part that was wrong. +double _permPFloor(int n, int n1, int b) { + final c = _binom(n, n1); + final k = (n1 * 2 == n) ? 2.0 : 1.0; + return math.max(1.0 / (b + 1), c <= 0 ? 1.0 : k / c); +} + +/// The smallest number of paired days at which this test could clear [alpha] +/// after a family-of-[m] BH correction, holding the observed group balance. +/// Null if it cannot get there within a season. +int? _needForFloor(int n, int n1, int b, int m, double alpha, + {required bool binary}) { + final p1 = n1 / n; + for (var N = n; N <= n + 90; N++) { + final floor = binary + ? _permPFloor(N, math.max(1, math.min(N - 1, (N * p1).round())), b) + : math.max(1.0 / (b + 1), 2.0 / _factorial(N)); + if (floor * m <= alpha) return N; + } + return null; +} /// Per-field relationship between numeric journal entries and each outcome. /// @@ -609,56 +823,179 @@ double? spearmanRho(List a, List b) => /// requirement because a correlation over a handful of points is close to /// meaningless — with 5 days, |rho| > 0.8 happens by chance often enough to /// fill a screen with confident nonsense. +/// +/// TWO STATISTICS, chosen by what the field actually is: +/// * a field that only ever took 0 and 1 is a TICK BOX — a habit, in this +/// app, since habits are just custom fields with `max == 1`. It gets a +/// difference of means with Cohen's d, the same comparison the tag path +/// runs, because "the difference it made" is the only question a yes/no +/// can answer. A rank correlation and a "slope per unit" on a two-valued +/// field is a group difference wearing the wrong clothes. +/// * anything else carries a dose and gets Spearman. +/// +/// MULTIPLICITY. Every test in the grid this call returns is corrected together +/// by Benjamini-Hochberg at [fdrAlpha], and `meaningful` is gated on the +/// resulting q, never on the raw p. With one field and one outcome the +/// correction is the identity, so a single test behaves exactly as it did. +/// Across the built-in grid (9 fields × 4 outcomes) it is the difference +/// between a screen of findings and a screen of noise. List journalNumericCorrelations({ required List journal, required List dates, required Map> outcomes, int minN = 8, double minAbsRho = 0.35, + double minCohensD = 0.5, + int minPerSide = 3, + double fdrAlpha = 0.05, + int permutations = 999, + int permutationSeed = 20260817, + Map fieldLagDays = journalFieldLagDays, }) { final byDate = >{ for (final d in journal) d.date: d.values, }; + final indexByDate = { + for (var i = 0; i < dates.length; i++) dates[i]: i, + }; final fields = {for (final d in journal) ...d.values.keys}.toList() ..sort(); - JournalNumericEffect none(String outcome, int n) => JournalNumericEffect( - outcome: outcome, - rho: null, - slopePerUnit: null, - rhoLow: null, - rhoHigh: null, - n: n, - insufficient: true, - meaningful: false, - ); + // One row per (field, outcome) in emission order. Built first, gated second: + // the FDR correction needs the whole family before any verdict can be given. + final rows = <({ + String field, + String outcome, + int n, + double? rho, + double? slope, + double? lo, + double? hi, + bool binary, + double? delta, + double? cohensD, + int? nWith, + int? nWithout, + double? p, + double pFloor, + int nSmallerSide, + bool floorOk, + bool insufficient, + })>[]; + + void addNone(String field, String outcome, int n) => rows.add(( + field: field, + outcome: outcome, + n: n, + rho: null, + slope: null, + lo: null, + hi: null, + binary: false, + delta: null, + cohensD: null, + nWith: null, + nWithout: null, + p: null, + pFloor: 1.0, + nSmallerSide: 0, + floorOk: false, + insufficient: true, + )); - final out = []; for (final field in fields) { - final effects = []; for (final entry in outcomes.entries) { final series = entry.value; if (series.length != dates.length) { - effects.add(none(entry.key, 0)); + addNone(field, entry.key, 0); continue; } // Pairwise-complete: a day counts only when the field was recorded AND - // the outcome exists for it. + // the outcome LAG DAYS LATER exists for it. A journal day whose lagged + // outcome is not in the series is DROPPED, which lowers n — never + // back-filled with the same day's outcome, which is the mispairing this + // whole change exists to remove. + final lag = fieldLagDays[field] ?? 0; final xs = []; final ys = []; for (var i = 0; i < dates.length; i++) { final v = byDate[dates[i]]?[field]; - final y = series[i]; - if (v == null || y == null) continue; + if (v == null) continue; + final int? at; + if (lag == 0) { + at = i; + } else { + final label = shiftDayLabel(dates[i], lag); + at = label == null ? null : indexByDate[label]; + } + if (at == null) continue; + final y = series[at]; + if (y == null) continue; xs.add(v); ys.add(y); } final n = xs.length; - final rho = n >= minN ? spearmanRho(xs, ys) : null; + if (n < minN) { + addNone(field, entry.key, n); + continue; + } + + // 0/1 FIELD → difference of means (see the doc comment). + final isBinary = xs.every((v) => v == 0.0 || v == 1.0) && + xs.contains(0.0) && + xs.contains(1.0); + if (isBinary) { + final inGroup = [for (final v in xs) v == 1.0]; + final withIt = []; + final without = []; + for (var i = 0; i < n; i++) { + (inGroup[i] ? withIt : without).add(ys[i]); + } + if (withIt.length < minPerSide || without.length < minPerSide) { + addNone(field, entry.key, n); + continue; + } + final mw = mean(withIt)!; + final mo = mean(without)!; + final delta = mw - mo; + final dof = n - 2; + final pooledVar = dof > 0 + ? ((withIt.length - 1) * _sampleVar(withIt, mw) + + (without.length - 1) * _sampleVar(without, mo)) / + dof + : 0.0; + final pooledSd = pooledVar > 0 ? math.sqrt(pooledVar) : 0.0; + final d = pooledSd > 0 ? delta / pooledSd : null; + rows.add(( + field: field, + outcome: entry.key, + n: n, + rho: null, + slope: null, + lo: null, + hi: null, + binary: true, + delta: delta, + cohensD: d, + nWith: withIt.length, + nWithout: without.length, + p: _permTwoSampleP(ys, inGroup, permutations, permutationSeed), + pFloor: _permPFloor(n, withIt.length, permutations), + nSmallerSide: math.min(withIt.length, without.length), + // Both sides exactly constant leaves d undefined; the per-side floor + // is already the tag path's zero-spread rule, so a real gap between + // two constants still counts. + floorOk: d != null ? d.abs() >= minCohensD : delta.abs() > 0, + insufficient: false, + )); + continue; + } + + final rho = spearmanRho(xs, ys); if (rho == null) { - effects.add(none(entry.key, n)); + addNone(field, entry.key, n); continue; } @@ -676,10 +1013,9 @@ List journalNumericCorrelations({ final zr = 0.5 * math.log((1 + r) / (1 - r)); // Bonett & Wright (2000) standard error, NOT Fisher's 1/sqrt(n−3). // That one is derived for Pearson's r under bivariate normality; ranks - // are neither, and it runs narrow for rho. Since `meaningful` is gated - // on this interval excluding zero, the narrower SE would let weaker - // relationships through — the error points the wrong way for a - // function whose job is to refuse. + // are neither, and it runs narrow for rho. Since the verdict is gated + // on this, the narrower SE would let weaker relationships through — + // the error points the wrong way for a function whose job is to refuse. final se = math.sqrt((1.0 + r * r / 2.0) / (n - 3)); double tanh(double v) { final e = math.exp(2 * v); @@ -689,23 +1025,89 @@ List journalNumericCorrelations({ lo = tanh(zr - 1.96 * se); hi = tanh(zr + 1.96 * se); } + // The GATE is a permutation p, not the interval's z. `normalTwoSidedP(zr + // / se)` read an interval SE evaluated at the observed r as if it were + // the null-hypothesis SE, on a rank statistic whose SE is a simulation + // result for CONTINUOUS data applied to tie-heavy ordinal journal fields. + // See [_permSpearmanP]. The interval above is unchanged and still what a + // screen displays. + final p = + n >= 4 ? _permSpearmanP(xs, ys, permutations, permutationSeed) : null; - final separated = lo != null && hi != null && (lo > 0) == (hi > 0); - effects.add( - JournalNumericEffect( - outcome: entry.key, - rho: rho, - slopePerUnit: theilSen(ys, xs), - rhoLow: lo, - rhoHigh: hi, - n: n, - insufficient: false, - meaningful: rho.abs() >= minAbsRho && separated, - ), - ); + rows.add(( + field: field, + outcome: entry.key, + n: n, + rho: rho, + slope: theilSen(ys, xs), + lo: lo, + hi: hi, + binary: false, + delta: null, + cohensD: null, + nWith: null, + nWithout: null, + p: p, + // A rank permutation has n! relabellings — 40,320 already at the n = 8 + // minimum, where 2/n! is far below the sampling floor — so in practice + // only 1/(b+1) binds. Nothing like the binary path's 1/C(n, n₁). + pFloor: math.max(1.0 / (permutations + 1), 2.0 / _factorial(n)), + nSmallerSide: n, + floorOk: rho.abs() >= minAbsRho, + insufficient: false, + )); } - out.add(JournalNumericCorrelation(field, effects)); } + + final qs = benjaminiHochberg([for (final r in rows) r.p]); + // The BH family size, so each row can be asked whether it COULD have passed. + // Computed once from the p's as they stand: dropping the unreachable rows and + // re-running would only shrink m and let more through, which is the wrong + // direction for a gate whose job is to refuse. + final m = rows.where((r) => r.p != null).length; + final byField = >{}; + for (var i = 0; i < rows.length; i++) { + final r = rows[i]; + final q = qs[i]; + // STRUCTURALLY UNREACHABLE ⇒ ABSTAIN, and say how many days it would take. + // A permutation p cannot go below [_permPFloor]; if even that floor fails + // the correction, this test could not have been published however strong + // the effect, and reporting it as a computed-but-weak "no" is a verdict we + // never actually reached. + final unreachable = r.p != null && r.pFloor * m > fdrAlpha; + final need = unreachable + ? _needForFloor(r.n, r.nSmallerSide, permutations, m, fdrAlpha, + binary: r.binary) + : null; + byField.putIfAbsent(r.field, () => []).add( + JournalNumericEffect( + outcome: r.outcome, + rho: r.rho, + slopePerUnit: r.slope, + rhoLow: r.lo, + rhoHigh: r.hi, + n: r.n, + insufficient: r.insufficient || unreachable, + meaningful: !unreachable && r.floorOk && q != null && q <= fdrAlpha, + note: unreachable + ? 'need_history:have=${r.n},need=${need ?? r.n + 90}' + : null, + binary: r.binary, + delta: r.delta, + cohensD: r.cohensD, + nWith: r.nWith, + nWithout: r.nWithout, + p: r.p, + q: q, + ), + ); + } + + final out = [ + for (final field in fields) + JournalNumericCorrelation(field, byField[field] ?? const [], + lagDays: fieldLagDays[field] ?? 0), + ]; out.sort((a, b) => a.field.compareTo(b.field)); return out; } diff --git a/lib/src/onehz/human/event_detection.dart b/lib/src/onehz/human/event_detection.dart index bb39a34..634a2b5 100644 --- a/lib/src/onehz/human/event_detection.dart +++ b/lib/src/onehz/human/event_detection.dart @@ -14,6 +14,31 @@ // Dose bands follow Pietilä's graded effects (light/moderate/heavy) expressed // as deviations from the PERSONAL baseline (median+MAD), not absolute bpm/ms — // we never hard-code another person's effect sizes onto this user's scale. +// +// TWO OF THE THREE FUNCTIONS IN THIS FILE STAY UNCALLED. that is the decision, +// not a gap in the wiring. `roughNight` is the one that may ship. +// +// `alcoholNightFlag` — do not wire it. it is unsupervised classification with +// no labels entering it: the real export has 0 journal rows, so there is no +// ground truth on this user at all, and an AUC computed over the 8-14 labelled +// nights a keen user might produce has a confidence interval spanning roughly +// 0.5-1.0. printing "AUC 0.81" next to it does not rescue that, it prints the +// refusal in small type. and the signature is admittedly non-specific — see the +// honesty rule above — so the classifier is being asked to do the exact thing +// every party to this concedes it cannot. the product reason outranks both: a +// phone is sometimes shared or seen over a shoulder, a wrong "looks like a +// drinking night" is an accusation, and for anyone in recovery an app that +// comments on drinking is not a neutral object. the MOST this may ever do is +// overlay the nights the user DID log onto the existing rhr/rmssd charts — +// descriptive marks on her own record, no centroid, no distance, no AUC, no +// classification. note the stated reason it "was never wired" is factually +// wrong wherever you read it: it already builds robustBaseline/modZ per user +// and gates on mdc(), so any proposed "replacement" is what it already does. +// +// `NightSignature` — the input struct. it may only be used if it REPLACES +// `multivariateAnomaly`, never alongside it. two anomaly surfaces on the same +// nocturnal channels means the same night gets flagged twice by two different +// rules, and the user has no way to tell they are one observation. import '../types.dart'; import '../util.dart'; diff --git a/lib/src/onehz/human/human.dart b/lib/src/onehz/human/human.dart index 463c24f..744e35c 100644 --- a/lib/src/onehz/human/human.dart +++ b/lib/src/onehz/human/human.dart @@ -16,6 +16,10 @@ export 'circadian_lifestyle.dart'; export 'sleep_regularity.dart'; export 'event_detection.dart'; export 'coaching.dart'; +export 'weekday_effect.dart'; +export 'alertness_forecast.dart'; +export 'session_cost.dart'; +export 'overreaching_conjunction.dart'; // readiness_glassbox.dart is DEPRECATED/INTERNAL (duplicate readiness). The // canonical readiness is wellness/readiness_composite.dart. Kept exported only // for back-compat — do NOT surface as the headline. diff --git a/lib/src/onehz/human/overreaching_conjunction.dart b/lib/src/onehz/human/overreaching_conjunction.dart new file mode 100644 index 0000000..62449fb --- /dev/null +++ b/lib/src/onehz/human/overreaching_conjunction.dart @@ -0,0 +1,138 @@ +// HUMAN — two facts that happen to point the same way (TS-12). +// +// "Your last 7 days of load are 1.6x your last 28, and your resting heart rate +// has been above your usual on 4 of 5 nights." Two numbers the app already +// computes, and a note when they coincide. Silence when they don't. +// +// IT IS A COINCIDENCE DETECTOR, NOT A DIAGNOSIS OF OVERREACHING. Functional +// overreaching is defined by a PERFORMANCE DECREMENT measured over weeks under +// controlled load. That cannot come from a wrist, and nothing here attempts it. +// Never "you are overtraining", never a rest-day instruction, never a score. +// +// ILLNESS, TRAVEL, ALTITUDE, ALCOHOL AND POOR SLEEP PRODUCE THE IDENTICAL +// CONJUNCTION. The copy has to name them; the detector cannot separate them. +// +// NO NEW SCORE AND NO NEW NOTIFICATION CLASS. In-app feed only. A push channel +// would turn this into exactly the forward alerting MT-10 refuses. + +import '../clinical/load_trimp.dart'; +import '../foundations/baseline.dart'; +import '../types.dart'; +import '../util.dart'; + +/// The two facts, side by side. There is deliberately no combined number. +class OverreachingConjunction { + /// Acute (7 d) over chronic (42 d) load, straight off the live [LoadState]. + final double loadRatio; + + /// Nights in the recent window whose resting HR was above baseline by more + /// than the smallest worthwhile change (0.5 × the baseline's robust SD). + final int nightsElevated; + final int nightsConsidered; + + /// True only when BOTH facts hold. False is the normal, quiet state. + final bool bothPointSameWay; + + const OverreachingConjunction({ + required this.loadRatio, + required this.nightsElevated, + required this.nightsConsidered, + required this.bothPointSameWay, + }); + + Map toJson() => { + 'load_ratio': round6(loadRatio), + 'nights_elevated': nightsElevated, + 'nights_considered': nightsConsidered, + 'both_point_same_way': bothPointSameWay, + }; +} + +/// Read the two live outputs and report whether they coincide. +/// +/// [load] the existing `ctlAtlTsb` result — absent in, absent out. [rhrRecent] +/// the last few nights' resting HR, oldest first, nulls for missing nights. +/// [rhrBaselineWindow] the trailing nights the recent ones are judged against; +/// it must NOT contain the recent nights, or the elevation is compared against +/// itself. +/// +/// A night counts as elevated only when it clears the smallest worthwhile +/// change — 0.5 × the robust SD of the baseline window, the same Plews-style SWC +/// clinical/readiness_lnrmssd.dart bands lnRMSSD with. A 1 bpm rise on four +/// nights is arithmetic, not physiology. No dispersion in the baseline means no +/// gate means no elevated nights, ever. +/// +/// THIS WAS `mdc()` UNTIL 2026-08-17, and with no measured typical error mdc() +/// falls back to the trailing scaled MAD, so the bar was 2.77 × the between-night +/// SD it was gating on: 7.93 bpm on 8 real gen4 nights, which NOTHING cleared on +/// the elevated side, so the conjunction could never fire — not "rarely", never. +/// Requiring [minNightsElevated] nights over the SWC is where this detector's +/// conservatism belongs; the per-night bar only has to mean "above your usual". +Metric overreachingConjunction({ + required Metric? load, + required List rhrRecent, + required List rhrBaselineWindow, + double loadRatioThreshold = 1.5, + int minNightsElevated = 4, + int minNightsObserved = 4, + int minBaseline = 14, +}) { + const inputs = ['daily_trimp', 'rhr']; + const note = 'TWO FACTS, no score: recent load against your usual load, and ' + 'recent nights against your usual nights. NOT a diagnosis of ' + 'overreaching — that needs a performance decrement measured over weeks ' + 'and cannot come from a wrist. Illness, travel, altitude, alcohol and a ' + 'run of poor sleep all produce this same pair. In-app only: no ' + 'notification, no rest-day instruction.'; + + final state = load?.value; + if (state == null || state.ctl <= 0) { + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'no load history to compare against. $note', + ); + } + final observed = [ + for (final v in rhrRecent) + if (v != null) v + ]; + if (observed.length < minNightsObserved) { + return Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: needBaselineNote(have: observed.length, need: minNightsObserved), + ); + } + final base = robustBaseline(rhrBaselineWindow, minValid: minBaseline); + final centre = base.center; + final scale = base.scale; + final gate = (scale != null && scale > 0) ? 0.5 * scale : null; + if (centre == null || gate == null || !base.sufficient) { + return Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: needBaselineNote(have: base.nValid, need: minBaseline), + ); + } + + var elevated = 0; + for (final v in observed) { + if (v - centre >= gate) elevated++; + } + final ratio = state.atl / state.ctl; + final both = ratio >= loadRatioThreshold && elevated >= minNightsElevated; + + return Metric( + value: OverreachingConjunction( + loadRatio: ratio, + nightsElevated: elevated, + nightsConsidered: observed.length, + bothPointSameWay: both, + ), + confidence: 0.4, + tier: Tier.estimate, + inputs_used: inputs, + note: note, + ); +} diff --git a/lib/src/onehz/human/percentile_of_you.dart b/lib/src/onehz/human/percentile_of_you.dart index 362621d..bfad5ba 100644 --- a/lib/src/onehz/human/percentile_of_you.dart +++ b/lib/src/onehz/human/percentile_of_you.dart @@ -1,5 +1,7 @@ -// HUMAN LAYER — Percentile-of-you, records, and forgiving streaks. -// Catalog §D "Percentile-of-you + records + streaks" [PUB order-statistics]. +// HUMAN LAYER — Percentile-of-you and records. +// Catalog §D "Percentile-of-you + records" [PUB order-statistics]. The streak +// half of that catalog line is deliberately NOT implemented: no streaks in this +// product, so there is nothing here to wire one up from. // // n-of-1 only: a value's rank is computed against the SAME USER's recent // history (a robust empirical CDF), never a population leaderboard. Records are @@ -13,7 +15,8 @@ import '../types.dart'; import '../util.dart'; import '../foundations/baseline.dart'; -/// Which direction is "better" for a metric (used for record labelling only). +/// Which direction is "better" for a metric. Required by every function here +/// that puts a WORD on a number: a rank is direction-free, a label is not. enum Better { higher, lower, neither } class PercentileOfYou { @@ -35,9 +38,17 @@ class PercentileOfYou { /// definition so ties map to a stable centre rank. /// /// [history] should NOT include [value]. Needs ≥[minN] prior points, else absent. +/// +/// [better] says which end of the scale is the good end, and the [label] is +/// derived from it — there is no default. Without it the ladder hard-coded +/// "high percentile = good", so resting HR (where high is bad) printed the +/// user's WORST-ever night as "among your best" and their best-ever night as +/// "among your lowest". [Better.neither] gets direction-only wording ("among +/// your highest") rather than a verdict. Metric percentileOfYou( double value, List history, { + required Better better, int minN = 14, }) { const inputs = ['metric_history']; @@ -58,7 +69,7 @@ Metric percentileOfYou( } // Midrank empirical CDF: below + half the ties, over n. final pct = 100.0 * (below + 0.5 * equal) / history.length; - final label = _bandLabel(pct); + final label = _bandLabel(pct, better); // Confidence grows with history depth (more of your own data => sturdier CDF). final conf = clamp(history.length / 60.0, 0.3, 0.95); return Metric( @@ -70,12 +81,23 @@ Metric percentileOfYou( ); } -String _bandLabel(double pct) { - if (pct >= 90) return 'among your best'; - if (pct >= 70) return 'better than usual'; - if (pct > 30) return 'typical for you'; - if (pct > 10) return 'below your usual'; - return 'among your lowest'; +String _bandLabel(double pct, Better better) { + if (better == Better.neither) { + // No good end of the scale: describe the position, don't judge it. + if (pct >= 90) return 'among your highest'; + if (pct >= 70) return 'higher than usual'; + if (pct > 30) return 'typical for you'; + if (pct > 10) return 'lower than usual'; + return 'among your lowest'; + } + // Rank on the GOODNESS axis: for a lower-is-better metric the 100th + // percentile is the worst night the user has had, not the best. + final good = better == Better.higher ? pct : 100.0 - pct; + if (good >= 90) return 'among your best'; + if (good >= 70) return 'better than usual'; + if (good > 30) return 'typical for you'; + if (good > 10) return 'worse than usual'; + return 'among your worst'; } class RecordCheck { @@ -133,9 +155,9 @@ Metric personalRecord( note: 'no MDC (degenerate scale) — refusing to claim a record', ); } - final prior = - better == Better.higher ? history.reduce((a, b) => a > b ? a : b) - : history.reduce((a, b) => a < b ? a : b); + final prior = better == Better.higher + ? history.reduce((a, b) => a > b ? a : b) + : history.reduce((a, b) => a < b ? a : b); final beats = better == Better.higher ? value - prior : prior - value; final isRecord = beats > gate; final kind = isRecord ? (better == Better.higher ? 'high' : 'low') : 'none'; @@ -154,56 +176,3 @@ Metric personalRecord( note: 'record only if it beats your prior extreme by > MDC', ); } - -class Streak { - final int current; // current run length (days meeting the goal) - final int best; // longest run in the supplied series - final int graceUsed; // grace days consumed inside the current run - final bool alive; // is the current streak still alive at the last day - const Streak(this.current, this.best, this.graceUsed, this.alive); - Map toJson() => { - 'current': current, - 'best': best, - 'grace_used': graceUsed, - 'alive': alive, - }; -} - -/// Forgiving streak over a chronological [met] boolean series (oldest→newest): -/// `met[i]` = did the day meet the goal. A run survives up to [grace] missed -/// days *within* it (Phillips/Windred-style "don't break on one bad night"), -/// but a miss still does not extend the count. Grace resets per run. -/// -/// Definition: scanning forward, a run continues across a miss as long as the -/// cumulative misses in the run ≤ [grace]; the (grace+1)-th miss ends the run. -/// `current` counts MET days in the live run; `alive` is whether the run that -/// includes the last day has not yet exceeded its grace. -Streak forgivingStreak(List met, {int grace = 1}) { - if (met.isEmpty) return const Streak(0, 0, 0, false); - var best = 0; - var runMet = 0; // met days in the current run - var runMiss = 0; // misses spent in the current run - var runGrace = 0; - // Track the run that reaches the end for `current`/`alive`. - for (var i = 0; i < met.length; i++) { - if (met[i]) { - runMet++; - } else { - if (runMiss < grace) { - runMiss++; - runGrace = runMiss; - } else { - // Run breaks; start fresh AFTER this miss. - best = runMet > best ? runMet : best; - runMet = 0; - runMiss = 0; - runGrace = 0; - } - } - } - best = runMet > best ? runMet : best; - // The trailing run defines current/alive: it is alive iff it never exceeded - // its grace budget (which is always true here — exceeding it starts a new run). - final alive = runMet > 0; - return Streak(runMet, best, runGrace, alive); -} diff --git a/lib/src/onehz/human/readiness_glassbox.dart b/lib/src/onehz/human/readiness_glassbox.dart index d7900dd..d317480 100644 --- a/lib/src/onehz/human/readiness_glassbox.dart +++ b/lib/src/onehz/human/readiness_glassbox.dart @@ -21,7 +21,9 @@ // contribution is the STANDARDIZED deviation w_i·z_i, where z_i is the input's // deviation from its own 50th percentile (in percentile points) — definitional // within the formula we control. Drivers are ranked by |w_i·z_i|; a driver is -// only NAMED when its underlying change clears its MDC. "Why" is therefore +// only NAMED when its underlying change clears the smallest worthwhile change +// (0.5 × the robust SD of that input's own history) — see the gate below for +// why that replaced an MDC that nothing could clear. "Why" is therefore // exactly "this input moved the score by this much", never a claim about the // world. @@ -48,33 +50,48 @@ class GlassBoxInput { class ReadinessBreakdownItem { final String label; - final double percentileOfYou; // 0..100, sign-ORIENTED (higher=better-for-you) + + /// 0..100, sign-ORIENTED (higher=better-for-you). NULL when the input had too + /// little of the user's own history to rank against — absence, never a + /// fabricated rank and never NaN (see [note] for the machine-readable reason). + final double? percentileOfYou; final double weight; final double weightedContribution; // w·(pct-50), signed — narrative driver - final bool pastMdc; // did the underlying change clear its MDC? + + /// Did tonight land outside the user's usual spread — |value − median| ≥ the + /// smallest worthwhile change (0.5 × the robust SD of their own history)? + /// This is the "worth mentioning" bar, NOT a claim the change is real beyond + /// measurement error. Serialized as `past_mdc` for the existing edge reader. + final bool beyondUsualSpread; final bool used; // was the input present + usable + /// Machine-readable reason this input was not used, e.g. + /// `need_baseline:have=2,need=7`. Null when the input WAS used. + final String? note; const ReadinessBreakdownItem({ required this.label, required this.percentileOfYou, required this.weight, required this.weightedContribution, - required this.pastMdc, + required this.beyondUsualSpread, required this.used, + this.note, }); Map toJson() => { 'label': label, - 'percentile_of_you': round6(percentileOfYou), + 'percentile_of_you': + percentileOfYou == null ? null : round6(percentileOfYou!), 'weight': round6(weight), 'weighted_contribution': round6(weightedContribution), - 'past_mdc': pastMdc, + 'past_mdc': beyondUsualSpread, // wire name kept; SWC gate since 2026-08-17 'used': used, + if (note != null) 'note': note, }; } class GlassBoxReadiness { final double score; // 0..100 final List breakdown; // ALWAYS present - final List drivers; // ranked by |w·z|, only NAMED past MDC + final List drivers; // ranked by |w·z|, only NAMED past the SWC final String narrative; // deterministic, definitional "why" final int inputsUsed; const GlassBoxReadiness(this.score, this.breakdown, this.drivers, this.narrative, @@ -97,7 +114,15 @@ class GlassBoxReadiness { /// /// Drivers: contribution_i = weight_i · (orientedPct_i − 50). Ranked by /// magnitude; a driver is NAMED in the narrative only if its raw change cleared -/// its MDC (robust baseline gate). The breakdown lists ALL inputs regardless. +/// the smallest worthwhile change on its own robust baseline. The breakdown +/// lists ALL inputs regardless. +/// +/// THE NARRATIVE DESCRIBES DRIVERS. IT NEVER PRESCRIBES. no "take it easy", no +/// "you're ready for a hard session", no session type attached to a score — +/// same refusal as `strainTarget` in human/coaching.dart, which carries the +/// argument in full. this function already complies; the note is here so the +/// next person to edit the narrative strings knows the constraint before they +/// write one in the imperative mood. @Deprecated( 'DUPLICATE readiness — use readinessComposite (wellness/readiness_composite.dart) ' 'as the canonical/headline recovery. glassBoxReadiness is retained ONLY for its ' @@ -115,17 +140,18 @@ Metric glassBoxReadiness( var wpsum = 0.0; // Σ w·orientedPct over usable inputs var nUsable = 0; - // First pass: percentile + MDC gate per input. + // First pass: percentile + worth-mentioning gate per input. final raw = <_RawItem>[]; for (final inp in inputs) { if (inp.history.length < minHistory) { items.add(ReadinessBreakdownItem( label: inp.label, - percentileOfYou: double.nan, + percentileOfYou: null, // no rank yet — absent, not NaN, not 50 weight: inp.weight, weightedContribution: 0, - pastMdc: false, + beyondUsualSpread: false, used: false, + note: 'need_baseline:have=${inp.history.length},need=$minHistory', )); continue; } @@ -141,11 +167,29 @@ Metric glassBoxReadiness( var pct = 100.0 * (below + 0.5 * equal) / inp.history.length; // Orient: higher should mean better-for-you. final oriented = inp.lowerIsBetter ? 100.0 - pct : pct; - // MDC gate on the RAW change vs robust baseline. + // WORTH-MENTIONING GATE on the RAW change vs robust baseline. + // + // This was `mdc()` until 2026-08-17, and mdc() with no measured typical + // error falls back to the trailing scaled MAD — so the bar was 2.77 × the + // user's own between-night SD, i.e. the very variation it was gating on. On + // 8 real gen4 nights of resting HR that is 7.93 bpm against an observed + // range of 11.70: ONE night in eight cleared it, `drivers` was empty on + // essentially every night, and the narrative fell to "nothing moved beyond + // your normal day-to-day noise" forever. A gate nothing can clear is + // silence, not rigour. + // + // MDC is not wrong, it is the wrong question. It asks "is this change real + // beyond measurement error", which needs a same-subject repeatability + // estimate we do not have (see the audit's STAT-01: do NOT substitute a + // concurrent-validity TE from a paper). The question here is "is this worth + // naming", and the package already has the standard answer for that: the + // smallest worthwhile change, 0.5 × the within-user SD — the same Plews-style + // SWC that clinical/readiness_lnrmssd.dart bands lnRMSSD with. Same window, + // same robust scale, honest bar: 4 of those 8 nights clear it. final base = robustBaseline(inp.history, minValid: minHistory); - final m = mdc(base); + final scale = base.scale; final delta = (base.center == null) ? 0.0 : (inp.value - base.center!); - final pastMdc = m != null && delta.abs() > m; + final beyond = scale != null && scale > 0 && delta.abs() >= 0.5 * scale; final contribution = inp.weight * (oriented - 50.0); items.add(ReadinessBreakdownItem( @@ -153,30 +197,34 @@ Metric glassBoxReadiness( percentileOfYou: oriented, weight: inp.weight, weightedContribution: contribution, - pastMdc: pastMdc, + beyondUsualSpread: beyond, used: true, )); - raw.add(_RawItem(inp.label, contribution, pastMdc)); + raw.add(_RawItem(inp.label, contribution, beyond)); wsum += inp.weight; wpsum += inp.weight * oriented; nUsable++; } if (nUsable == 0 || wsum == 0) { - return const Metric.absent( + var have = 0; + for (final inp in inputs) { + if (inp.history.length > have) have = inp.history.length; + } + return Metric.absent( tier: Tier.estimate, inputs_used: used, - note: 'no readiness input has enough of your history yet', + note: 'need_baseline:have=$have,need=$minHistory', ); } final score = clamp(wpsum / wsum, 0, 100); - // Drivers ranked by |contribution|; only NAME a driver past its MDC. + // Drivers ranked by |contribution|; only NAME a driver past the SWC. final ranked = [...raw]..sort((a, b) => b.c.abs().compareTo(a.c.abs())); final drivers = []; for (final r in ranked) { - if (!r.pastMdc) continue; // never name a sub-MDC mover + if (!r.beyondUsualSpread) continue; // never name a mover inside the noise drivers.add(Driver( r.label, r.c, @@ -202,8 +250,8 @@ Metric glassBoxReadiness( class _RawItem { final String label; final double c; // weighted contribution - final bool pastMdc; - const _RawItem(this.label, this.c, this.pastMdc); + final bool beyondUsualSpread; + const _RawItem(this.label, this.c, this.beyondUsualSpread); } String _buildNarrative(double score, List drivers) { diff --git a/lib/src/onehz/human/session_cost.dart b/lib/src/onehz/human/session_cost.dart new file mode 100644 index 0000000..28ff8a2 --- /dev/null +++ b/lib/src/onehz/human/session_cost.dart @@ -0,0 +1,167 @@ +// HUMAN — what each session type actually cost you the next morning (TS-11). +// +// "On the 14 mornings after a football session, your resting heart rate was a +// median 5 bpm above your baseline." A description of your own history, with +// its sample size printed next to it — never "football is costing you +// recovery", never a recommendation to skip anything. +// +// CONFOUNDING IS TOTAL AND THAT IS WHY THE REFUSALS EXIST. Hard sessions +// cluster with late nights, with alcohol, with stress, with travel. Nothing +// here can separate the session from the evening around it. So: +// * a day with MORE THAN ONE session is dropped entirely — it belongs to no +// single type and averaging it into both is how a gentle type inherits a +// brutal one's mornings; +// * a morning whose night had poor wear coverage is dropped — a resting HR +// from two hours of contact is not a morning; +// * a type under [minSessions] mornings is REFUSED, not shown with a small n; +// * a median move smaller than the metric's MDC is reported as +// indistinguishable from noise rather than as a finding. +// +// Nothing new is measured here. This reuses the daily series and the existing +// robust baseline; it is deliberately NOT built on journalNumericCorrelations, +// which is journal-field-shaped and does not group by session type. + +import '../foundations/baseline.dart'; +import '../types.dart'; +import '../util.dart'; + +/// One session type's typical next-morning move on one metric. +class SessionMorningEffect { + final String sessionType; + + /// Which daily metric moved — 'rhr', 'rmssd', whatever the caller passed. + final String metric; + + /// Mornings that survived every refusal. Print it next to the claim. + final int n; + + /// Median of (that morning's value − that morning's own baseline centre), + /// in the metric's own units and SIGNED. + final double medianDelta; + + /// The metric's minimal detectable change over the same baselines. A + /// [medianDelta] inside this is not a finding. + final double? mdc; + + bool get exceedsMdc => mdc != null && medianDelta.abs() >= mdc!; + + const SessionMorningEffect({ + required this.sessionType, + required this.metric, + required this.n, + required this.medianDelta, + required this.mdc, + }); + + Map toJson() => { + 'session_type': sessionType, + 'metric': metric, + 'n': n, + 'median_delta': round6(medianDelta), + if (mdc != null) 'mdc': round6(mdc!), + 'exceeds_mdc': exceedsMdc, + }; +} + +/// Minimum mornings after a session type before anything is said about it. +/// +/// Ten is not a statistical guarantee, it is a floor under the most obvious +/// nonsense: a "median" over three mornings is one morning with two opinions. +const int sessionCostMinSessions = 10; + +/// Next-morning effect of each session type, one row per type. +/// +/// POSITIONAL ALIGNMENT, contiguous daily series, oldest first: [dates], +/// [values] and [coverage] are index-aligned, and index i+1 is the morning +/// AFTER index i. The caller passes the day series it already has; nothing here +/// parses a date, so a local day label never has to survive a timezone. +/// +/// [sessionTypesByDate] maps a day label to every session that started that +/// day. A day with more than one entry is dropped, not split. +/// +/// Returns ABSENT when no type clears the floor — that is the normal state for +/// months, and it is the honest one. +Metric> sessionMorningEffects({ + required List dates, + required List values, + required String metric, + required Map> sessionTypesByDate, + List? coverage, + int minSessions = sessionCostMinSessions, + int baselineDays = 28, + int minBaseline = 14, + double minCoverage = 0.5, +}) { + final inputs = [metric, 'sessions', if (coverage != null) 'coverage']; + if (values.length != dates.length || + (coverage != null && coverage.length != dates.length)) { + return Metric>.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'dates/values/coverage must be index-aligned daily series', + ); + } + + final deltasByType = >{}; + final mdcsByType = >{}; + + for (var i = 0; i + 1 < dates.length; i++) { + final types = sessionTypesByDate[dates[i]]; + if (types == null || types.length != 1) continue; // none, or ambiguous + final morning = i + 1; + final v = values[morning]; + if (v == null) continue; + if (coverage != null) { + final c = coverage[morning]; + if (c == null || c < minCoverage) continue; + } + // Baseline from the days BEFORE the morning, excluding the morning itself. + // Including it would drag the baseline toward the very value under test. + final from = morning - baselineDays < 0 ? 0 : morning - baselineDays; + final window = [ + for (var k = from; k < morning; k++) + if (values[k] != null) values[k]! + ]; + if (window.length < minBaseline) continue; + final base = robustBaseline(window, minValid: minBaseline); + final centre = base.center; + if (centre == null) continue; + final m = mdc(base); + deltasByType.putIfAbsent(types.single, () => []).add(v - centre); + if (m != null) mdcsByType.putIfAbsent(types.single, () => []).add(m); + } + + final out = []; + for (final e in deltasByType.entries) { + if (e.value.length < minSessions) continue; // refuse, don't show a small n + out.add(SessionMorningEffect( + sessionType: e.key, + metric: metric, + n: e.value.length, + medianDelta: median(e.value)!, + mdc: median(mdcsByType[e.key] ?? const []), + )); + } + out.sort((a, b) => a.sessionType.compareTo(b.sessionType)); + + const note = 'Your own history, per session type, n printed with every row. ' + 'ASSOCIATION ONLY — hard sessions cluster with late nights, alcohol, ' + 'stress and travel, and none of that is separable here. Never a claim ' + 'that a session type is costing you recovery, never a suggestion to ' + 'skip one. Multi-session days and low-coverage nights are excluded.'; + + if (out.isEmpty) { + return Metric>.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'need_sessions:need=$minSessions per type. $note', + ); + } + return Metric>( + value: out, + confidence: 0.4, + tier: Tier.estimate, + inputs_used: inputs, + note: note, + ); +} diff --git a/lib/src/onehz/human/sleep_regularity.dart b/lib/src/onehz/human/sleep_regularity.dart index 301f5d2..1cda870 100644 --- a/lib/src/onehz/human/sleep_regularity.dart +++ b/lib/src/onehz/human/sleep_regularity.dart @@ -1,79 +1,45 @@ -// HUMAN LAYER — Sleep Regularity Index + sleep debt vs personal need. -// Catalog §A: True SRI [PUB Phillips 2017; Windred 2023] and sleep debt -// vs personal need (Kitamura 2016 OSD) [PUB]. +// HUMAN LAYER — your longer unconstrained nights vs your habitual sleep. // -// SRI = 200·(P_agreement) − 100, where P_agreement is the probability that a -// person is in the SAME state (asleep/awake) at two time-points exactly 24 h -// apart, averaged over all epochs. This is the TRUE epoch-by-epoch concordance, -// NOT the SD-of-midsleep shortcut (catalog explicitly forbids the shortcut). +// SRI does NOT live here despite the file name — the live one is `phillipsSri` +// in sleep/sri.dart. This file used to carry a second SRI header and an unused +// result class after the implementation was deleted; both are gone. +// +// THIS IS NOT KITAMURA'S OSD, and it used to say it was (SLP-03). Kitamura 2016 +// (PMC5075948) defines optimal sleep duration as the ASYMPTOTE of an +// exponential-decay fit to PSG total sleep time across NINE CONSECUTIVE NIGHTS +// AT 12 h TIME IN BED: OSD 8.41 ± 0.18 h, habitual 7.37 ± 0.27 h, potential debt +// 1.04 ± 0.24 h, with night 1 at 10.59 ± 0.19 h — a 3.22 h rebound that then +// decays. The paper's whole point is that UNCONSTRAINED SLEEP IS NOT OSD; it +// understates it by about an hour. +// +// What we actually compute is `percentile(freeNightSleepH, 75)` over ordinary +// ad-lib nights — a habitual-sleep percentile. So the "debt" it yields is +// systematically near zero or negative, and downstream (`crossday_pipeline` +// clamps this into [7.0, 9.5] h and floors debt at 0.0) what most users see is +// the clamp, not a measurement. Reported as what it is: how long your longer +// unconstrained nights run, against your recent typical night. +// +// Doing it properly needs a saturating fit across a RUN of consecutive free +// days, and a refusal when no such run exists — not a wider percentile. // -// Sleep debt: Kitamura's "optimal sleep duration" is estimated from the -// rebound on unconstrained (free) nights; debt = OSD − recent habitual sleep. // Returns null when there is no free night yet, rather than assuming a generic // 8 h target. import '../types.dart'; import '../util.dart'; -class Sri { - final double sri; // -100..100, higher = more regular - final int epochsPerDay; // epochs in 24 h (sample rate) - final int comparedPairs; // # of (t, t+24h) comparisons used - final String band; // coarse within-context label - const Sri(this.sri, this.epochsPerDay, this.comparedPairs, this.band); - Map toJson() => { - 'sri': round6(sri), - 'epochs_per_day': epochsPerDay, - 'compared_pairs': comparedPairs, - 'band': band, - }; -} - -/// True Phillips Sleep Regularity Index from a CONTIGUOUS binary sleep/wake -/// vector. `asleep[i]` = is the person asleep in epoch i. [epochsPerDay] is the -/// number of epochs spanning 24 h (e.g. 1440 for 1-min epochs, 96 for 15-min). -/// -/// We compare every epoch with the epoch exactly one day later and score -/// agreement. Needs strictly more than one day of data. -Metric sleepRegularityIndex(List asleep, {required int epochsPerDay}) { - const inputs = ['sleep_wake_binary']; - if (epochsPerDay <= 0 || asleep.length <= epochsPerDay) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'SRI needs more than one full day of sleep/wake epochs', - ); - } - var agree = 0; - final pairs = asleep.length - epochsPerDay; - for (var i = 0; i < pairs; i++) { - if (asleep[i] == asleep[i + epochsPerDay]) agree++; - } - final p = agree / pairs; - final sri = 200.0 * p - 100.0; - final band = sri >= 80 - ? 'very regular' - : sri >= 60 - ? 'regular' - : sri >= 40 - ? 'somewhat irregular' - : 'irregular'; - // Confidence scales with how many days were compared. - final days = pairs / epochsPerDay; - return Metric( - value: Sri(sri, epochsPerDay, pairs, band), - confidence: clamp(days / 7.0, 0.3, 0.95), - tier: Tier.high, - inputs_used: inputs, - note: 'epoch-by-epoch 24-h concordance (true SRI, not SD-of-midsleep)', - ); -} - class SleepDebt { - final double? osdHours; // estimated personal optimal sleep duration (h) + /// p75 of your UNCONSTRAINED nights (h) — "where your longer free nights + /// land". NOT Kitamura's optimal sleep duration; see the file header. + final double? osdHours; final double habitualHours; // recent habitual sleep (h) - final double? debtHours; // OSD − habitual (positive = under-slept) - final bool hasFreeNight; // could we estimate OSD honestly? + + /// [osdHours] − [habitualHours]. The gap between your longer free nights and + /// your typical one, not a sleep debt in Kitamura's sense — an ad-lib + /// percentile understates OSD by ~1 h, so this runs near zero by + /// construction. + final double? debtHours; + final bool hasFreeNight; // did we have any unconstrained night at all? const SleepDebt( this.osdHours, this.habitualHours, this.debtHours, this.hasFreeNight); Map toJson() => { @@ -84,13 +50,14 @@ class SleepDebt { }; } -/// Sleep debt vs personal need (Kitamura OSD). +/// Your longer unconstrained nights vs your habitual night. /// /// [recentSleepH] recent nightly sleep durations (h), oldest→newest. /// [freeNightSleepH] sleep durations on UNCONSTRAINED nights (no alarm / free -/// days), used to estimate the personal optimal sleep duration as their rebound -/// plateau. With no free nights we report habitual sleep but DECLINE to claim a -/// debt (honest: "no free night yet"). +/// days). We take their 75th percentile. With no free nights we report habitual +/// sleep but DECLINE to compare (honest: "no free night yet"). +/// +/// NOT Kitamura's OSD — see the file header for what that would require. Metric sleepDebt( List recentSleepH, List freeNightSleepH, { @@ -115,8 +82,9 @@ Metric sleepDebt( note: 'no free night yet — cannot honestly estimate your sleep need', ); } - // OSD ≈ the rebound plateau: the upper part of free-night durations (75th - // percentile is a robust "when unconstrained, this is where you settle"). + // The upper part of free-night durations: "when unconstrained, this is where + // your longer nights land". A percentile of ad-lib nights, NOT a rebound + // asymptote — see the file header. final osd = percentile(freeNightSleepH, 75)!; final debt = osd - habitual; return Metric( @@ -124,6 +92,8 @@ Metric sleepDebt( confidence: clamp(freeNightSleepH.length / 5.0, 0.3, 0.85), tier: Tier.high, inputs_used: inputs, - note: 'OSD from free-night rebound (Kitamura); debt = need − habitual', + note: 'p75 of your unconstrained nights vs your habitual night — a ' + 'percentile of ad-lib sleep, NOT a measured sleep need (that needs a ' + 'run of consecutive free days and a saturating fit)', ); } diff --git a/lib/src/onehz/human/weekday_effect.dart b/lib/src/onehz/human/weekday_effect.dart new file mode 100644 index 0000000..2f7fdd1 --- /dev/null +++ b/lib/src/onehz/human/weekday_effect.dart @@ -0,0 +1,259 @@ +// HUMAN — does one day of the week actually cost you? (MIND-12) +// +// The question is easy to ask and almost impossible to answer honestly. Take +// any 90 days of any daily metric, split by weekday, and one of the seven will +// look worse — that is what seven groups DO. Publishing that day is a machine +// for manufacturing weekday superstitions, and it will do it on pure noise for +// every user, forever. +// +// So the gate is the feature, and it has two stages: +// +// 1. KRUSKAL-WALLIS OMNIBUS across all seven groups. Rank-based, so a couple +// of wrecked nights do not carry it, and it asks one question — "do these +// seven groups differ at all?" — instead of seven. +// 2. Only if that clears: a PERMUTATION TEST ON THE MAX DEVIATION. The +// statistic is the largest |group median − overall median| over the seven, +// and its null distribution is built by reshuffling the weekday labels. A +// max over seven groups has a null distribution nothing like a single +// group's, and permuting is how the 7-way selection gets paid for rather +// than pocketed. +// +// Both p-values are permutation p-values off the SAME shuffles, so there is one +// loop and no chi-square table. +// +// HONESTY CEILING: purely descriptive and totally confounded. Saturday is not a +// cause, it is a container for everything you do on Saturdays. No advice +// attaches to this, ever, and nothing here licenses telling anyone to change a +// weekend. Under 8 weeks it is absent, not weak. + +import 'dart:math' as math; + +import '../types.dart'; +import '../util.dart'; + +/// One weekday's slice, and the two-stage verdict over all seven. +class WeekdayEffect { + /// Observations per weekday, `DateTime.weekday` keys (1 = Mon … 7 = Sun). + final Map nByWeekday; + + /// Median of the outcome per weekday, same keys. + final Map medianByWeekday; + + /// Median over every day, the reference the deviations are measured from. + final double overallMedian; + + /// Kruskal-Wallis H (tie-corrected). Reported for auditability; the verdict + /// rides on [omnibusP], not on H. + final double kruskalH; + + /// Permutation p for the omnibus — stage 1. + final double omnibusP; + + /// The weekday with the largest |median − [overallMedian]|. + final int peakWeekday; + + /// That deviation, SIGNED, in the outcome's own units. + final double peakDelta; + + /// Permutation p for the max deviation — stage 2, the one that pays for + /// having looked at seven days and picked the worst. + final double peakP; + + /// Both stages cleared. The ONLY field a screen may gate on. + final bool meaningful; + + const WeekdayEffect({ + required this.nByWeekday, + required this.medianByWeekday, + required this.overallMedian, + required this.kruskalH, + required this.omnibusP, + required this.peakWeekday, + required this.peakDelta, + required this.peakP, + required this.meaningful, + }); + + Map toJson() => { + 'n_by_weekday': { + for (final e in nByWeekday.entries) '${e.key}': e.value + }, + 'median_by_weekday': { + for (final e in medianByWeekday.entries) '${e.key}': round6(e.value) + }, + 'overall_median': round6(overallMedian), + 'kruskal_h': round6(kruskalH), + 'omnibus_p': round6(omnibusP), + 'peak_weekday': peakWeekday, + 'peak_delta': round6(peakDelta), + 'peak_p': round6(peakP), + 'meaningful': meaningful, + }; +} + +/// Kruskal-Wallis H over pre-computed [ranks] split by [group] (0..6). +double _kruskalH(List ranks, List group, int k) { + final n = ranks.length; + final sums = List.filled(k, 0); + final counts = List.filled(k, 0); + for (var i = 0; i < n; i++) { + sums[group[i]] += ranks[i]; + counts[group[i]]++; + } + var acc = 0.0; + for (var g = 0; g < k; g++) { + if (counts[g] == 0) continue; + acc += sums[g] * sums[g] / counts[g]; + } + return 12.0 / (n * (n + 1)) * acc - 3.0 * (n + 1); +} + +/// Largest |group median − [overall]|, and which group it was. +({double delta, int group}) _maxDeviation( + List values, + List group, + int k, + double overall, +) { + final buckets = List>.generate(k, (_) => []); + for (var i = 0; i < values.length; i++) { + buckets[group[i]].add(values[i]); + } + var best = 0.0; + var bestG = 0; + for (var g = 0; g < k; g++) { + if (buckets[g].isEmpty) continue; + final d = median(buckets[g])! - overall; + if (d.abs() > best.abs()) { + best = d; + bestG = g; + } + } + return (delta: best, group: bestG); +} + +/// Which day of the week costs you — omnibus first, then the max deviation. +/// +/// [dates] are `yyyy-MM-dd` LOCAL day labels and [values] is positionally +/// aligned to them; a null value is a day with no measurement and is dropped. +/// The weekday comes off the local label, never off an epoch — mixing the two +/// is how a Sunday night becomes a Monday. +/// +/// [minPerWeekday] and [minSpanDays] together are the "≥ 8 weeks" floor. Both +/// are needed: eight weeks of data with three Sundays in it cannot say anything +/// about Sundays, and 56 Mondays in a row is not eight weeks of anything else. +/// +/// [permutations] trades runtime for p resolution; the smallest p this can +/// report is 1/(permutations+1), which is the honest floor for a permutation +/// test and not a rounding artefact. [seed] fixes the shuffles so the same +/// history always gives the same answer. +Metric weekdayEffect( + List dates, + List values, { + int minPerWeekday = 5, + int minSpanDays = 56, + double alpha = 0.05, + int permutations = 999, + int seed = 20260817, +}) { + const inputs = ['metric_series']; + if (dates.length != values.length) { + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'misaligned_series', + ); + } + + final xs = []; + final group = []; // 0 = Monday … 6 = Sunday + final days = []; + final cal = calendarDays(dates); + for (var i = 0; i < dates.length; i++) { + final v = values[i]; + if (v == null || !v.isFinite) continue; + final d = DateTime.tryParse(dates[i]); + if (d == null) continue; + xs.add(v); + group.add(d.weekday - 1); + days.add(cal[i]); + } + + final counts = List.filled(7, 0); + for (final g in group) { + counts[g]++; + } + final span = + days.isEmpty ? 0 : days.reduce(math.max) - days.reduce(math.min) + 1; + final thin = counts.indexWhere((c) => c < minPerWeekday); + if (span < minSpanDays || thin >= 0) { + return Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'need_history:span=${span}d/${minSpanDays}d,' + 'min_per_weekday=${counts.reduce(math.min)}/$minPerWeekday', + ); + } + + final overall = median(xs)!; + // Ranks are invariant to a label shuffle, so they are computed ONCE and every + // permutation just re-partitions them. + final ranks = averageRanks(xs); + final n = xs.length; + + // Tie correction: a constant divisor, so it cannot move the permutation p — + // it only makes the reported H comparable to a published one. + final tieCounts = {}; + for (final x in xs) { + tieCounts[x] = (tieCounts[x] ?? 0) + 1; + } + var tieSum = 0.0; + for (final t in tieCounts.values) { + tieSum += t * t * t - t; + } + final tieDiv = 1.0 - tieSum / (n * n * n - n); + final hObs = _kruskalH(ranks, group, 7); + final devObs = _maxDeviation(xs, group, 7, overall); + + final rng = math.Random(seed); + final shuffled = [...group]; + var hGe = 0, dGe = 0; + for (var b = 0; b < permutations; b++) { + shuffled.shuffle(rng); + if (_kruskalH(ranks, shuffled, 7) >= hObs - 1e-12) hGe++; + if (_maxDeviation(xs, shuffled, 7, overall).delta.abs() >= + devObs.delta.abs() - 1e-12) { + dGe++; + } + } + final omnibusP = (hGe + 1) / (permutations + 1); + final peakP = (dGe + 1) / (permutations + 1); + + return Metric( + value: WeekdayEffect( + nByWeekday: {for (var g = 0; g < 7; g++) g + 1: counts[g]}, + medianByWeekday: { + for (var g = 0; g < 7; g++) + g + 1: median([ + for (var i = 0; i < n; i++) + if (group[i] == g) xs[i] + ])! + }, + overallMedian: overall, + kruskalH: tieDiv > 0 ? hObs / tieDiv : hObs, + omnibusP: omnibusP, + peakWeekday: devObs.group + 1, + peakDelta: devObs.delta, + peakP: peakP, + // BOTH stages. Either alone is a weekday superstition with a p-value + // stapled to it. + meaningful: omnibusP <= alpha && peakP <= alpha, + ), + confidence: clamp(0.25 + 0.25 * (span / 365.0), 0.25, 0.5), + tier: Tier.estimate, + inputs_used: inputs, + note: 'descriptive weekday split, Kruskal-Wallis omnibus then a ' + 'permutation test on the max deviation. a weekday is a container for ' + 'what you do on it, never a cause — no advice attaches to this.', + ); +} diff --git a/lib/src/onehz/motion/energy_fusion.dart b/lib/src/onehz/motion/energy_fusion.dart index fb1aa5c..0091e37 100644 --- a/lib/src/onehz/motion/energy_fusion.dart +++ b/lib/src/onehz/motion/energy_fusion.dart @@ -19,6 +19,33 @@ // calibration is supplied (restingHr + maxHr), in which case the HR branch // uses %HR-reserve (still an estimate, ESTIMATE tier). // * Never asserts absolute EE without calibration. +// +// --------------------------------------------------------------------------- +// MT-05 OFFLINE VALIDATION — 2026-08-17. STILL UNCALLED, AND IT MUST STAY THAT +// WAY UNTIL THE THRESHOLDS MOVE. The proposal was to use the branch label to +// split the day's minutes and SUBTRACT the non-locomotor ones from active +// energy. Run over a real gen4 capture (671k 1 Hz rows, 9 days) against the +// user's own LABELLED sessions, with Brage's defaults below and the +// minute-majority aggregation the proposal calls for: +// +// labelled 83-min WALK -> 17-21% of its minutes called NON-LOCOMOTOR +// labelled 50-min RUN -> 22% +// labelled TENNIS -> 35-83% +// +// So it eats a fifth of a known walk, and a fifth of a known walk's calories is +// what the wiring would delete. A sweep (accelHi 0.15->0.04, hrHi 0.4/0.6) can +// push the walk down to 4-5%, but that is a fit to one person's nine days on +// one device family, and one unlabelled session in the same capture swings +// between 8% and 100% non-locomotor across the same grid. +// +// The root cause is not the numbers, it is the input: Brage's branch thresholds +// were developed on high-rate hip/chest accelerometry, and our accel channel is +// ONE GRAVITY VECTOR PER SECOND — a low-passed orientation, not raw +// acceleration. |‖a‖−1| computed off it is a different quantity from ENMO and +// no threshold sweep converts one into the other. Fixing this needs a motion +// input the branch model can actually read, or per-user calibration (which is +// what Brage's own equation assumed), not a constant. +// --------------------------------------------------------------------------- import '../types.dart'; import '../util.dart'; @@ -95,8 +122,7 @@ Metric branchedEnergyFusion( note: 'HR/ENMO/ts must be equal-length and time-aligned', ); } - final calibrated = - restingHr != null && maxHr != null && maxHr > restingHr; + final calibrated = restingHr != null && maxHr != null && maxHr > restingHr; final pts = []; var load = 0.0; var usable = 0; diff --git a/lib/src/onehz/motion/enmo.dart b/lib/src/onehz/motion/enmo.dart index 6b45931..4398581 100644 --- a/lib/src/onehz/motion/enmo.dart +++ b/lib/src/onehz/motion/enmo.dart @@ -57,20 +57,34 @@ class MotionMinute { final double tsMinStartMs; // wall-clock start of the minute (ms) final int nSamples; // valid samples that fed this minute - /// ⚠️ MEANINGLESS ON THE WHOOP 1 Hz SUBSTRATE — diagnostics only. + /// ⚠️ SECOND-CHOICE ON THE WHOOP 1 Hz SUBSTRATE, and USELESS against a FIXED + /// gravity reference — prefer [dynAmp] for any decision. /// - /// ENMO is `mean(max(0, ‖a‖ − gRef))`, which assumes ‖a‖ carries dynamic - /// acceleration. The band's 1 Hz historical record does NOT: it ships a fused - /// gravity/orientation vector (see [dynAmp]). Measured over 269,486 real - /// samples, ‖a‖ sits at p50 = 1.027 g with only 0.030% above 1.3 g — during - /// the single most vigorous minute of a day it was 1.033 g ± 0.006. + /// ENMO is `mean(max(0, ‖a‖ − gRef))`, so it is only as good as `gRef`. The + /// band's 1 Hz record is a heavily smoothed, largely gravity-dominated + /// vector, and the per-family gravity offset is the same order as the signal: + /// measured over the FULL 671,847-row gen4 corpus, p50 ‖a‖ = 1.0239 g against + /// a calibrated gRef of 1.0277 (MG 1.0014, W5 0.9977). Subtract a hard 1.0 + /// and you are reading calibration, not movement. /// - /// So on this substrate ENMO reduces to roughly `1.03 − gRef`: a pure - /// calibration artifact carrying ZERO signal. That is precisely why an early - /// step estimator built on it reported 42,155 steps at gRef 0.97 and 0 at - /// 1.02. Do NOT threshold this, and do NOT feed it to anything expecting - /// accelerometry (Brage fusion, MET/cut-point models). It stays only because - /// a HIGH-RATE source (the 100 Hz live stream) does carry real accel. + /// CORRECTED 2026-08-17 (audit MOT-07). This docstring used to say ‖a‖ was + /// "only 0.030 % above 1.3 g" and that ENMO carried "ZERO signal". Both are + /// wrong, and they were being cited as the reason real capability stayed + /// unwired. Re-measured over the full corpus: **2,018 samples (0.300 %) + /// exceed 1.3 g**, 419 exceed 1.5 g, 22 exceed 2 g, max 3.0437 g. Per-minute + /// ENMO against the auto-calibrated reference is p50 0.0050 / p75 0.0151 / + /// p90 0.0271 / p99 0.1184 / max 0.3802 g, and 391 minutes (3.49 %) clear + /// 0.05 g — the dominant-wrist light-activity boundary of the Hildebrand + /// cut-point family. Association with the same minute's mean HR: + /// **Spearman(ENMO, HR) = 0.244** against **Spearman(dynAmp, HR) = 0.689** + /// (Spearman(ENMO, dynAmp) = 0.207). So ENMO carries roughly a THIRD of + /// dynAmp's association with effort — not zero. + /// + /// What stays true: ENMO is calibration-FRAGILE where dynAmp is + /// calibration-INVARIANT, which is why an early step estimator built on it + /// reported 42,155 steps at gRef 0.97 and 0 at 1.02. Threshold it only + /// against a reference calibrated on the SAME data ([EnmoResult.gRef]), never + /// against a literal 1.0. final double enmo; /// ⚠️ Same caveat as [enmo] on the 1 Hz substrate — see above. @@ -188,6 +202,7 @@ EnmoResult enmoSeries( double? gRef, int minSamplesPerMinute = 30, double gravityWindowS = defaultGravityWindowS, + int? expectedMinutes, }) { final valid = samples.where((s) => s.valid).toList() ..sort((a, b) => a.tsMs.compareTo(b.tsMs)); @@ -268,7 +283,14 @@ EnmoResult enmoSeries( dynAmp, )); } - final coverage = minutes.isEmpty ? 0.0 : covered / minutes.length; + // COVERAGE DENOMINATOR. `minutes` holds only the minutes that had at least + // one sample, so `covered / minutes.length` reported 1.0 for a day worn 4 h + // out of 24. Divide by the elapsed minute SPAN instead, which at least counts + // interior holes; pass [expectedMinutes] (e.g. 1440 for a calendar day) to + // count the unworn ends too. + final spanMinutes = minutes.isEmpty ? 0 : keys.last - keys.first + 1; + final denom = expectedMinutes ?? spanMinutes; + final coverage = denom <= 0 ? 0.0 : clamp(covered / denom, 0.0, 1.0); return EnmoResult(ref, minutes, coverage); } @@ -277,10 +299,13 @@ EnmoResult enmoSeries( /// minute: sedentary / light / moderate / vigorous, by quartile of the user's /// own moving (ENMO>0) distribution. Sedentary is anything at/near zero ENMO. class IntensityBands { - /// percentile cut-points (g) on the user's moving distribution - final double lightCut; - final double moderateCut; - final double vigorousCut; + /// percentile cut-points (g) on the user's moving distribution. + /// NULL when there were too few moving minutes to set personal cut-points — + /// the labels are still valid (sedentary/light), the cut-points are simply + /// not yet knowable. Never NaN. + final double? lightCut; + final double? moderateCut; + final double? vigorousCut; final List labels; // per input minute final Map minutesInBand; const IntensityBands( @@ -297,9 +322,25 @@ class IntensityBands { /// Cut-points are personal percentiles (50/75/90) of the user's MOVING /// minutes (ENMO above [sedentaryEnmo]); minutes at/under that floor are /// "sedentary". Honest: this is percentile-of-you, never a MET threshold. +/// +/// THE CUT-POINTS MUST BE FROZEN. Without [frozenCuts] this takes percentiles +/// OF ITS OWN INPUT, which means fed one day it labels the top decile of a rest +/// day "vigorous" and a deconditioning period silently lowers the vigorous +/// threshold to meet it — the same defect `personalDynFloor` is already frozen +/// to avoid. Pass cut-points established once over a pooled history and +/// persisted; the self-percentile path stays only for computing that pool the +/// first time (and for tests), and its note says so. +/// +/// SO: THE CONDITION FOR CALLING THIS AT ALL is that [frozenCuts] exists and is +/// persisted. it has zero callers today for exactly that reason — nothing in +/// edge stores a cut-point triple yet. wiring it on the self-percentile path +/// "just to see it on screen" ships a label that redefines itself every time +/// the user's week changes, which is worse than no label. build the pool and +/// the column first, then call this with them. Metric relativeIntensityBands( List enmoPerMin, { double sedentaryEnmo = 0.01, + ({double light, double moderate, double vigorous})? frozenCuts, }) { const inputs = ['enmo_per_min']; if (enmoPerMin.isEmpty) { @@ -309,19 +350,47 @@ Metric relativeIntensityBands( note: 'no ENMO minutes', ); } + if (frozenCuts != null) { + // Monotone or the labelling is nonsense (a "vigorous" cut under the + // "moderate" one makes every moderate minute vigorous). Refuse rather than + // reorder them: whatever produced them is wrong and should hear about it. + if (!(frozenCuts.light < frozenCuts.moderate && + frozenCuts.moderate < frozenCuts.vigorous)) { + return const Metric.absent( + tier: Tier.relative, + inputs_used: inputs, + note: 'frozen cut-points not strictly increasing', + ); + } + return _labelWithCuts( + enmoPerMin, + sedentaryEnmo, + frozenCuts.light, + frozenCuts.moderate, + frozenCuts.vigorous, + // The anchoring quality belongs to whoever froze the cuts, not to this + // day's minute count, so it is not re-derived from this input. + 0.8, + 'RELATIVE within-user intensity vs FROZEN personal cut-points; NOT METs', + ); + } final moving = enmoPerMin.where((e) => e > sedentaryEnmo).toList(); // Need a moving distribution to set personal cut-points. if (moving.length < 4) { final labels = [ for (final e in enmoPerMin) e > sedentaryEnmo ? 'light' : 'sedentary' ]; - final counts = {'sedentary': 0, 'light': 0, 'moderate': 0, 'vigorous': 0}; + final counts = { + 'sedentary': 0, + 'light': 0, + 'moderate': 0, + 'vigorous': 0 + }; for (final l in labels) { counts[l] = counts[l]! + 1; } return Metric( - value: IntensityBands( - double.nan, double.nan, double.nan, labels, counts), + value: IntensityBands(null, null, null, labels, counts), confidence: 0.25, tier: Tier.relative, inputs_used: inputs, @@ -329,9 +398,30 @@ Metric relativeIntensityBands( 'too few moving minutes for personal cut-points; RELATIVE, not METs', ); } - final light = percentile(moving, 50)!; - final moderate = percentile(moving, 75)!; - final vigorous = percentile(moving, 90)!; + return _labelWithCuts( + enmoPerMin, + sedentaryEnmo, + percentile(moving, 50)!, + percentile(moving, 75)!, + percentile(moving, 90)!, + // confidence scales with how much moving data anchors the percentiles. + clamp(moving.length / 60.0, 0.3, 0.8), + 'RELATIVE within-user intensity (50/75/90th moving pct OF THIS INPUT — ' + 'cut-points not frozen); NOT METs', + ); +} + +/// The labelling half, shared by the frozen-cut and self-percentile paths so +/// the two can never band the same minute differently. +Metric _labelWithCuts( + List enmoPerMin, + double sedentaryEnmo, + double light, + double moderate, + double vigorous, + double confidence, + String note, +) { final labels = []; final counts = { 'sedentary': 0, @@ -353,13 +443,11 @@ Metric relativeIntensityBands( labels.add(l); counts[l] = counts[l]! + 1; } - // confidence scales with how much moving data anchors the percentiles. - final conf = clamp(moving.length / 60.0, 0.3, 0.8); return Metric( value: IntensityBands(light, moderate, vigorous, labels, counts), - confidence: conf, + confidence: confidence, tier: Tier.relative, - inputs_used: inputs, - note: 'RELATIVE within-user intensity (50/75/90th moving pct); NOT METs', + inputs_used: const ['enmo_per_min'], + note: note, ); } diff --git a/lib/src/onehz/motion/orientation.dart b/lib/src/onehz/motion/orientation.dart index ea6f9ec..75109a7 100644 --- a/lib/src/onehz/motion/orientation.dart +++ b/lib/src/onehz/motion/orientation.dart @@ -9,8 +9,22 @@ // Wrist tilt correlates with, but is not identical to, torso posture. // * Static-tilt only. Dynamic/quaternion orientation (Madgwick/Mahony) needs // the ~100 Hz foreground stream and is DEFERRED to a foreground module. -// * Computed ONLY on low-motion epochs (|‖a‖−g|, sample jitter small); high- -// motion windows return absent rather than a fabricated posture. +// * Computed ONLY on epochs that are still in BOTH senses — low rotation and +// low |Δ‖a‖|. A wrist that is turning has no one posture, so those windows +// return absent rather than a fabricated one. +// +// The rotation half is what this substrate needs, and it is not a +// refinement — it is the only half that works here. The gate used to be +// ONLY mean +// |Δ‖a‖|, jitter of the vector MAGNITUDE, which is blind to rotation BY +// CONSTRUCTION: rotating a fixed-length vector does not change its length. +// And the 1 Hz record ships a firmware-fused ~1 g vector (enmo.dart: p50 +// 1.027 g, 1.033 ± 0.006 g during the most vigorous minute of a day), so +// that gate could not fire on real data at all — measured, a 180° rotation +// scored jitter 4.6e-17 g, stillness 1.000, confidence 0.90, and ten +// minutes of continuous rotation published 10 lateral_right + 10 +// lateral_left postures. We now measure the ANGLE between successive +// gravity vectors. // // Pitch/roll convention (device frame, x=lateral, y=longitudinal, z=normal): // pitch = atan2(-x, sqrt(y²+z²)) (forward/back tilt) @@ -57,13 +71,26 @@ String classifyPosition(double pitchDeg, double rollDeg) { return rollDeg > 0 ? 'lateral_right' : 'lateral_left'; } +/// Allowed mean sample-to-sample ROTATION of the gravity vector (degrees) for +/// an epoch to count as static. +/// +/// 3° is the angular equivalent of the 0.05 g magnitude gate this replaced (a +/// 0.05 g chord on a 1 g vector subtends ~2.87°), so the intended strictness is +/// unchanged — it is only now measuring the quantity it always claimed to. +const double defaultMaxRotationDeg = 3.0; + /// Estimate the static tilt + body-position proxy over an accel epoch. /// -/// Returns absent when the epoch is too short OR too dynamic (motion swamps -/// gravity, so direction is meaningless). [maxJitterG] is the allowed mean -/// sample-to-sample magnitude change for the epoch to count as "static". +/// Returns absent when the epoch is too short, the wrist is TURNING (a rotating +/// wrist has no single posture), or ‖a‖ is swinging (linear acceleration along +/// the gravity axis, which rotation cannot see). [maxRotationDeg] is the allowed +/// mean sample-to-sample angle between consecutive accel vectors; +/// [maxJitterG] the allowed mean |Δ‖a‖|. BOTH gates are needed: rotation is +/// blind to an axial shake, and magnitude is blind to rotation — and on this +/// substrate the magnitude one alone is blind to everything. Metric staticTilt( List epoch, { + double maxRotationDeg = defaultMaxRotationDeg, double maxJitterG = 0.05, int minSamples = 3, }) { @@ -76,24 +103,59 @@ Metric staticTilt( note: 'epoch too short for a static-tilt estimate', ); } - // mean gravity vector over the epoch final mags = [ for (final s in valid) math.sqrt(s.x * s.x + s.y * s.y + s.z * s.z) ]; - // jitter = mean |Δ‖a‖| between consecutive samples + // Rotation = mean angle between consecutive accel vectors. Samples with a + // degenerate (zero) magnitude carry no direction and are skipped. + var rotSum = 0.0; + var rotN = 0; + for (var i = 1; i < valid.length; i++) { + final a = valid[i - 1], b = valid[i]; + final ma = mags[i - 1], mb = mags[i]; + if (ma <= 0 || mb <= 0) continue; + final cos = clamp((a.x * b.x + a.y * b.y + a.z * b.z) / (ma * mb), -1.0, 1.0); + rotSum += math.acos(cos) * _rad2deg; + rotN++; + } + if (rotN == 0) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'accel vectors carry no direction (degenerate magnitude)', + ); + } + final rotDeg = rotSum / rotN; + // Magnitude jitter — the old gate, kept as the SECOND condition. It is the + // only one that sees a linear shake along the gravity axis (direction + // constant, ‖a‖ swinging). It cannot see rotation, which is why it is no + // longer the only one. var jitter = 0.0; for (var i = 1; i < mags.length; i++) { jitter += (mags[i] - mags[i - 1]).abs(); } jitter = mags.length > 1 ? jitter / (mags.length - 1) : 0.0; - final stillness = clamp(1.0 - jitter / maxJitterG, 0.0, 1.0); + // Stillness is the WORSE of the two — a posture is only as trustworthy as + // the loosest thing that could have disturbed it. + final stillness = clamp( + math.min(1.0 - rotDeg / maxRotationDeg, 1.0 - jitter / maxJitterG), + 0.0, + 1.0, + ); + if (rotDeg > maxRotationDeg) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'wrist rotating ${rotDeg.toStringAsFixed(2)}°/sample > ' + '${maxRotationDeg}°; no single posture during motion', + ); + } if (jitter > maxJitterG) { return Metric.absent( tier: Tier.high, inputs_used: inputs, - note: - 'epoch too dynamic (jitter ${jitter.toStringAsFixed(3)}g > ${maxJitterG}g); ' - 'no static posture during motion', + note: 'epoch too dynamic (jitter ${jitter.toStringAsFixed(3)}g > ' + '${maxJitterG}g); no static posture during motion', ); } final mx = mean([for (final s in valid) s.x])!; @@ -117,10 +179,12 @@ Metric staticTilt( /// /// Splits [samples] into fixed [epochSec] windows (bucketed by wall-clock /// `tsMs`), runs [staticTilt] on each, and returns the present postures. -/// Epochs that are too dynamic/short are skipped (no fabricated posture). +/// Epochs the wrist spent turning, or that are too short, are skipped (no +/// fabricated posture). List positionSeries( List samples, { int epochSec = 30, + double maxRotationDeg = defaultMaxRotationDeg, double maxJitterG = 0.05, }) { if (samples.isEmpty) return const []; @@ -134,7 +198,8 @@ List positionSeries( final out = []; final keys = buckets.keys.toList()..sort(); for (final k in keys) { - final m = staticTilt(buckets[k]!, maxJitterG: maxJitterG); + final m = staticTilt(buckets[k]!, + maxRotationDeg: maxRotationDeg, maxJitterG: maxJitterG); if (m.present) out.add(m.value!); } return out; diff --git a/lib/src/onehz/motion/steps.dart b/lib/src/onehz/motion/steps.dart index 275c652..dab8c28 100644 --- a/lib/src/onehz/motion/steps.dart +++ b/lib/src/onehz/motion/steps.dart @@ -12,11 +12,13 @@ // So we split the problem the only honest way: // // TIER A — [pedometer] / [livePedometer]: a real step counter on the 100 Hz -// foreground stream. The locked Analog Devices AN-2554 "full step detection" -// algorithm, ported VERBATIM from the OpenStrap backend -// (openstrap-analytics/src/steps.ts) where it was calibrated on a 100-step -// ground-truth walk (raw ×1.11 gain). Its CONFIRM=8 regularity gate rejects -// waving/typing/handling and reads 0 at rest. Directly testable: walk N +// foreground stream, ported from the OpenStrap backend +// (openstrap-analytics/src/steps.ts). See [StepParams] for what it actually +// is and is not — it is NOT AN-2554 verbatim, and the ×1.11 "calibration" +// it used to carry was measured to be a fabricated +10.6%. Its CONFIRM=8 +// regularity gate rejects IRREGULAR handling and reads 0 at rest; it does +// NOT reject rhythmic arm work (measured: a steady 1.8 Hz stir over 300 s +// of no locomotion counts 538 steps at 108 spm). Directly testable: walk N // steps with the app open and compare. // // TIER B — [dailyActiveMinutes]: a 24/7 MOVEMENT-VOLUME index from the 1 Hz @@ -30,13 +32,15 @@ // // Two things make the minute detector stable, and both matter: // 1. The feature is [MotionMinute.dynAmp] — per-axis high-passed dynamic -// amplitude — NOT ENMO. On this substrate ENMO is worse than unstable, -// it is EMPTY: the band ships a fused gravity vector whose magnitude -// sits at ~1.03 g even during the most vigorous minute of a day, so -// ENMO reduces to ~(1.03 − gRef), a pure calibration artifact. dynAmp -// removes gravity as a VECTOR and measures how fast the wrist -// re-orients. (Vähä-Ypyä 2015 argues for calibration-robust amplitude -// measures over ENMO for exactly this reason.) +// amplitude — NOT ENMO. ENMO is not empty on this substrate (audit +// MOT-07 re-measured it: Spearman(ENMO, HR) = 0.244 vs +// Spearman(dynAmp, HR) = 0.689 over 11,205 covered gen4 minutes), it +// is CALIBRATION-FRAGILE: it is `‖a‖ − gRef` and the per-family gRef +// (1.0277 gen4 / 1.0014 MG / 0.9977 W5) is the same order as the +// movement it is supposed to measure. dynAmp removes gravity as a +// VECTOR and measures how fast the wrist re-orients, so no reference +// has to be right. (Vähä-Ypyä 2015 argues for calibration-robust +// amplitude measures over ENMO for exactly this reason.) // 2. The cut-point is a MULTI-DAY PERSONAL REFERENCE // ([personalDynFloorFromDailySummaries]) that is FROZEN after an // enrollment window — neither an absolute g constant @@ -98,32 +102,95 @@ class PedometerResult { }; } -/// Locked AN-2554 parameters — ported VERBATIM from the OpenStrap backend -/// pedometer (`openstrap-analytics/src/steps.ts`), which was calibrated against -/// a 100-step ground-truth walk on our ~100 Hz wrist IMU. Do not retune without -/// a fresh ground-truth calibration. +/// WHAT THIS ALGORITHM ACTUALLY IS — the citation, corrected. +/// +/// It is NOT AN-2554, despite what this file said for a long time. It is: +/// +/// * AN-2554's *structure* — centred-window max/min peak search judged +/// against a dynamic threshold that is refreshed from recent (max+min)/2 — +/// but on Euclidean magnitude √(x²+y²+z²), where AN-2554 uses the sum of +/// absolutes, and retuned for 100 Hz where AN-2554 is specified at 50 Hz; +/// * plus Zhao's ADXL345 Analog Dialogue pedometer's confirm/"regulation" +/// mechanic ([confirm] consecutive valid pairs before anything is credited, +/// then free-running until the run breaks) — also a 50 Hz design; +/// * plus [sens] and [confirm], which appear in NEITHER source. They are +/// ours. Do not defend them by citation. +/// +/// What IS taken straight from AN-2554 is the pair of physiological step- +/// interval bounds ([minStepIntervalS] / [maxStepIntervalS]) — max ~5 steps/s +/// running, min ~0.5 steps/s very slow walking. Those were missing from this +/// port until they were added back. +/// +/// Neither ancestor is validated evidence: AN-2554's "~97% on a wrist" is a +/// vendor figure on an unpublished protocol, and Zhao's note has no validation +/// at all. Every independent free-living wrist evaluation puts peak detection +/// between 60% and 230% error. Treat this as a counter that is accurate on the +/// walking it supervises, not as an all-day step source. class StepParams { - static const int fs = 100; // assembled IMU sample rate (Hz) + /// Default assembled IMU sample rate (Hz). No longer decorative: it is the + /// default for [pedometer]'s `sampleRateHz`, which the interval bounds below + /// are converted through. Every other constant here is in SAMPLES. + static const double fs = 100; static const int filter = 8; // low-pass moving-average taps static const int window = 33; // centered peak window (~0.33 s @100 Hz) static const double sens = 0.10; // g — dead-zone around the dynamic threshold static const int thrOrder = 4; // dynamic-threshold smoothing buffer static const int confirm = 8; // consecutive possible steps before counting static const int maxMinTimeout = 120; // samples to find a min after a max - static const double gain = 1.11; // calibration: raw 90 → ~100 ground truth + + /// AN-2554's step-interval window: a human runs at most ~5 steps/s and walks + /// at slowest ~0.5 steps/s, so a peak-to-peak interval outside 0.2–2.0 s is + /// not gait and breaks the run. Distinct from [maxMinTimeout], which bounds + /// the max→min HALF cycle, not the interval between steps. + static const double minStepIntervalS = 0.2; + static const double maxStepIntervalS = 2.0; + + /// Per-user calibration trim. **Default 1.00, and it must stay 1.00 as a + /// default** — a population fudge factor is not a calibration. + /// + /// This was 1.11 and it was shipping a measured +10.6% over-count. The + /// docstring claimed "raw 90 → ~100 ground truth"; on clean synthetic gait + /// the raw count is exact — raw/truth 1.00 at 60, 80, 100, 120, 140 and + /// 180 spm. The 90→100 figure is only reproducible at ~0.050 g peak + /// amplitude, on the [sens] dead-zone cliff where a 100-step walk counts + /// 0 → 56 → 98 → 100 across 0.048–0.055 g: n=1, at the least reproducible + /// operating point in the parameter space. + /// + /// The real systematic deficit on a still→walk→still bout is a FLAT −0.9%, + /// identical from a 20-step bout to a 10,000-step one. Flat means it is + /// neither a gain error (which would scale) nor an offset (which would + /// shrink): it is per-minute CHUNK-BOUNDARY loss inside [calcSteps], which + /// restarts the state machine every minute and pays [confirm] again. Fixing + /// that means changing how [calcSteps] segments minutes; it is 0.9% and it is + /// deliberately not chased. Do not re-derive a gain from it. + /// + /// A real strap on a real wrist may still need trimming — that is what this + /// knob is for. Measure it against ground truth, per user, and store it with + /// the user; never bake a population constant back in here. + static const double gain = 1.00; } -/// AN-2554 time-domain step count over ONE contiguous accelerometer-MAGNITUDE -/// signal (g, ~100 Hz, gravity INCLUDED). Raw count — no calibration gain. +/// Time-domain step count over ONE contiguous accelerometer-MAGNITUDE signal +/// (g, gravity INCLUDED). Raw count — no calibration gain. See [StepParams] for +/// what this algorithm is (it is not AN-2554 verbatim). /// -/// Faithful port of `pedometer()` from the backend: /// low-pass (trailing MA) → centered-window max/min extrema → dynamic -/// threshold ± [StepParams.sens]/2 dead-zone → [StepParams.confirm] -/// consecutive "possible steps" before counting (the regularity gate that -/// rejects waving/typing/handling — validated to read 0 at rest). -int pedometer(List sig) { +/// threshold ± [StepParams.sens]/2 dead-zone → step-interval bounds +/// ([StepParams.minStepIntervalS]…[StepParams.maxStepIntervalS]) → +/// [StepParams.confirm] consecutive "possible steps" before counting (the +/// regularity gate — it rejects IRREGULAR handling and reads 0 at rest, but +/// it does NOT reject rhythmic arm work). +/// +/// [sampleRateHz] is the rate of THIS buffer. It is used only to convert the +/// step-interval bounds into samples — every other constant is in samples and +/// the peak window is wide enough that 50–200 Hz all count correctly (measured; +/// 25 Hz collapses to 0.29 of truth at 110 spm and no bound saves that). +int pedometer(List sig, {double sampleRateHz = StepParams.fs}) { final n = sig.length; - if (n < StepParams.window) return 0; + if (n < StepParams.window || sampleRateHz <= 0) return 0; + // Step-interval bounds in SAMPLES, from the buffer's real rate. + final minGap = StepParams.minStepIntervalS * sampleRateHz; + final maxGap = StepParams.maxStepIntervalS * sampleRateHz; const filter = StepParams.filter; // low-pass: trailing moving average final lp = List.filled(n, 0); @@ -168,6 +235,7 @@ int pedometer(List sig) { var stateMax = true; // 'max' → looking for a max; else looking for a min var curMax = 0.0; var curMaxIdx = -1; + var lastStepIdx = -1; // peak index of the last credited pair (interval anchor) for (var k = 0; k < candI.length; k++) { final ci = candI[k], cMax = candMax[k], cv = candV[k]; if (stateMax) { @@ -188,10 +256,26 @@ int pedometer(List sig) { stateMax = true; poss = 0; regulation = false; + lastStepIdx = -1; continue; } final mx = curMax, mn = cv; if (mx > dynVal + StepParams.sens / 2 && mn < dynVal - StepParams.sens / 2) { + // Step-interval bound (AN-2554). The interval between two steps is the + // gap between their peaks; outside 0.2–2.0 s no human produced both, so + // this pair cannot CONTINUE the run — it can only start a new one. + // Without it the counter tracks 250, 300 and 320 spm at raw/truth 1.00 + // and only loses lock near 350 spm, by accident of the peak window's + // width. Checked here rather than above the amplitude test so a noise + // pair can never move the anchor. + final gap = lastStepIdx < 0 ? null : curMaxIdx - lastStepIdx; + lastStepIdx = curMaxIdx; + if (gap != null && (gap < minGap || gap > maxGap)) { + poss = 0; + regulation = false; + stateMax = true; + continue; + } if (mx - mn > StepParams.sens) { dyn.add((mx + mn) / 2); if (dyn.length > StepParams.thrOrder) dyn.removeAt(0); @@ -218,27 +302,37 @@ int pedometer(List sig) { return steps; } -/// Daily total: AN-2554 over each per-minute contiguous magnitude signal, -/// summed and scaled by the calibration [StepParams.gain]. Faithful port of -/// `calcSteps()`. Per-minute chunking is the configuration the gain was -/// calibrated under — keep it. -int calcSteps(List> minuteSignals) { +/// Daily total: [pedometer] over each per-minute contiguous magnitude signal, +/// summed and scaled by [StepParams.gain] (1.00 by default — see there). +/// +/// KNOWN, DELIBERATELY UNFIXED: the per-minute chunking costs a flat −0.9%. +/// The state machine restarts at every boundary and has to re-earn +/// [StepParams.confirm], losing ~2 steps per minute of continuous walking. +/// Measured on still→walk→still bouts of 20 / 100 / 1000 / 10000 steps: −0.9% +/// at every size. It is not a gain error and must not be "corrected" with one. +/// Fixing it properly means carrying state across chunk boundaries, i.e. +/// changing how this function segments minutes. +int calcSteps( + List> minuteSignals, { + double sampleRateHz = StepParams.fs, +}) { var total = 0; for (final sig in minuteSignals) { - total += pedometer(sig); + total += pedometer(sig, sampleRateHz: sampleRateHz); } return (total * StepParams.gain).round(); } -/// Convenience wrapper for the live foreground stream: AN-2554 over a tri-axial -/// buffer (g, gravity included). Returns the count + an estimated cadence over -/// the buffer span. The RAW (pre-gain) count is in [PedometerResult.steps]; -/// apply [StepParams.gain] at the display/daily-sum layer (as [calcSteps] does). +/// Convenience wrapper for the live foreground stream: [pedometer] over a +/// tri-axial buffer (g, gravity included). Returns the count + an estimated +/// cadence over the buffer span. The RAW (pre-gain) count is in +/// [PedometerResult.steps]; apply [StepParams.gain] at the display/daily-sum +/// layer (as [calcSteps] does). PedometerResult livePedometer( List x, List y, List z, { - double sampleRateHz = 100.0, + double sampleRateHz = StepParams.fs, }) { final n = math.min(x.length, math.min(y.length, z.length)); if (n < StepParams.window || sampleRateHz <= 0) return PedometerResult.none; @@ -246,7 +340,7 @@ PedometerResult livePedometer( for (var i = 0; i < n; i++) math.sqrt(x[i] * x[i] + y[i] * y[i] + z[i] * z[i]) ]; - final steps = pedometer(mag); + final steps = pedometer(mag, sampleRateHz: sampleRateHz); final durationS = n / sampleRateHz; if (steps <= 0) return PedometerResult(0, durationS, 0, 0, 0); // Cadence over the buffer span. For a dedicated walk this is the walking @@ -379,6 +473,12 @@ const double personalDynFloorQuantile = 0.90; /// Minimum pooled trailing minutes before a personal floor is trustworthy. /// 2000 minutes ≈ 1.5 days of continuous wear, in practice several partial /// days — enough that the quantile is not dominated by one posture or one day. +/// +/// This gates [personalDynFloor], which needs the RAW minute pool. A caller +/// that prunes its substrate cannot supply that and uses +/// [personalDynFloorFromDailySummaries] instead, which is gated on DAYS — +/// so do NOT report cold-start progress against this constant unless the +/// caller genuinely counted minutes. See [dailyActiveMinutes]. const int personalDynFloorMinMinutes = 2000; /// Minimum trailing DAYS for [personalDynFloorFromDailySummaries]. @@ -542,7 +642,12 @@ double? dailyDynSummary( class DailyMovementEstimate { final int activeMinutes; // minutes with sustained wrist movement final double dynFloorG; // personal floor actually applied (g) - final double coverage; // fraction of the day with valid motion data + + /// Covered minutes / EXPECTED minutes when the caller supplies + /// `expectedMinutes` (1440 for a calendar day); otherwise covered minutes / + /// the worn SPAN, which counts interior holes but not the unworn ends. + /// Read the two cases differently — only the first is "fraction of the day". + final double coverage; final int boutCount; // number of qualifying bouts const DailyMovementEstimate({ @@ -590,11 +695,17 @@ class DailyMovementEstimate { /// which is what this returns. A covered minute counts when BOTH of these hold: /// • its [MotionMinute.dynAmp] is above [personalDynFloorG]; /// • it belongs to a run of at least [minBoutMin] CONSECUTIVE qualifying -/// minutes, where consecutive means adjacent in ORIGINAL minute index — a -/// coverage gap breaks the run and cannot stitch two short stretches into -/// one qualifying bout. This is a DENOISER, not a health threshold: the -/// 2018 US Physical Activity Guidelines removed the 10-minute minimum-bout -/// rule, so do not defend this number physiologically. +/// minutes, where consecutive means ADJACENT ON THE CLOCK — the next +/// minute's `tsMinStartMs` is exactly 60 s later. A coverage gap breaks the +/// run and cannot stitch two short stretches into one qualifying bout. This +/// is a DENOISER, not a health threshold: the 2018 US Physical Activity +/// Guidelines removed the 10-minute minimum-bout rule, so do not defend +/// this number physiologically. +/// +/// [expectedMinutes] mirrors `enmoSeries`: pass 1440 for a calendar day so +/// [DailyMovementEstimate.coverage] is measured against the day rather than +/// against the worn span. Without it, a day worn 4 h out of 24 reports 1.0 +/// while `enmoSeries` reports 0.167 for the identical substrate. /// /// There is deliberately no upper ceiling and no HR gate. Both were measured /// against real substrate and found to be dead or harmful — see the comments @@ -603,9 +714,15 @@ class DailyMovementEstimate { /// [personalDynFloorG] is REQUIRED and MAY BE NULL. Null means "not enough /// history to know this user's movement scale", and the honest answer to that /// is an ABSENT metric carrying a `need_baseline:have=…,need=…` note — build -/// the floor with [personalDynFloor] and pass [pooledMinutesAvailable] so the -/// note can report progress. There is deliberately NO constant fallback: a -/// constant absolute floor is precisely the failure this design removes. +/// the floor with [personalDynFloorFromDailySummaries] and pass +/// [historyDaysAvailable] so the note can report progress. There is +/// deliberately NO constant fallback: a constant absolute floor is precisely +/// the failure this design removes. +/// +/// The note counts DAYS, against [personalDynFloorMinDays]. It used to count +/// them against [personalDynFloorMinMinutes] (2000) while every caller passed a +/// day count, so a user three days in was told `have=3,need=2000` — ~1997 more +/// of a unit nobody was measuring, for a metric that unlocks in two. /// /// Tier is always ESTIMATE. /// @@ -618,7 +735,8 @@ Metric dailyActiveMinutes( required double? personalDynFloorG, double minSamplesPerMinute = 30, int minBoutMin = 3, - int pooledMinutesAvailable = 0, + int historyDaysAvailable = 0, + int? expectedMinutes, }) { const inputs = ['dyn_amp_per_min', 'personal_dyn_floor']; if (motion.isEmpty) { @@ -637,8 +755,8 @@ Metric dailyActiveMinutes( tier: Tier.estimate, inputs_used: inputs, note: needBaselineNote( - have: pooledMinutesAvailable, - need: personalDynFloorMinMinutes, + have: historyDaysAvailable, + need: personalDynFloorMinDays, ), ); } @@ -661,7 +779,15 @@ Metric dailyActiveMinutes( } } final covered = idx.length; - final coverage = covered / motion.length; + // COVERAGE DENOMINATOR — same rule as enmo.dart, and it must be the same + // rule, because both are published for the same day. The worn SPAN counts + // interior holes but NOT the unworn ends, so a day worn 4 h out of 24 read + // coverage 1.0 here while enmoSeries(expectedMinutes: 1440) read 0.167 on the + // identical substrate. Pass [expectedMinutes] to count the ends too. + final spanMinutes = + ((motion.last.tsMinStartMs - motion.first.tsMinStartMs) / 60000).round() + 1; + final denom = expectedMinutes ?? spanMinutes; + final coverage = denom <= 0 ? 0.0 : clamp(covered / denom, 0.0, 1.0); if (covered < dailyStepMinCoveredMinutes) { return Metric.absent( tier: Tier.estimate, @@ -701,9 +827,14 @@ Metric dailyActiveMinutes( // pass 2 — bout gate: only credit minutes inside a run of >= minBoutMin // CONSECUTIVE gate-passing minutes, so a scattered single minute (a brief // movement/HR blip mid-turnover in bed, say) never becomes phantom activity. - // Runs are broken by ORIGINAL minute index (idx[k]), not position in the - // covered-minutes array, so a coverage gap can't stitch two separate - // stretches into one fake long bout. + // + // Adjacency is tested ON THE CLOCK. It used to be `idx[end+1] == idx[end]+1`, + // which is adjacency in the gap-COMPACTED list: enmoSeries emits no + // MotionMinute at all for a fully-absent minute, so an off-wrist minute never + // occupies a slot and could not break a run. Measured: four isolated moving + // minutes an HOUR apart were credited as one 4-minute bout. Partially-covered + // minutes were already handled — they stay in `motion` and are excluded from + // `idx` — so only the fully-absent (off-wrist) case slipped through. var activeMin = 0; var boutCount = 0; var k = 0; @@ -715,7 +846,10 @@ Metric dailyActiveMinutes( var end = k; while (end + 1 < idx.length && gateOk[end + 1] && - idx[end + 1] == idx[end] + 1) { + ((motion[idx[end + 1]].tsMinStartMs - motion[idx[end]].tsMinStartMs) / + 60000) + .round() == + 1) { end++; } if (end - k + 1 >= minBoutMin) { diff --git a/lib/src/onehz/respiration/cvhr_apnea.dart b/lib/src/onehz/respiration/cvhr_apnea.dart index 633239a..c50c645 100644 --- a/lib/src/onehz/respiration/cvhr_apnea.dart +++ b/lib/src/onehz/respiration/cvhr_apnea.dart @@ -12,16 +12,25 @@ // 1. Build a 1 Hz HR-equivalent envelope from cleaned NN (instantaneous HR // = 60000/NN), smoothed with a 2nd-order (quadratic) Savitzky-Golay-like // local polynomial to suppress beat jitter while preserving cycle shape. -// 2. Adaptive 5th / 95th percentile envelopes over a sliding ~130 s window -// define the "normal band"; a dip = an excursion below the 5th-pct -// envelope of width 10–120 s. -// 3. Keep a dip as a CVHR cycle only if depth-to-width ratio > 0.7 ms/s -// (the steep bradycardia-then-tachycardia signature, not slow drift). +// 2. An adaptive baseline — the sliding MEDIAN over a ~130 s window, which +// spans ≥2 apnea cycles so it tracks the eupneic level without being +// pulled into any single event. The bradycardia is an NN MAXIMUM (NN ↑ ⇔ +// HR ↓), so the scored quantity is the POSITIVE excursion above that +// baseline; negative excursions (the recovery tachycardia) are clipped. +// A run clears the baseline when it exceeds max(0.5 × the 75th percentile +// of the positive excursions, 5 ms) — the only percentile in this file. +// 3. Keep a run as a CVHR cycle only if its width is 10–120 s AND its +// depth-to-width ratio > 0.7 ms/s (the steep bradycardia-then-tachycardia +// signature, not slow drift). // 4. CVHR cycles per hour => an apnea SCREEN index (NOT an AHI, NOT a // diagnosis). Report night-to-night variability caveat. // // HONESTY: this is a SCREEN. We never output an AHI or a clinical category, and // we flag that single-night CVHR has substantial night-to-night variability. +// The denominator is OBSERVED time: the night is split at recording gaps, each +// contiguous stretch is analysed on its own, and the hole contributes neither +// interpolated beats nor hours. Dividing by the first-to-last span instead let +// a 2 h charging break cut the index from 80.0/h to 53.3/h on identical beats. import 'dart:math' as math; import '../types.dart'; @@ -30,30 +39,65 @@ import '../util.dart'; class CvhrResult { final int cycleCount; // detected CVHR cycles final double cvhrPerHour; // cycles / hour (the screen index) - final double analyzedHours; // night length analyzed - final double meanDepthMs; // mean dip depth (ms of NN excursion) - final double meanWidthSec; // mean dip width (s) + final double analyzedHours; // OBSERVED hours analysed (gaps excluded) + + /// Mean dip depth (ms of NN excursion) — NULL when no cycle was detected. + /// It used to be 0 in that case, which serialised an absence as a + /// measurement. `cycleCount == 0` and these being null are the same fact. + final double? meanDepthMs; + + /// Mean dip width (s) — NULL when no cycle was detected. See [meanDepthMs]. + final double? meanWidthSec; + + /// [p25, p50, p75] of the per-cycle dip depths (ms). NULL when no cycle was + /// detected. The mean alone hides the shape: a night of a few deep dips and a + /// night of many shallow ones can share it. These are the raw quartiles of + /// the same per-cycle list the means come from — no adjective attached, no + /// threshold, and specifically no category. + final List? depthQuartilesMs; + + /// [p25, p50, p75] of the per-cycle widths (s) — the cycle LENGTH spread. + /// See [depthQuartilesMs]. + final List? widthQuartilesSec; const CvhrResult({ required this.cycleCount, required this.cvhrPerHour, required this.analyzedHours, required this.meanDepthMs, required this.meanWidthSec, + this.depthQuartilesMs, + this.widthQuartilesSec, }); Map toJson() => { 'cycle_count': cycleCount, 'cvhr_per_hour': round6(cvhrPerHour), 'analyzed_hours': round6(analyzedHours), - 'mean_depth_ms': round6(meanDepthMs), - 'mean_width_sec': round6(meanWidthSec), + 'mean_depth_ms': meanDepthMs == null ? null : round6(meanDepthMs!), + 'mean_width_sec': meanWidthSec == null ? null : round6(meanWidthSec!), + 'depth_quartiles_ms': + depthQuartilesMs?.map(round6).toList(growable: false), + 'width_quartiles_sec': + widthQuartilesSec?.map(round6).toList(growable: false), }; } +/// [p25, p50, p75] of [xs], or null when empty. Same linear-interpolated +/// percentile the rest of the package uses. +List? _quartiles(List xs) => xs.isEmpty + ? null + : [percentile(xs, 25)!, percentile(xs, 50)!, percentile(xs, 75)!]; + /// CVHR / ACAT apnea screen on a cleaned NN series. /// /// [nnMs] cleaned NN intervals (ms), [nnTimesMs] their cumulative beat times /// (ms). [artifactFraction] from RR-correction. Tunables follow Hayano: -/// dip width 10–120 s, depth/width ratio > 0.7 ms/s, ~130 s percentile window. +/// dip width 10–120 s, depth/width ratio > 0.7 ms/s, ~130 s baseline window. +/// +/// Callers pass one unsegmented block (the whole sleep window), so a charging +/// or off-wrist stretch arrives here inline. Beats more than [maxGapSec] apart +/// start a NEW segment: each is resampled and scored on its own, and only the +/// segments' own spans enter [CvhrResult.analyzedHours]. The gap is not +/// interpolated across and is not charged to the denominator. Metric cvhrApneaScreen( List nnMs, List nnTimesMs, { @@ -63,6 +107,7 @@ Metric cvhrApneaScreen( double depthWidthRatioMin = 0.7, // ms per second double envWindowSec = 130, double maxArtifact = 0.30, + double maxGapSec = 30, }) { const inputs = ['rr_cleaned', 'beat_times']; if (nnMs.length < 60 || nnTimesMs.length != nnMs.length) { @@ -81,31 +126,303 @@ Metric cvhrApneaScreen( ); } - // Resample NN onto a uniform 1 Hz grid by piecewise-linear interpolation of - // the tachogram (NN vs beat time). This gives evenly-spaced points for the - // sliding-window percentile envelopes and the polynomial smoother. (Times in - // seconds.) final tSec = [for (final t in nnTimesMs) t / 1000.0]; - final t0 = tSec.first; - final t1 = tSec.last; - final analyzedHours = (t1 - t0) / 3600.0; - if (analyzedHours <= 0) { + if (tSec.last - tSec.first <= 0) { return const Metric.absent( tier: Tier.high, inputs_used: inputs, note: 'degenerate beat times', ); } - final nGrid = (t1 - t0).floor() + 1; - if (nGrid < 60) { + + // SEGMENT AT GAPS. Beats more than [maxGapSec] apart are two recordings, not + // one: interpolating a straight line across the hole invents a flat stretch + // with no cycles in it, and charging the hole to `analyzedHours` divides a + // real cycle count by time we never observed. Both push the index DOWN, so a + // charging break used to read as an improvement. + final segStart = [0]; + for (var i = 1; i < tSec.length; i++) { + if (tSec[i] - tSec[i - 1] > maxGapSec) segStart.add(i); + } + + var cycleCount = 0; + var analyzedHours = 0.0; + final depths = []; // ms + final widths = []; // s + for (var s = 0; s < segStart.length; s++) { + final lo = segStart[s]; + final hi = (s + 1 < segStart.length ? segStart[s + 1] : tSec.length) - 1; + final segT = tSec.sublist(lo, hi + 1); + final segNn = nnMs.sublist(lo, hi + 1); + final spanSec = segT.last - segT.first; + // Too short to hold even one cycle plus its baseline window — it also can't + // be scored, so it contributes neither cycles nor hours. + if (segT.length < 60 || spanSec < 60) continue; + analyzedHours += spanSec / 3600.0; + _scoreSegment( + segT, + segNn, + minWidthSec: minWidthSec, + maxWidthSec: maxWidthSec, + depthWidthRatioMin: depthWidthRatioMin, + envWindowSec: envWindowSec, + onCycle: (depth, width) { + cycleCount++; + depths.add(depth); + widths.add(width); + }, + ); + } + + if (analyzedHours <= 0) { return const Metric.absent( tier: Tier.high, inputs_used: inputs, - note: 'span too short for a CVHR screen (<60 s)', + note: 'no gap-free stretch long enough for a CVHR screen (<60 s)', + ); + } + + final perHour = cycleCount / analyzedHours; + final conf = clamp((1 - artifactFraction) * 0.85, 0.2, 0.85); + return Metric( + value: CvhrResult( + cycleCount: cycleCount, + cvhrPerHour: perHour, + analyzedHours: analyzedHours, + meanDepthMs: depths.isEmpty ? null : mean(depths), + meanWidthSec: widths.isEmpty ? null : mean(widths), + depthQuartilesMs: _quartiles(depths), + widthQuartilesSec: _quartiles(widths), + ), + confidence: conf, + tier: Tier.high, + inputs_used: inputs, + note: 'CVHR/ACAT (Hayano) apnea SCREEN — NOT a diagnosis, NOT an AHI; ' + 'single-night CVHR has substantial night-to-night variability, ' + 'interpret as a trend over multiple nights', + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// RESP-01 — the 30-night PERSONAL DISTRIBUTION. +// +// The per-night index above is not shippable on its own: single-night CVHR has +// substantial night-to-night variability, it fires on AF, on periodic breathing +// at altitude and on any arousal-rich fragmented night, and it is BLUNTED by +// beta-blockers, autonomic neuropathy and diabetes — so it misses real apnea +// too. Both directions are harm. What survives that is one statement about a +// user's OWN recent nights against their OWN earlier ones. +// +// WHAT THIS MAY NEVER BECOME: an AHI, a severity band, a per-night value on a +// screen, a finding about the user, or a notification. A quiet screen means +// NOTHING — that line is permanent copy, not a tooltip. +// +// RESP-02 (gate half) — nights where `irregular_rhythm_flag` fired are EXCLUDED +// here, not down-weighted. AF produces the same cyclic RR pattern this screen +// scores, so an AF night's index is not a weak apnea signal, it is a different +// signal wearing the same shape. The corroboration half of RESP-02 (10-second +// ENMO) is deliberately not built: it is new accel work, and its absence costs +// this aggregate nothing. + +/// One stored night, as the aggregate needs it. Straight off `day_result`. +class CvhrNight { + /// Local day label the night is filed under (used only for ordering/dedupe). + final String dayKey; + + /// `respiration.cvhr_apnea.cvhr_per_hour` for that night. + final double cvhrPerHour; + + /// `respiration.cvhr_apnea.analyzed_hours` — the OBSERVED, gap-excluded hours + /// behind [cvhrPerHour], and this night's weight. + final double analyzedHours; + + /// Whether `irregular_rhythm_flag` fired on that day. Null counts as NOT + /// fired: the flag predates nothing here, but a night we cannot ask about is + /// not a night we may silently drop from the denominator. + final bool irregularRhythm; + + const CvhrNight({ + required this.dayKey, + required this.cvhrPerHour, + required this.analyzedHours, + this.irregularRhythm = false, + }); +} + +/// The result of the 30-night screen. Deliberately holds NO per-night value. +class CvhrDistribution { + /// Nights that passed every gate and are behind [recentWeightedMean]. + final int nightsUsed; + + /// Nights dropped because `irregular_rhythm_flag` fired (RESP-02 cross-gate). + final int nightsExcludedIrregular; + + /// Nights dropped for having under [minAnalyzedHoursPerNight] observed hours. + final int nightsExcludedThin; + + /// Analyzed-hours-weighted mean cvhr/h over the whole retained window. This + /// is the user's OWN usual level — it is not a threshold and has no published + /// normal range. + final double weightedMean; + + /// The same weighted mean over the most recent third of the retained nights. + final double recentWeightedMean; + + /// [p25, p50, p75] of the retained nights' indices — the spread the recent + /// mean is being read against. + final List quartiles; + + /// True when the recent nights sit above the retained distribution's own p75. + /// This is the ONLY comparative statement the screen makes, and it is a + /// statement about this user's own history, not about apnea. + final bool aboveOwnUsual; + + const CvhrDistribution({ + required this.nightsUsed, + required this.nightsExcludedIrregular, + required this.nightsExcludedThin, + required this.weightedMean, + required this.recentWeightedMean, + required this.quartiles, + required this.aboveOwnUsual, + }); + + Map toJson() => { + 'nights_used': nightsUsed, + 'nights_excluded_irregular': nightsExcludedIrregular, + 'nights_excluded_thin': nightsExcludedThin, + 'weighted_mean': round6(weightedMean), + 'recent_weighted_mean': round6(recentWeightedMean), + 'quartiles': quartiles.map(round6).toList(growable: false), + 'above_own_usual': aboveOwnUsual, + }; +} + +/// A night needs this many OBSERVED hours before it may weigh on the screen. +const double minAnalyzedHoursPerNight = 4.0; + +/// And this many such nights must exist before the screen says anything. +const int minNightsForCvhrDistribution = 5; + +/// Nights considered, newest-first, once the gates have run. +const int cvhrDistributionWindowNights = 30; + +/// RESP-01 — the 30-night personal CVHR distribution. +/// +/// [nights] in any order; the newest [cvhrDistributionWindowNights] retained +/// nights are used. Returns ABSENT — never a partial number — until at least +/// [minNightsForCvhrDistribution] nights clear both gates. +Metric cvhrPersonalDistribution(List nights) { + const inputs = ['cvhr_apnea', 'irregular_rhythm_flag']; + // Newest first, one row per day (a re-derive can leave two). + final byDay = {}; + for (final n in nights) { + byDay[n.dayKey] = n; + } + final sorted = byDay.values.toList() + ..sort((a, b) => b.dayKey.compareTo(a.dayKey)); + + var excludedIrregular = 0; + var excludedThin = 0; + final kept = []; + for (final n in sorted) { + if (kept.length >= cvhrDistributionWindowNights) break; + if (!n.cvhrPerHour.isFinite || !n.analyzedHours.isFinite) continue; + // RESP-02 cross-gate FIRST: an AF night is not a thin night, and counting + // it as one would misreport why the screen is quiet. + if (n.irregularRhythm) { + excludedIrregular++; + continue; + } + if (n.analyzedHours < minAnalyzedHoursPerNight) { + excludedThin++; + continue; + } + kept.add(n); + } + + if (kept.length < minNightsForCvhrDistribution) { + return Metric.absent( + tier: Tier.relative, + inputs_used: inputs, + note: 'need_baseline:nights=${kept.length}/' + '$minNightsForCvhrDistribution with ≥' + '${minAnalyzedHoursPerNight.round()} analyzed hours ' + '(excluded ${excludedIrregular} for irregular rhythm, ' + '${excludedThin} too thinly observed)', ); } + + double weighted(List xs) { + var num = 0.0, den = 0.0; + for (final n in xs) { + num += n.cvhrPerHour * n.analyzedHours; + den += n.analyzedHours; + } + return den > 0 ? num / den : 0.0; + } + + final indices = [for (final n in kept) n.cvhrPerHour]; + // "Recent" is the newest third of the retained nights (≥2 by construction: + // 5 nights ⇒ 2). One night is not a comparison. + final recentCount = math.max(2, (kept.length / 3).round()); + final recent = kept.take(recentCount).toList(); + final all = weighted(kept); + final rec = weighted(recent); + final q = [ + percentile(indices, 25)!, + percentile(indices, 50)!, + percentile(indices, 75)!, + ]; + + return Metric( + value: CvhrDistribution( + nightsUsed: kept.length, + nightsExcludedIrregular: excludedIrregular, + nightsExcludedThin: excludedThin, + weightedMean: all, + recentWeightedMean: rec, + quartiles: q, + aboveOwnUsual: rec > q[2], + ), + // Relative tier on purpose: this compares the user only with themselves. + tier: Tier.relative, + // The floor is 5 nights, so a 5-night answer must not read like a 30-night + // one. Confidence rises with the nights actually behind it. + confidence: clamp( + 0.3 + 0.4 * (kept.length / cvhrDistributionWindowNights), 0.3, 0.7), + inputs_used: inputs, + note: 'CVHR is a CARDIAC SURROGATE, not a breathing measurement: this is ' + 'how often the pattern showed up across ${kept.length} of your own ' + 'nights, never an AHI, never a severity, never a per-night value. ' + 'It also fires on atrial fibrillation (those nights are excluded), on ' + 'periodic breathing at altitude and on any fragmented night, and it is ' + 'blunted by beta-blockers, autonomic neuropathy and diabetes — so a ' + 'quiet screen means nothing.', + ); +} + +/// Score ONE gap-free stretch: resample → smooth → sliding-median baseline → +/// positive-excursion runs → width + steepness gates. Calls [onCycle] with +/// (depthMs, widthSec) per surviving cycle. [t] in seconds, [nn] in ms. +void _scoreSegment( + List t, + List nn, { + required double minWidthSec, + required double maxWidthSec, + required double depthWidthRatioMin, + required double envWindowSec, + required void Function(double depthMs, double widthSec) onCycle, +}) { + // Resample NN onto a uniform 1 Hz grid by piecewise-linear interpolation of + // the tachogram (NN vs beat time). This gives evenly-spaced points for the + // sliding-window baseline and the polynomial smoother. Only ever called on a + // stretch with no gap larger than the caller's tolerance, so no interpolation + // here spans a hole. + final t0 = t.first; + final nGrid = (t.last - t0).floor() + 1; final grid = List.generate(nGrid, (i) => t0 + i); - final nnGrid = _resampleLinear(tSec, nnMs, grid); + final nnGrid = _resampleLinear(t, nn, grid); // 2nd-order local-polynomial smoothing (Savitzky-Golay quadratic, ~7 s half // window) to suppress beat jitter while keeping the cyclic shape. @@ -124,14 +441,19 @@ Metric cvhrApneaScreen( } // Baseline-subtracted bradycardic excursion (clip negatives — we only score // the NN-up bradycardia, not the tachycardia trough). - final exc = [for (var j = 0; j < nGrid; j++) math.max(0.0, smooth[j] - base[j])]; + final exc = [ + for (var j = 0; j < nGrid; j++) math.max(0.0, smooth[j] - base[j]) + ]; // Prominence threshold: a dip must clear a robust noise floor AND a fraction // of the typical excursion amplitude. We anchor it BELOW the median positive // excursion (Hayano's adaptive amplitude criterion is permissive enough to // capture the whole bradycardia run, not just its tip) — a threshold at the // peak would clip the run width to a few seconds and miss the cycle. - final posExc = [for (final e in exc) if (e > 0) e]; + final posExc = [ + for (final e in exc) + if (e > 0) e + ]; final excP75 = (posExc.isNotEmpty ? percentile(posExc, 75) : null) ?? 0; // half of the upper-quartile excursion: well inside each genuine dip's run. final prom = math.max(0.5 * excP75, 5.0); // ms @@ -139,9 +461,6 @@ Metric cvhrApneaScreen( // Find peaks of `exc`: local maxima exceeding `prom`, each isolated to one // contiguous super-threshold run (so one bradycardia = one cycle). Measure // the run width and the peak depth; apply the width + depth/width gates. - var cycleCount = 0; - final depths = []; // ms - final widths = []; // s var i = 0; while (i < nGrid) { if (exc[i] <= prom) { @@ -158,30 +477,9 @@ Metric cvhrApneaScreen( if (widthSec < minWidthSec || widthSec > maxWidthSec) continue; // Depth-to-width steepness gate (ms per second): the bradycardia-tachycardia // swing is steep; slow baseline drift is shallow per second. - final ratio = peakDepth / widthSec; - if (ratio < depthWidthRatioMin) continue; - cycleCount++; - depths.add(peakDepth); - widths.add(widthSec); + if (peakDepth / widthSec < depthWidthRatioMin) continue; + onCycle(peakDepth, widthSec); } - - final perHour = analyzedHours > 0 ? cycleCount / analyzedHours : 0.0; - final conf = clamp((1 - artifactFraction) * 0.85, 0.2, 0.85); - return Metric( - value: CvhrResult( - cycleCount: cycleCount, - cvhrPerHour: perHour, - analyzedHours: analyzedHours, - meanDepthMs: depths.isEmpty ? 0 : mean(depths)!, - meanWidthSec: widths.isEmpty ? 0 : mean(widths)!, - ), - confidence: conf, - tier: Tier.high, - inputs_used: inputs, - note: 'CVHR/ACAT (Hayano) apnea SCREEN — NOT a diagnosis, NOT an AHI; ' - 'single-night CVHR has substantial night-to-night variability, ' - 'interpret as a trend over multiple nights', - ); } /// Piecewise-linear resample of (t,y) onto a sorted [grid] of times (same unit). diff --git a/lib/src/onehz/respiration/resp_rate.dart b/lib/src/onehz/respiration/resp_rate.dart index e1f15cc..50a8a1d 100644 --- a/lib/src/onehz/respiration/resp_rate.dart +++ b/lib/src/onehz/respiration/resp_rate.dart @@ -4,9 +4,10 @@ // * RSA respiratory rate (PRIMARY) — Lomb-Scargle HF-peak on cleaned NN beat // times. Respiratory sinus arrhythmia modulates RR at the breathing // frequency; the HF (0.15–0.40 Hz) spectral peak => breaths/min. -// Pimentel 2017-style robustness: estimate over several Lomb-Scargle grid -// resolutions ("multiple AR-order surrogates") and keep the result only if -// they agree (low dispersion) — otherwise withhold. +// Welch 1967 segmentation: the peak is estimated on ~5-minute sub-windows +// and the night's rate is the MEDIAN across them; the robustness check is +// agreement of those sub-windows with each other. See [rsaRespRate] for why +// the whole-window periodogram it replaced could not work. // * RIIV respiratory rate — band-pass 0.1–0.5 Hz on the 1 Hz green PPG ADC // (respiratory-induced intensity variation), peak frequency => breaths/min. // * Karlen 2013 SD-gate fusion — discard a window when the two estimates @@ -14,8 +15,19 @@ // fuse them into a single honest rate. // // HONESTY CEILINGS: -// * 1 Hz Nyquist caps any rate at 0.5 Hz = 30 br/min. We refuse to report a -// peak at/above the ceiling (aliasing) and tag the limit in the note. +// * RIIV rides the 1 Hz green ADC, whose Nyquist caps any rate at 0.5 Hz = +// 30 br/min. We refuse to report a peak at/above that ceiling (aliasing). +// * RSA rides the TACHOGRAM, which is sampled once per BEAT, not once per +// second — so its ceiling is the beat-rate Nyquist (0.5/meanNN), ~37 br/min +// at HR 75 but only ~25 br/min at HR 50. [rsaRespRate] computes that +// ceiling per window, caps it at [respHiHz], and withholds the whole window +// when that ceiling falls below the classic HF band (the alias would be +// indistinguishable from a genuine slower rate). A rate the window cannot +// resolve is ABSENT, never a spurious in-band peak dressed up as a normal +// number. +// * Neither estimator can see a TRUE rate above [respHiHz] (30 br/min). Such +// a window yields a low peak near the ceiling, not an absence — sustained +// adult tachypnea is outside what this module claims to measure. // * RSA is HIGH tier (continuous 24/7, the structural edge); RIIV is MED // (1 Hz green is a coarse intensity proxy, not the 419 Hz waveform). // * Absent / insufficient input => null + confidence 0, never a guess. @@ -32,9 +44,29 @@ const double respHiHz = 0.5; /// RSA uses the classic HRV HF band (0.15–0.40 Hz = 9–24 br/min) where the /// respiratory peak lives in the RR spectrum. +/// +/// [rsaHiHz] is the top of the *classic HF band*, NOT the search ceiling. +/// Searching only to 0.40 Hz was a silent 24 br/min cap: a real 26–29 br/min +/// breather has no peak inside the band, so the search returned the largest +/// spurious in-band structure instead — measured 26.4 → 22.3 and 28.8 → 17.4, +/// both published at confidence 0.90. [rsaRespRate] therefore searches up to +/// the window's own resolvable ceiling (see [rsaCeilingHz]) and withholds a +/// peak that lands there. const double rsaLoHz = 0.15; const double rsaHiHz = 0.40; +/// The highest respiratory frequency an RSA window can actually resolve (Hz). +/// +/// The tachogram is sampled once per BEAT, so its Nyquist is `0.5 / meanNN`, +/// not 0.5 Hz. At HR 75 (NN 800 ms) that is 0.625 Hz; at HR 50 it is 0.417 Hz. +/// [respHiHz] caps it because nothing downstream of a 1 Hz record should claim +/// more than 30 br/min. +double rsaCeilingHz(double meanNnSec) { + if (!meanNnSec.isFinite || meanNnSec <= 0) return respHiHz; + final nyq = 0.5 / meanNnSec; + return nyq < respHiHz ? nyq : respHiHz; +} + /// One respiratory-rate estimate (breaths/min) plus its provenance. class RespEstimate { final double? brpm; // breaths per minute @@ -50,17 +82,63 @@ class RespEstimate { }; } +/// Longest span (seconds) a single RSA periodogram may cover. +/// +/// The Lomb-Scargle periodogram is an INCONSISTENT estimator: its variance does +/// not fall as the record lengthens — a longer record buys more independent +/// frequency bins (spacing 1/T), each still ~exponentially distributed around +/// the true PSD. Over an 8-hour night 1/T is 3.5e-5 Hz, so the 0.15–0.5 Hz band +/// holds ~10⁴ independent bins and (measured on 14 real nights) ~2600 local +/// maxima; its global maximum is a noise spike, not the respiratory rate. +/// Breathing is also non-stationary across a night — measured, this person's +/// rate falls ~19 → 16 br/min from the first quarter to the last — so one +/// spectrum over the whole night smears a drifting peak anyway. +/// +/// 300 s is the classic compromise: long enough that 1/T = 0.0033 Hz (0.2 +/// br/min) resolves the HF band, short enough that breathing is stationary +/// inside it. +const double rsaSegmentSec = 300; + /// RSA respiratory rate from cleaned NN beat times (PRIMARY 24/7 source). /// /// [nnMs] cleaned NN intervals (ms), [nnTimesMs] their cumulative beat times /// (ms). [artifactFraction] from RR-correction drives the confidence gate. -/// Pimentel-style robustness: re-estimate the HF peak across several grid -/// resolutions; accept only if they agree within [agreeBrpm] br/min. +/// +/// METHOD (Welch 1967 segmentation + a data-perturbation robustness check). +/// The input is cut into ~[rsaSegmentSec] sub-windows overlapping 50%, each gets +/// its own Lomb-Scargle HF peak, and the reported rate is the MEDIAN of those +/// peaks. The robustness test is whether the sub-windows AGREE: at least +/// [minConsensus] of them must land within [tolBrpm] br/min of that median, +/// otherwise the rate is withheld. That perturbs the DATA (different minutes of +/// the same night), which is the actual question — is there a stable +/// respiratory signal here? +/// +/// WHAT THIS REPLACED, AND WHY. The previous check took ONE periodogram over +/// the whole input and re-sampled it on 300/450/700-point grids, calling that +/// "a deterministic analogue" of Pimentel 2017's AR-model-order surrogate. It +/// is not one. Refining a grid re-reads the SAME spectrum, and over a night +/// that spectrum's bins are spaced ~30× finer than the grid step, so the three +/// grids were drawing three near-independent noise samples: measured on 14 real +/// nights the three peaks scattered by up to 8 br/min, the gate withheld 17 of +/// 30 nights, and on nights it did publish the number was often not even the +/// band's true maximum (one night published 10.66 br/min where the same night's +/// sub-windows agree on 16.96, and the fine-grid maximum was 18.55). Same night +/// replayed twice with a 24-minute-longer sleep window: withheld once, 17.56 +/// the other time. The citation went with it — this is Welch segmentation, not +/// Pimentel's surrogate. +/// +/// [tolBrpm] 2.0 and [minConsensus] 0.5 are calibrated against a SURROGATE null +/// (the same nights with their NN values shuffled, which destroys RSA and keeps +/// the sampling geometry): surrogate consensus measured 15–28% across 14 real +/// nights, real consensus 52–85%. Uniform peaks over the ~21 br/min searchable +/// band would put ±2 br/min agreement at 19% by chance, which is what the +/// surrogates show. 50% is ~2.5× chance and sits in the measured gap. Metric rsaRespRate( List nnMs, List nnTimesMs, { required double artifactFraction, - double agreeBrpm = 3.0, + double tolBrpm = 2.0, + double minConsensus = 0.5, double maxArtifact = 0.30, }) { const inputs = ['rr_cleaned', 'beat_times']; @@ -89,74 +167,154 @@ Metric rsaRespRate( ); } - // Pimentel 2017 robustness surrogate: vary the spectral resolution (a - // deterministic analogue of varying the AR model order) and require the HF - // peak to be stable across them. A respiratory peak is sharp & resolution- - // invariant; spurious HRV structure or artifact is not. + // SEARCH CEILING. The window's own beat-rate Nyquist, capped at [respHiHz]. + // The search used to stop at [rsaHiHz] while the guard below tested against + // [respHiHz], so the guard was unreachable and the 24 br/min cap was silent. + final meanNnSec = spanSec / (nnMs.length - 1); + final hiHz = rsaCeilingHz(meanNnSec); + // The ceiling must at least cover the classic HF band. Below that, a rate + // inside the band the literature defines would fold back down into the band + // as an alias and be indistinguishable from a genuine slower one — measured: + // 26 br/min at NN 1400 ms folds to 16.9 br/min with a full-height peak. No + // spectral test can separate the two, so the honest answer is nothing at all. + if (hiHz < rsaHiHz) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'beat rate ${round6(60 / meanNnSec)} bpm resolves only to ' + '${round6(hiHz * 60)} br/min — below the HF band, any peak could be ' + 'an alias; rate withheld', + ); + } + + // WELCH SEGMENTATION. Sub-windows of [rsaSegmentSec] (or half the input when + // it is shorter than two of them), overlapping 50%. Each gets its own + // periodogram on a grid oversampled 4× ITS OWN resolution (1/span) — a grid + // finer than that only re-reads the same bins, which is the trap the old + // 300/450/700 surrogate fell into. + var segSec = rsaSegmentSec; + if (spanSec < 2 * segSec) segSec = spanSec / 2; + if (segSec < 60) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'window spans ${round6(spanSec)} s — needs ≥120 s to test the ' + 'respiratory peak against itself over time', + ); + } final peaks = []; // br/min final peakHz = []; // the same peaks in Hz, index-aligned final peakPwr = []; // their spectral power, index-aligned - for (final grid in const [300, 450, 700]) { - final ls = lombScargle(tSec, nnMs, freqGrid(rsaLoHz, rsaHiHz, grid)); - if (ls == null) continue; - final pk = ls.peakFreq(rsaLoHz, rsaHiHz); - if (pk == null) continue; - // Reject aliasing: a peak at/above Nyquist is not a real breathing rate. - if (pk >= respHiHz) continue; + var atCeiling = 0; + var belowBand = 0; + var thin = 0; + for (var s = tSec.first; s + segSec <= tSec.last + 1e-9; s += segSec / 2) { + final lo = _lowerBound(tSec, s); + final hi = _lowerBound(tSec, s + segSec); + final k = hi - lo; + // A sub-window has to be BOTH beat-dense and time-complete: a dropout in + // the middle leaves few beats spanning the full 5 minutes, and its + // periodogram is a window function, not a spectrum. + if (k < 30 || tSec[hi - 1] - tSec[lo] < segSec * 0.8) { + thin++; + continue; + } + final segT = tSec.sublist(lo, hi); + final segNn = nnMs.sublist(lo, hi); + final segSpan = segT.last - segT.first; + // Per sub-window Nyquist: heart rate moves through the night, so the + // resolvable ceiling does too. A sub-window whose beat rate cannot cover + // the HF band is dropped for the SAME alias reason as the whole window. + final segHi = rsaCeilingHz(segSpan / (k - 1)); + if (segHi < rsaHiHz) { + belowBand++; + continue; + } + final grid = math.max(64, ((segHi - rsaLoHz) * 4 * segSpan).ceil()); + final ls = lombScargle(segT, segNn, freqGrid(rsaLoHz, segHi, grid)); + if (ls == null) { + thin++; + continue; + } + final pk = ls.peakFreq(rsaLoHz, segHi); + if (pk == null) { + thin++; + continue; + } + // A peak pinned to the top of the searchable band means the true rate is + // at or above what this sub-window can resolve. That is an ABSENCE, not a + // rate: reporting the edge would publish the ceiling as a measurement. + if (pk >= segHi - (segHi - rsaLoHz) / (grid - 1)) { + atCeiling++; + continue; + } peaks.add(pk * 60.0); peakHz.add(pk); peakPwr.add(_powerAt(ls, pk)); } - if (peaks.length < 2) { - return const Metric.absent( + if (peaks.length < 3) { + final dropped = atCeiling + belowBand + thin; + final why = dropped == 0 + ? 'only ${peaks.length} usable sub-windows' + : (belowBand >= atCeiling && belowBand >= thin + ? '$belowBand of ${peaks.length + dropped} sub-windows had a beat ' + 'rate too low to cover the HF band (any peak could be an alias)' + : (atCeiling >= thin + ? '$atCeiling of ${peaks.length + dropped} sub-windows peaked ' + 'at/above the resolvable ceiling ' + '(${round6(hiHz * 60)} br/min)' + : '$thin of ${peaks.length + dropped} sub-windows were too ' + 'sparse or gappy to spectrum')); + return Metric.absent( tier: Tier.high, inputs_used: inputs, - note: 'no stable HF respiratory peak resolved', + note: 'no stable HF respiratory peak resolved — $why', ); } - // Agreement gate (the robustness check). - final spread = (peaks.reduce(math.max) - peaks.reduce(math.min)); - if (spread > agreeBrpm) { + // AGREEMENT GATE — over TIME, not over grid resolution. How much of the night + // agrees with the night's own median? + final medBrpm0 = median(peaks)!; + var within = 0; + for (final p in peaks) { + if ((p - medBrpm0).abs() <= tolBrpm) within++; + } + final consensus = within / peaks.length; + if (consensus < minConsensus) { return Metric.absent( tier: Tier.high, inputs_used: inputs, - note: 'HF peak unstable across spectral resolutions ' - '(spread ${round6(spread)} br/min) — withheld', + note: 'HF peak unstable across the window\'s own sub-windows — only ' + '$within of ${peaks.length} ${round6(segSec)}s sub-windows fall ' + 'within ${round6(tolBrpm)} br/min of the median; withheld', ); } // ONE SOURCE for the reported triple. `brpm` used to be median(peaks) while - // `peakHz`/`power` came from the highest-POWER grid, so `peak_hz * 60` and - // `brpm` could disagree by up to [agreeBrpm] inside a single RespEstimate. - // We now pick the MEDOID grid — the grid whose peak is closest to the median - // across grids — and report its rate, frequency and power together, so + // `peakHz`/`power` came from the highest-POWER estimate, so `peak_hz * 60` + // and `brpm` could disagree inside a single RespEstimate. We pick the MEDOID + // sub-window — the one whose peak is closest to the median across + // sub-windows — and report its rate, frequency and power together, so // `brpm == peakHz * 60` holds exactly and `power` is the power measured AT - // the reported frequency. (With three grids the medoid IS the median; with - // two, the nearer of the pair. Robustness still comes from the agreement gate - // above, which already rejected any disagreeing set.) - final medBrpm = median(peaks)!; + // the reported frequency. var best = 0; for (var i = 1; i < peaks.length; i++) { - if ((peaks[i] - medBrpm).abs() < (peaks[best] - medBrpm).abs()) best = i; + if ((peaks[i] - medBrpm0).abs() < (peaks[best] - medBrpm0).abs()) best = i; } final brpm = peaks[best]; final bestPeakHz = peakHz[best]; final bestPower = peakPwr[best]; - // Confidence: high when clean & resolution-stable; penalize artifacts and - // wide spread. Cap below 1 (PRV ceiling). - final conf = clamp( - (1 - artifactFraction) * (1 - spread / (agreeBrpm * 2)), - 0.2, - 0.9, - ); + // Confidence: how much of the window agrees with itself, discounted by + // artifacts. Cap below 1 (PRV ceiling). + final conf = clamp((1 - artifactFraction) * consensus, 0.2, 0.9); return Metric( value: RespEstimate(brpm, bestPeakHz, bestPower, 'rsa'), confidence: conf, tier: Tier.high, inputs_used: inputs, note: 'RSA HF-peak respiratory rate (Lomb-Scargle on native beat times, ' - 'medoid of ${peaks.length} spectral resolutions — brpm, peak_hz and ' - 'power all come from that one grid); PRV-derived; 1 Hz Nyquist caps ' - 'rate at 30 br/min', + 'median of ${peaks.length} ${round6(segSec)}s sub-windows, $within of ' + 'them within ${round6(tolBrpm)} br/min of it — brpm, peak_hz and power ' + 'all come from the medoid sub-window); PRV-derived; this window could ' + 'resolve up to ${round6(hiHz * 60)} br/min', ); } @@ -361,6 +519,20 @@ double _confToVar(double conf) { return 1.0 / (c * c); } +/// First index of [a] whose value is >= [v]. [a] must be non-decreasing. +int _lowerBound(List a, double v) { + var lo = 0, hi = a.length; + while (lo < hi) { + final m = (lo + hi) >> 1; + if (a[m] < v) { + lo = m + 1; + } else { + hi = m; + } + } + return lo; +} + /// Power at (or nearest to) a given frequency in a Lomb-Scargle spectrum. double _powerAt(LombScargle ls, double fHz) { double best = 0; @@ -369,7 +541,7 @@ double _powerAt(LombScargle ls, double fHz) { final d = (pt.freqHz - fHz).abs(); if (d < bestDist) { bestDist = d; - best = pt.power; + best = pt.psd; } } return best; diff --git a/lib/src/onehz/sleep/accounting.dart b/lib/src/onehz/sleep/accounting.dart index 70bf721..28270a4 100644 --- a/lib/src/onehz/sleep/accounting.dart +++ b/lib/src/onehz/sleep/accounting.dart @@ -1,179 +1,16 @@ -// SLEEP/CIRCADIAN TIER-1 — sleep accounting. +// SLEEP — the 3-class stage label shared across the sleep family. // -// Given the van Hees REST window + a per-second wake/sleep classification -// inside it (from the immobility mask, optionally refined by the autonomic -// stager), compute the standard accounting figures (AASM-style definitions): -// - onset = first sleep second within the window -// - offset = last sleep second within the window -// - TST = total sleep time (sum of sleep seconds in [onset, offset]) -// - WASO = wake after sleep onset (wake seconds in [onset, offset]) -// - SPT = sleep-period time (offset − onset) -// - efficiency = TST / in-bed, where in-bed = (offset − onset + 1), the sleep -// PERIOD itself (NOT the whole captured mask). Per ARCHITECTURE_V2: -// "efficiency = TST/in-bed". Onset latency and post-offset tail are excluded -// from the denominator so efficiency reflects [onset..offset] only. -// -// NREM–REM cycle detection (~90 min ultradian): we count cycles as the number -// of REM episodes (or, if no stage labels, the number of distinct sleep bouts -// separated by WASO), capped to a plausible 4–6/night. This is a COUNT, not a -// staged hypnogram — honesty: we never emit N1/N2/N3. - -import 'dart:math' as math; -import '../types.dart'; -import '../util.dart'; - -/// 3-class label used across the sleep family. +// This file used to also host `sleepAccounting`, a second implementation of +// onset/offset/TST/WASO/efficiency/cycles. It was exported from the barrel, +// called by nothing outside one test, and defined efficiency against the sleep +// PERIOD (offset − onset + 1) while the figure that actually ships defines it +// against the observed in-bed window (`segment.dart`) — two different numbers +// under one name, waiting for a future caller to pick the wrong one. Deleted +// rather than reconciled: segmentation is the SINGLE source (ARCHITECTURE_V2 +// invariant 2/4), and a second accounting path is exactly what that invariant +// exists to forbid. + +/// 3-class label used across the sleep family. There is no 'unobserved' member: +/// unobserved time is carried per-second by `SleepSegmentation.stages4` and in +/// aggregate by `SleepSegmentation.unobservedSec`. enum SleepStage { wake, nrem, rem } - -class SleepAccounting { - final int onsetIdx; // first sleep second (rel. to window start input) - final int offsetIdx; // last sleep second - final int tstSec; // total sleep time - final int wasoSec; // wake after sleep onset - final int sptSec; // sleep-period time (offset−onset) - final double efficiencyPct; // TST / time-in-bed window - final int cycles; // detected NREM-REM cycles (~90 min) - const SleepAccounting({ - required this.onsetIdx, - required this.offsetIdx, - required this.tstSec, - required this.wasoSec, - required this.sptSec, - required this.efficiencyPct, - required this.cycles, - }); - Map toJson() => { - 'onset_idx': onsetIdx, - 'offset_idx': offsetIdx, - 'tst_sec': tstSec, - 'waso_sec': wasoSec, - 'spt_sec': sptSec, - 'efficiency_pct': round6(efficiencyPct), - 'cycles': cycles, - }; -} - -/// Compute sleep accounting from a per-second sleep/wake classification of the -/// in-bed window. [asleep] = true where the second is classified sleep. -/// Efficiency = TST / in-bed, where in-bed = (offset − onset + 1) — the sleep -/// PERIOD bounded by first/last sleep second, NOT the length of [asleep]. -/// [stages] optional per-second 3-class labels (same length); when supplied, -/// REM episode count drives the cycle estimate. -Metric sleepAccounting( - List asleep, { - List? stages, -}) { - const inputs = ['in_bed_sleep_wake']; - final n = asleep.length; - if (n < 60) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'in-bed window too short for sleep accounting', - ); - } - // Onset / offset. - var onset = -1, offset = -1; - for (var i = 0; i < n; i++) { - if (asleep[i]) { - onset = i; - break; - } - } - for (var i = n - 1; i >= 0; i--) { - if (asleep[i]) { - offset = i; - break; - } - } - if (onset < 0) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'no sleep detected within the in-bed window', - ); - } - - final spt = offset - onset; // sleep-period time (offset − onset) - var tst = 0, waso = 0; - for (var i = onset; i <= offset; i++) { - if (asleep[i]) { - tst++; - } else { - waso++; - } - } - // efficiency = TST / in-bed, where in-bed = [onset..offset] inclusive - // (offset − onset + 1), the sleep PERIOD — NOT the whole captured mask `n`. - final inBed = offset - onset + 1; - final efficiency = inBed > 0 ? 100.0 * tst / inBed : 0.0; - - // Cycle count. - int cycles; - if (stages != null && stages.length == n) { - // Count REM episodes (≥3 contiguous min REM, gap-tolerant). - cycles = _countEpisodes( - stages.map((s) => s == SleepStage.rem).toList(), - onset, - offset, - minLenSec: 180, - bridgeSec: 300, - ); - } else { - // No stages: estimate from SPT assuming ~90-min ultradian cycles. - cycles = (spt / (90 * 60)).round(); - } - cycles = cycles.clamp(0, 6); - - // Confidence: high efficiency + plausible duration => trust. - final conf = clamp(0.4 + spt / (8 * 3600) * 0.5, 0.4, 0.9); - return Metric( - value: SleepAccounting( - onsetIdx: onset, - offsetIdx: offset, - tstSec: tst, - wasoSec: waso, - sptSec: spt, - efficiencyPct: efficiency, - cycles: cycles, - ), - confidence: conf, - tier: Tier.high, - inputs_used: stages != null ? [...inputs, 'stages_3class'] : inputs, - note: 'onset/offset/WASO/TST/efficiency; cycles ~90-min ultradian ' - '(REM-episode count when staged); 3-class only, never N1/N2/N3', - ); -} - -/// Count contiguous episodes where [flag] is true, within [lo,hi], requiring -/// each episode ≥ [minLenSec] and bridging gaps < [bridgeSec]. -int _countEpisodes(List flag, int lo, int hi, - {required int minLenSec, required int bridgeSec}) { - var count = 0; - var i = lo; - while (i <= hi) { - if (!flag[i]) { - i++; - continue; - } - var j = i; - while (j <= hi) { - if (flag[j]) { - j++; - continue; - } - var k = j; - while (k <= hi && !flag[k] && (k - j) < bridgeSec) { - k++; - } - if (k <= hi && flag[k] && (k - j) < bridgeSec) { - j = k; - } else { - break; - } - } - if ((j - i) >= minLenSec) count++; - i = math.max(j + 1, i + 1); - } - return count; -} diff --git a/lib/src/onehz/sleep/advanced_stager.dart b/lib/src/onehz/sleep/advanced_stager.dart index 8f12676..fa2358a 100644 --- a/lib/src/onehz/sleep/advanced_stager.dart +++ b/lib/src/onehz/sleep/advanced_stager.dart @@ -120,7 +120,10 @@ class HypnogramMetrics { final int tibS; final int tstS; final int sptS; - final int solS; + /// Sleep-onset latency (s). NULL when NOTHING staged as sleep — reporting + /// the whole time in bed as "time to fall asleep" alongside TST 0 is a + /// fabricated measurement, not an abstention. + final int? solS; final double remLatencyS; // NaN if no REM final int wasoS; final double efficiency; // [0,1] @@ -1852,7 +1855,8 @@ class AdvancedSleepStager { if (s.stage == 'rem') remS += d; if (s.stage == 'light') lightS += d; } - int onset, sptEnd, sol; + int onset, sptEnd; + int? sol; if (sleepSegs.isNotEmpty) { onset = sleepSegs.first.start; sptEnd = sleepSegs.last.end; @@ -1860,7 +1864,7 @@ class AdvancedSleepStager { } else { onset = session.end; sptEnd = session.end; - sol = tib; + sol = null; // never observed sleep — no latency to report } final remSegs = sleepSegs.where((s) => s.stage == 'rem').toList(); final remLatency = diff --git a/lib/src/onehz/sleep/cardio_stager.dart b/lib/src/onehz/sleep/cardio_stager.dart index 417eee4..7a7957f 100644 --- a/lib/src/onehz/sleep/cardio_stager.dart +++ b/lib/src/onehz/sleep/cardio_stager.dart @@ -36,14 +36,31 @@ // (DREAMT) via `tool/stager_harness.dart`, that design scored kappa 0.036 — // deep PPV 5.7% against a 4.5% base rate, REM PPV 12.1% against 14.0%, i.e. at // or below chance for both minority classes. Per-subject effect sizes explain -// why: RMSSD does not separate the stages at all (54% of subjects, d -0.02; -// Herzig 2017 reports the same flatness on PSG+ECG), mean HR in deep sleep runs +// why: RMSSD did not separate the stages in our measurement (54% of subjects, +// d -0.02) — BUT SEE THE NEXT PARAGRAPH, that number is probably wrong — mean +// HR in deep sleep runs // HIGHER than light on the wrist (62% of subjects, d +0.31 — the old gate was // inverted), and the two axes that DO separate (Rk d -0.53/+0.43, sdnn // -0.41/+0.32) were respectively throttled to z>4 and not computed at all. // AND-ing one good gate with one null and one inverted gate is why deep sleep // emerged as isolated 30-second specks that the 3-min bout rule then deleted. // A weighted sum degrades gracefully where a conjunction vetoes. +// +// THE RMSSD d -0.02 IS PROBABLY WRONG, AND IT IS NOT HERZIG'S FAULT (SLP-06). +// R(k) is defined below as mean |ΔIHR| over the window — a successive-difference +// statistic on the SAME beat train RMSSD uses, differing only by the 1/RR² +// rescaling and mean-abs vs RMS. Measured on three real rest blocks at the +// windows this file actually uses (R(k) ±90 s, RMSSD ±150 s) they correlate +// 0.876 / 0.778 / 0.778; at a matched ±90 s, 0.928 / 0.853 / 0.856. And +// corr(R(k), mean HR) is 0.065 / 0.282 / -0.341, so R(k) is not a disguised +// heart-rate feature either. Two statistics correlated 0.78-0.93 on the target +// signal cannot both score d -0.02 and d -0.53 against the same labels; under a +// bivariate-normal argument RMSSD should be at least ~0.4 in the same +// direction. So the local DREAMT effect-size measurement for RMSSD is +// unreliable and Herzig 2017 is currently lending its authority to it. +// TODO: re-run the DREAMT effect size for RMSSD at ±90 s and publish both +// numbers side by side. DO NOT move the shipped weights on the strength of +// this — it says the measurement is untrustworthy, not which way it moves. // Post: Webster continuity rescore (bridges brief arousals into sleep — this is // what kills the over-call) + consolidateSleepStages (no single-epoch flicker). // @@ -76,6 +93,12 @@ class CardioStagerResult { const int _epochSec = 30; +/// Minimum fraction of epochs that must carry a valid HR before this stager +/// will label anything. Below it every HR-relative gate is silent and the +/// classifier degenerates to "all light sleep" — see the gate in +/// [classifyCardioEpochs]. +const double minHrCoverage = 0.5; + // The previous rule OR-ed three boolean axes (RMSSD drop as PRIMARY, plus // LF/HF and R(k) throttled to z>4.0) and was calibrated against a single // Apple-Watch-labelled night. Measurement against 99 PSG-labelled wrist nights @@ -137,9 +160,24 @@ const double _wDeepLfhf = 0.33; /// Held-out result at the shipped cutoffs (49 unseen subjects): kappa 0.132, /// Deep 56.0% sens / 10.9% PPV, REM 51.9% / 21.5%, against base rates of 4.5% /// Deep and 14% REM — so both minority classes are called well above chance -/// where the previous rules sat at or below it. Still far short of the -/// 0.60-0.66 literature ceiling; the bulk of what remains is the WAKE rule -/// (11% sensitivity), untouched by this change. +/// where the previous rules sat at or below it. The bulk of what remains is the +/// WAKE rule (11% sensitivity), untouched by this change. +/// +/// THE COMPARATOR (SLP-08). The honest one is the consumer-device range in +/// `segment.dart`'s [kMaxSleepConfidence] — κ 0.21-0.53 on wrist devices vs PSG +/// — which we are BELOW, the vendor firmware on this very band scoring 0.37. +/// This used to measure itself against a "0.60-0.66 literature ceiling". That +/// is Radha et al. 2021 (npj Digital Medicine 4), four-class κ 0.65 ± 0.11, +/// accuracy 76.4 ± 7.6 %, from a deep recurrent network PRETRAINED ON 584 ECG +/// RECORDINGS and transfer-learned to 101 CONTINUOUS WRIST-PPG recordings with +/// beat-to-beat intervals at native sampling resolution. It is the ceiling for +/// continuous-PPG deep models, not for rules on our substrate: `rr_ts_ms` is +/// `rec_ts * 1000` on 100 % of rows, RR is band-saturated at [333, 2400], +/// 3.2-11.8 % of beats in a rest block share a timestamp with another beat, +/// coverage runs 78.5-92.9 %, and there is no PPG waveform in the database at +/// all. Quoting it as a ceiling made 0.132 look like a gap to close rather than +/// a different problem. Reaching 0.65 needs a continuous PPG or true beat +/// timestamps — a protocol/edge change, not a stager change. /// /// HONESTY: proportion-matching is distributional plausibility, NOT accuracy. /// It says the hypnogram has a believable shape, not that any given epoch is @@ -310,7 +348,8 @@ SleepUserProfile? cardioUserProfile; /// internal buffer. The edge sets this before a night's staging and reads the /// buffer with [takeCardioObservations] afterward to fold the MAIN sleep. bool cardioRecordObservations = false; -final List _cardioObservations = []; +final List _cardioObservations = + []; /// Drain and clear the recorded observations (returns a copy). List takeCardioObservations() { @@ -405,7 +444,10 @@ CardioStagerResult cardioStager( } motion[e] = ms / (t - s); // hr mean/sd over valid (>0) seconds - final hv = [for (var i = s; i < t; i++) if (hr1hz[i] > 0) hr1hz[i]]; + final hv = [ + for (var i = s; i < t; i++) + if (hr1hz[i] > 0) hr1hz[i] + ]; if (hv.isNotEmpty) hr[e] = mean(hv)!; if (hv.length >= 2) hrSd[e] = stddev(hv) ?? 0; // rmssd over RR beats within a ±2.5-min window centred on the epoch @@ -542,22 +584,34 @@ CardioStagerResult classifyCardioEpochs( // local, not this repositioning-detection threshold. final motSample = [for (final m in motion) m]; final motMed = median(motSample) ?? 0; - final motMad = (mad(motSample) ?? 0).clamp(1e-9, double.infinity); + // MAD == 0 means MORE THAN HALF the epochs share one motion value — the + // guaranteed case when accel is mostly absent. Clamping to 1e-9 made + // `bigMoveCut` = motMed + 5e-9, so every epoch fractionally above the median + // became a "big move" and drove the WAKE gate. This is the same MAD-0 class + // that produced the blank-readiness and nocturnalRhr bugs, and RobustScale.of + // already refuses it. Abstain from the motion axis instead: null thresholds + // mean `still` is true everywhere (baseline selection falls back to the whole + // window) and `bigMove` is never asserted, so WAKE has to come from HR. + final motMadRaw = mad(motSample) ?? 0; + final motUsable = motMadRaw > 0 && motMadRaw.isFinite; + final motMad = motUsable ? motMadRaw : 0.0; // Personal-baseline blend (P2): pull each LOCAL threshold a bounded fraction // toward the sleeper's rolling profile value. `_pw` (≤0.5) grows with nights, // so per-night-local always leads; a null profile axis ⇒ no blend for it. final double _pw = profile?.personalWeight ?? 0.0; double blendP(double local, double? personal) => - (personal == null || _pw == 0) ? local : local * (1 - _pw) + personal * _pw; + (personal == null || _pw == 0) + ? local + : local * (1 - _pw) + personal * _pw; // "still" (for baseline selection) = motion near the night's typical low. final stillCut = blendP(motMed + 1.5 * motMad, profile?.enmoStillCut); - bool still(int e) => motion[e] <= stillCut; + bool still(int e) => !motUsable || motion[e] <= stillCut; // "big move" = clearly elevated motion (getting up / large reposition), used // for the WAKE decision — a much higher bar than `still` so normal in-sleep // repositioning (which the van-Hees window already certified as sleep) is NOT // mistaken for wake. final bigMoveCut = blendP(motMed + 5.0 * motMad, profile?.enmoMoveCut); - bool bigMove(int e) => motion[e] > bigMoveCut; + bool bigMove(int e) => motUsable && motion[e] > bigMoveCut; final sleepHr = [ for (var e = 0; e < nEpoch; e++) @@ -572,8 +626,19 @@ CardioStagerResult classifyCardioEpochs( // fabrication look like a real baseline. A window with SOME HR still gets the // whole-window mean as the baseline when no epoch qualifies as `still` (that // is a real measurement, just not a quiet one). - final hrAll = [for (final h in hr) if (!h.isNaN) h]; - if (hrAll.isEmpty) return _abstain(epochSec); + final hrAll = [ + for (final h in hr) + if (!h.isNaN) h + ]; + // ...and "no HR anywhere" is not the same test as "enough HR to stage with". + // The gate used to be `hrAll.isEmpty`, i.e. ONE valid epoch in 900 passed it. + // With HR NaN for most epochs neither `hrUp` nor `hrTowardWake` can ever be + // true, so no epoch can be wake and none can be REM: a sparse HR stream (the + // gen5 device family) came out ~100 % light sleep, TST = time in bed, + // efficiency ~100 %, at confidence up to 0.6. Abstention is the honest + // failure. + final hrCov = nEpoch == 0 ? 0.0 : hrAll.length / nEpoch.toDouble(); + if (hrCov < minHrCoverage) return _abstain(epochSec); final hrMedGlobal = median(sleepHr) ?? mean(hrAll)!; // ── LOCAL rolling HR baseline for the WAKE/REM autonomic gates ───────────── @@ -600,8 +665,7 @@ CardioStagerResult classifyCardioEpochs( // median already fixed for the arousal gate above. const int _hrWinEpochs = 180; // 90 min half-window — one ultradian cycle final hrMedLocal = List.filled(nEpoch, hrMedGlobal); - final hrArousalLocal = - List.filled(nEpoch, hrMedGlobal + 6.0); + final hrArousalLocal = List.filled(nEpoch, hrMedGlobal + 6.0); final hrP25Local = List.filled(nEpoch, hrMedGlobal); for (var e = 0; e < nEpoch; e++) { final lo = math.max(0, e - _hrWinEpochs); @@ -678,6 +742,15 @@ CardioStagerResult classifyCardioEpochs( // WAKE is autonomic-led: HR risen to/above the arousal threshold, OR a big // movement that ALSO carries some HR lift (truly up), OR sustained big // movement. Movement at sleeping HR = repositioning, NOT wake. + // ASYMMETRY, DELIBERATELY LEFT HERE AND HANDLED UPSTREAM. Every gate that + // can produce WAKE or REM needs a non-NaN HR, while the fall-through below + // is an unconditional NREM — so an epoch with no heart rate at all can only + // ever come out asleep. [minHrCoverage] catches the all-or-nothing case; + // between 50 % and 100 % coverage it does not. There is no fourth + // [SleepStage] to express "no cardiac evidence" in, so the abstention is + // made where the numbers are: `segmentSleep` labels a second with no HR + // within half an epoch 'unobserved' and keeps it out of TST, WASO and the + // efficiency denominator. final hrUp = !hr[e].isNaN && hr[e] >= hrArousal; final bigPrev = e > 0 && bigMove(e - 1); final bigMoveWake = @@ -721,11 +794,14 @@ CardioStagerResult classifyCardioEpochs( // // rmssd and mean HR are deliberately ABSENT from both scores. Neither // earns a weight, and hr's deep contribution was actively harmful. - final rkZ = (sleepRk.length >= 4 && !rk[e].isNaN) ? rkScale?.z(rk[e]) : null; - final sdnnZ = - (sleepSdnn.length >= 4 && !sdnn[e].isNaN) ? sdnnScale?.z(sdnn[e]) : null; - final lfhfZ = - (sleepLfhf.length >= 4 && !lfhf[e].isNaN) ? lfhfScale?.z(lfhf[e]) : null; + final rkZ = + (sleepRk.length >= 4 && !rk[e].isNaN) ? rkScale?.z(rk[e]) : null; + final sdnnZ = (sleepSdnn.length >= 4 && !sdnn[e].isNaN) + ? sdnnScale?.z(sdnn[e]) + : null; + final lfhfZ = (sleepLfhf.length >= 4 && !lfhf[e].isNaN) + ? lfhfScale?.z(lfhf[e]) + : null; // Same >=4-sample floor and unmeasurable-epoch rejection as the axes above. // hrSd[e] == 0 means "fewer than 2 valid HR samples", never "perfectly // steady" — see the sleepHrSd comment. An absent input must contribute @@ -827,10 +903,20 @@ CardioStagerResult classifyCardioEpochs( final tot = sm.length.toDouble(); // Confidence: a wrist ESTIMATE ceiling, scaled by RR coverage (RMSSD is what // makes REM/deep honest — with no RR we're motion+HR only, lower confidence). - final rrCov = nEpoch == 0 + final rrCov = nEpoch == 0 ? 0.0 : sleepRmssd.length / nEpoch.toDouble(); + // HR coverage is folded in: `rrCov` alone said nothing about whether the + // HR-relative gates (wake, REM floor, deep trough) had anything to fire on. + final hrCovConf = nEpoch == 0 ? 0.0 - : sleepRmssd.length / nEpoch.toDouble(); - final conf = clamp(0.35 + 0.25 * rrCov, 0.3, 0.6); + : clamp( + [ + for (final h in hr) + if (!h.isNaN) h + ].length / + nEpoch.toDouble(), + 0.0, + 1.0); + final conf = clamp((0.35 + 0.25 * rrCov) * hrCovConf, 0.15, 0.6); return CardioStagerResult( StagerResult( @@ -988,6 +1074,7 @@ void _websterRescore(List sm, int epochSec) { } return c; } + int runAfter(int i) { var c = 0, k = i + 1; while (k <= lastSleep && isSleep(snap[k])) { @@ -996,14 +1083,23 @@ void _websterRescore(List sm, int epochSec) { } return c; } - // Context (min sleep flanking) → max bridgeable wake (min). Slightly more - // generous than Webster's classic table: the van Hees window already certified - // this span as the consolidated rest period, so brief arousals inside it are - // far more likely repositioning than true wake. + + // Context (min sleep flanking) → max bridgeable wake (min). THE PUBLISHED + // TABLE, unmodified: Webster et al. 1982 / Cole et al. 1992 rescoring rules — + // 4 min of sleep bridges ≤1 min of wake, 10 bridges ≤3, 15 bridges ≤4. + // + // This used to run [15→10], [10→5], [4→2] — 2.5x the published bridge — on + // the argument that the van Hees window had already certified the span as the + // consolidated rest period. That argument does not hold: on the forced-window + // paths (manual / confirmed / auto_fallback) van Hees never runs at all, and + // even where it does, a genuine 9-minute 3 a.m. awakening flanked by 15 min of + // sleep was being relabelled light sleep. TST was over-reported, WASO + // under-reported and efficiency inflated, potentially several times a night, + // disclosed only in a code comment the user never sees. final rules = >[ - [minToEp(15), minToEp(10)], - [minToEp(10), minToEp(5)], - [minToEp(4), minToEp(2)], + [minToEp(15), minToEp(4)], + [minToEp(10), minToEp(3)], + [minToEp(4), minToEp(1)], ]; var i = onset; while (i <= lastSleep) { @@ -1064,17 +1160,18 @@ void _websterRescore(List sm, int epochSec) { } beats.add(v); // Rebase to the window start (lo), NOT absolute epoch ms. Lomb–Scargle is - // time-shift invariant (the τ phase reference cancels any offset), so the - // LF/HF output is unchanged — but this keeps beat times in [0, 180] s - // instead of ~1.75e9 s. Absolute epoch seconds force every sin/cos in the - // periodogram onto libm's __kernel_rem_pio2 multi-precision slow path - // (args ~9e9 rad), which — run per 30-s epoch over a full night on the - // main isolate — caused main-thread ANRs on Android (Crashlytics: libm.so - // __kernel_rem_pio2 / sin / cos, 0.9.13). - beatTsSec.add((ts - lo) / 1000.0); + // time-shift invariant (the τ phase reference cancels any offset), so the + // LF/HF output is unchanged — but this keeps beat times in [0, 180] s + // instead of ~1.75e9 s. Absolute epoch seconds force every sin/cos in the + // periodogram onto libm's __kernel_rem_pio2 multi-precision slow path + // (args ~9e9 rad), which — run per 30-s epoch over a full night on the + // main isolate — caused main-thread ANRs on Android (Crashlytics: libm.so + // __kernel_rem_pio2 / sin / cos, 0.9.13). + beatTsSec.add((ts - lo) / 1000.0); prev = v; } - if (beats.length < 16) return (lfhf: null, rk: null); // spectral stability gate + if (beats.length < 16) + return (lfhf: null, rk: null); // spectral stability gate // R(k): mean absolute successive difference of instantaneous HR (bpm). var rkSum = 0.0; var rkCnt = 0; diff --git a/lib/src/onehz/sleep/cpc.dart b/lib/src/onehz/sleep/cpc.dart index c009060..9216359 100644 --- a/lib/src/onehz/sleep/cpc.dart +++ b/lib/src/onehz/sleep/cpc.dart @@ -1,33 +1,46 @@ -// SLEEP/CIRCADIAN TIER MED-HIGH — Cardiopulmonary Coupling (CPC). +// SLEEP/CIRCADIAN — Cardiopulmonary Coupling (CPC). WITHDRAWN. // -// Thomas et al. 2005 (Sleep) — CPC quantifies sleep stability from the -// cross-spectral COHERENCE × power of two signals derived from the ECG/PPG: -// (1) heart-rate variability (here: the NN/RR series), and -// (2) an ECG-derived respiration (EDR) surrogate. We have continuous beat-to- -// beat RR, so we use an RSA/RIIV-style RESPIRATION SURROGATE built from -// the RR amplitude itself (the respiratory sinus arrhythmia signature), -// a published substitute for EDR. +// IF YOU GOT HERE FROM A GREP: this function has zero callers ON PURPOSE. it is +// not an oversight, not half-wired, not waiting for someone with time. every +// sweep of this repo re-proposes it, because a tested function with no caller +// reads as a loose end from the outside and as a decision only once you open +// the file. you have opened the file. the reason is below, and edge's +// compute/onehz_pipeline.dart records the same verdict where the call would go. // -// Thomas bands (cycles/beat in the original; we map to Hz on the resampled -// tachogram — see below): -// - High-Frequency Coupling (HFC, ~0.1–0.4 Hz): STABLE NREM sleep. -// - Low-Frequency Coupling (LFC, ~0.01–0.1 Hz): UNSTABLE sleep / CAP / -// apnea-rich periods. -// - Very-Low-Frequency (VLFC, <0.01 Hz): wake/REM. +// Thomas et al. 2005 (Sleep) derives CPC from the cross-spectral coherence of +// TWO signals: the RR tachogram, and an ECG-DERIVED RESPIRATION (EDR) surrogate +// taken from the R-wave amplitude — a separate channel that is modulated by +// breathing but is NOT a function of the beat times. // -// Method here (deterministic, Lomb-Scargle based — no FFT-on-resample needed): -// * Build the RR tachogram on native beat times. -// * Derive a respiration surrogate = the band-limited RSA component of RR. -// * Compute the CROSS-coherence-weighted spectrum: at each frequency, the -// coupling power = sqrt(P_rr(f) · P_resp(f)) · coherence(f), where we -// approximate coherence by the normalized cross term. Integrate per band. -// * Report HFC, LFC, VLFC band powers + the CPC ratio (HFC / LFC) — the -// stability index. High HFC/LFC ⇒ stable sleep. +// This implementation had no such channel. Its "respiration surrogate" was the +// NN series itself, linearly detrended: // -// This is a SCREEN (sleep-stability spectrogram + apnea-risk hint), not a -// diagnosis — the catalog tier is MED-HIGH. +// resp = detrend([v - mean(nnMs) for v in nnMs]) +// +// which differs from the tachogram only by a straight line. The coupling +// spectrum sqrt(P_rr . P_resp) therefore collapses to P_rr, and `cpc_ratio` +// came out equal to the plain RR periodogram HF/LF ratio — measured 84.493910 +// vs 84.493190 on a 9000-beat synthetic, a ratio of 1.0000085. It was the LF/HF +// of the same tachogram we already publish as `lf_hf`, wearing the name of a +// different, published, sleep-stability measure. +// +// The honest options were "derive a real EDR" or "stop shipping it". We cannot +// derive an EDR here — this function is handed beat intervals and beat times +// and nothing else, and a respiration channel independent of the tachogram is +// by definition not recoverable from them. So the metric is WITHDRAWN: it now +// abstains, with the reason attached, and the UI element it fed should be +// DELETED rather than made to explain a dash. +// +// Wiring it back up means changing the signature to take an actual respiration +// channel (PPG AC amplitude / RIIV, cf. respiration/resp_rate.dart) and +// validating the result against something. Until then there is no number here. +// +// RSA-derived respiration DOES NOT QUALIFY as that channel. it comes out of the +// same tachogram, so it reproduces exactly the failure above with more steps. +// the input has to be independent of the beat times or the coupling spectrum +// collapses again, and shipping the collapse under a clinical-sounding name is +// republishing `lf_hf` — already charted — as a second, fake metric. -import 'dart:math' as math; import '../types.dart'; import '../util.dart'; @@ -55,123 +68,26 @@ class CpcResult { /// CPC from a cleaned NN series + its beat times. /// -/// [nnMs] cleaned NN intervals (ms). [nnTimesMs] matching cumulative beat times -/// (ms). Both from `correctRr`. The respiration surrogate is built internally. +/// ALWAYS ABSENT — see the file header. There is no respiration channel in this +/// signature that is independent of [nnMs], so there is no coupling to measure; +/// the previous implementation republished the RR periodogram's own HF/LF ratio +/// under Thomas 2005's name. The signature is kept so callers keep compiling +/// while the key is removed at their end. +@Deprecated( + 'WITHDRAWN — this never measured cardiopulmonary coupling. Its "respiration ' + 'surrogate" was the NN series itself, so cpc_ratio equalled the RR ' + 'periodogram HF/LF ratio (measured agreement 1.0000085). Delete the cpc_ratio ' + 'surface; reinstate only with a real respiration channel in the signature.', +) Metric cardiopulmonaryCoupling( List nnMs, List nnTimesMs, ) { - const inputs = ['rr_corrected', 'rsa_respiration_surrogate']; - final n = nnMs.length; - if (n < 60 || nnTimesMs.length != n) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'too few beats for cardiopulmonary coupling', - ); - } - - // Times in seconds (Lomb-Scargle expects consistent unit → Hz output). - final tSec = [for (final t in nnTimesMs) t / 1000.0]; - - // Respiration surrogate (RSA): the RR series band-limited to the respiratory - // band is itself the EDR substitute. We use the same native beat times but a - // high-pass-detrended copy of NN to emphasize the respiratory modulation. - final nnMean = mean(nnMs)!; - final resp = _detrend([for (final v in nnMs) v - nnMean]); - - // Frequency grid over the coupling bands (VLF..HF), in Hz. - final freqs = freqGrid(0.001, 0.45, 200); - final lsRr = lombScargle(tSec, nnMs, freqs); - final lsResp = lombScargle(tSec, resp, freqs); - if (lsRr == null || lsResp == null) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'spectral estimate degenerate (no variance)', - ); - } - - // Coupling spectrum = geometric-mean coherence: at each f, the coupled power - // is sqrt(P_rr·P_resp) (high only where BOTH have power — a coherence proxy). - final coupling = []; - var domF = 0.0, domP = double.negativeInfinity; - for (var i = 0; i < freqs.length; i++) { - final p = math.sqrt( - math.max(0, lsRr.spectrum[i].power) * - math.max(0, lsResp.spectrum[i].power)); - coupling.add(LsPoint(freqs[i], p)); - if (p > domP) { - domP = p; - domF = freqs[i]; - } - } - final couplingLs = LombScargle(coupling); - - final vlfc = couplingLs.bandPower(0.001, 0.01); - final lfc = couplingLs.bandPower(0.01, 0.1); - final hfc = couplingLs.bandPower(0.1, 0.4); - - // A non-finite band power means the input itself was poisoned (a NaN/±inf RR - // or beat time survives every arithmetic step, and `variance <= 0` does not - // catch NaN). Nothing measured here is real, so nothing is reported. - if (!hfc.isFinite || !lfc.isFinite || !vlfc.isFinite) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'coupling spectrum non-finite (NaN/inf in the NN series or beat ' - 'times) — no coupling was measured', - ); - } - - // Thomas et al. 2005 defines the stability index as the RATIO of high- to - // low-frequency coupling power. With no LFC power at all the ratio is a - // DIVISION BY ZERO — mathematically undefined, not "infinitely stable" (and - // 0/0 is not "maximally unstable" either). The published method says nothing - // about a spectrum with an empty 0.01–0.1 Hz band, so we abstain rather than - // report a number. (Pre-2026-07 this emitted the sentinel 999.0 inside a live - // Metric at confidence up to 0.85, which downstream read as a real, - // extraordinarily stable night.) - if (lfc <= 0) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'no low-frequency (0.01-0.1 Hz) coupling power — the Thomas 2005 ' - 'HFC/LFC stability ratio is undefined here, not "perfectly stable"', - ); - } - final ratio = hfc / lfc; - if (!ratio.isFinite) { - return const Metric.absent( - tier: Tier.high, - inputs_used: inputs, - note: 'HFC/LFC stability ratio overflowed — not a measurement', - ); - } - - // Confidence grows with record length; capped (a screen, not a diagnosis). - final conf = clamp(n / 3600.0, 0.3, 0.85); - return Metric( - value: CpcResult( - hfc: hfc, - lfc: lfc, - vlfc: vlfc, - cpcRatio: ratio, - dominantHz: domF, - ), - confidence: conf, + return const Metric.absent( tier: Tier.high, - inputs_used: inputs, - note: 'Thomas 2005 CPC via Lomb-Scargle coupling spectrum; ' - 'HFC=stable NREM, LFC=unstable/apnea-rich. Screen, not diagnosis', + inputs_used: ['rr_corrected'], + note: 'cardiopulmonary coupling needs a respiration channel independent of ' + 'the beat times (Thomas 2005 uses an ECG-derived respiration); we have ' + 'only the tachogram, so no coupling is measured', ); } - -/// Remove a linear trend (OLS) from a series — leaves the oscillatory part. -List _detrend(List y) { - final fit = olsFit(y); - if (fit == null) return y; - return [ - for (var i = 0; i < y.length; i++) y[i] - (fit.slope * i + fit.intercept) - ]; -} diff --git a/lib/src/onehz/sleep/hr_fallback.dart b/lib/src/onehz/sleep/hr_fallback.dart index 89bc9f7..9592571 100644 --- a/lib/src/onehz/sleep/hr_fallback.dart +++ b/lib/src/onehz/sleep/hr_fallback.dart @@ -46,6 +46,7 @@ HrLedWindow? hrLedSleepWindow( int minDurationSec = 2 * 3600, int bridgeGapSec = 30 * 60, double smoothSec = 300, + int maxSampleGapSec = 5 * 60, }) { final n = hr1hz.length < tsSec.length ? hr1hz.length : tsSec.length; if (n < minDurationSec ~/ 2) return null; @@ -87,13 +88,22 @@ HrLedWindow? hrLedSleepWindow( } var j = i; while (j < n) { + // A HOLE IN THE RECORD ends the run. Two "low" samples three hours apart + // are two observations, not a three-hour low-HR stretch — and the run's + // duration below is WALL CLOCK, so without this the proposed window (the + // one the user is shown and asked to confirm) was inflated by hours of + // data we never had. + if (j > i && tsSec[j] - tsSec[j - 1] > maxSampleGapSec) break; if (low(j)) { j++; continue; } // Look ahead: bridge a short non-low gap (arousal / brief wake) by TIME. var k = j; - while (k < n && !low(k) && (tsSec[k] - tsSec[j]) < bridgeGapSec) { + while (k < n && + !low(k) && + (tsSec[k] - tsSec[j]) < bridgeGapSec && + (k == 0 || tsSec[k] - tsSec[k - 1] <= maxSampleGapSec)) { k++; } if (k < n && low(k) && (tsSec[k] - tsSec[j]) < bridgeGapSec) { diff --git a/lib/src/onehz/sleep/nap.dart b/lib/src/onehz/sleep/nap.dart index 3437ef0..c2e4418 100644 --- a/lib/src/onehz/sleep/nap.dart +++ b/lib/src/onehz/sleep/nap.dart @@ -10,13 +10,20 @@ // detector instead, run strictly on the complement of the main sleep window. // // METHOD -// 1. van Hees z-angle immobility (`immobilityMask`) — the SAME primitive the -// nocturnal spine uses. An angle is orientation-invariant, so it does not -// inherit the ~13% spread in |accel| across static wrist postures that -// makes magnitude-based stillness false-positive on a merely resting arm. +// 1. van Hees z-angle immobility (`immobilityMask`). An angle is +// orientation-invariant, so it does not inherit the ~13% spread in |accel| +// across static wrist postures that makes magnitude-based stillness +// false-positive on a merely resting arm. +// NOT the nocturnal spine's stillness test — this used to claim it was. +// `AdvancedSleepStager` uses `_gravityDeltas`, the norm of the DIFFERENCE +// between successive gravity vectors. Both are orientation-CHANGE +// detectors and neither is the |accel| magnitude test van_hees.dart argues +// against, but they are different features over different windows and the +// two detectors have never been shown to agree on real nights. // 2. Enumerate EVERY immobility bout (the nocturnal path keeps only the // longest), bridging brief arousals. -// 3. Reject hard: off-wrist, charging/workout spans, the main sleep window. +// 3. Reject hard: off-wrist, charging/workout spans, the main sleep window, +// and bouts built on seconds the band never measured (`minNapAccelCoverage`). // 4. Require an autonomic signature — median HR inside the bout at or below // `napRestingHrMult` × the DAYTIME-AWAKE HR baseline. Stillness alone is // also desk work, reading and a car passenger seat. @@ -103,6 +110,18 @@ const double napRestingHrMult = 0.95; /// all. Below it we abstain — daytime HR is opportunistic on this device. const double minNapHrCoverage = 0.5; +/// A bout needs a DECODED gravity vector on at least this fraction of its +/// wall-clock seconds. Mirrors `kMinAccelCoverageForVanHees` in the edge, which +/// guards the nocturnal path the same way — but the gate belongs HERE so no +/// caller can forget it. +/// +/// It is a CONTRACT, not a live fix: `stillAt` already refuses an unmeasured +/// second, and a bout can only absorb unmeasured time through a bridge, which +/// is capped at [napBridgeSec] between runs of at least the sustained window — +/// so today's arithmetic cannot get coverage below ~50% anyway. Encoded here so +/// a future change to either bound cannot silently reopen the hole. +const double minNapAccelCoverage = 0.5; + /// A bout overlapping off-wrist or excluded (charging / workout) spans by at /// least this fraction is discarded. A charging band is perfectly still. const double maxNapOffWristFraction = 0.5; @@ -172,6 +191,13 @@ Metric> detectNaps( // positional array, not a uniform grid: pruning and sync gaps leave holes, so // two samples an hour apart can be adjacent indices. Joining them would read // an unobserved hour as unbroken stillness and count it as sleep. + // + // It breaks at an UNMEASURED second too, and that falls out of the primitive: + // `deltaDeg` is NaN wherever gravity was not decoded (see [immobilityMask]), + // and `NaN < thr` is false. A second the band never measured therefore fails + // the still test outright instead of contributing the constant 0.0° of an + // exact (0,0,0) sample — which is what emitted a 50-minute decode gap as a + // nap at the 0.85 confidence cap. bool stillAt(int k) => mask.deltaDeg[k] < thr && (k == 0 || absAt(k) - absAt(k - 1) == 1); @@ -262,7 +288,8 @@ Metric> detectNaps( final awakeIdx = []; for (var k = 0; k < n; k++) { if (hr[k] <= 0 || deferredSec[k]) continue; - if (mainSleep != null && k >= mainSleep.start && k < mainSleep.end) continue; + if (mainSleep != null && k >= mainSleep.start && k < mainSleep.end) + continue; awakeIdx.add(k); } if (awakeIdx.length < minAwakeHrSamples) { @@ -280,7 +307,7 @@ Metric> detectNaps( // Every rejection path increments one of these and reports it in `skipped`. // A silent `continue` turns "your 7-hour still block is too long to be a nap" // into a bare "no qualifying nap", which tells the caller nothing about why. - var outOfRange = 0, inMainSleep = 0, noBaseline = 0; + var outOfRange = 0, inMainSleep = 0, noBaseline = 0, unobserved = 0; for (var bi = 0; bi < bouts.length; bi++) { final start = bouts[bi][0], end = bouts[bi][1]; @@ -304,17 +331,26 @@ Metric> detectNaps( continue; } - if (mainSleep != null && - start < mainSleep.end && - end > mainSleep.start) { + if (mainSleep != null && start < mainSleep.end && end > mainSleep.start) { inMainSleep++; continue; } + // Wall-clock accel coverage — the seconds inside the bout the band actually + // measured gravity for, over the bout's elapsed span (so a recording hole + // counts against it exactly as an invalid sample does). + var accelSec = 0; + for (var k = start; k < end; k++) { + if (series[k].valid) accelSec++; + } + if (accelSec / tib < minNapAccelCoverage) { + unobserved++; + continue; + } + final offFrac = _overlapFraction(aStart, aEnd, wristOff); final exFrac = _overlapFraction(aStart, aEnd, exclude); - if (offFrac >= maxNapOffWristFraction || - exFrac >= maxNapOffWristFraction) { + if (offFrac >= maxNapOffWristFraction || exFrac >= maxNapOffWristFraction) { offWrist++; continue; } @@ -421,6 +457,7 @@ Metric> detectNaps( if (noBaseline > 0) '$noBaseline without an awake HR baseline', if (outOfRange > 0) '$outOfRange outside 15 min–6 h', if (inMainSleep > 0) '$inMainSleep inside the main sleep window', + if (unobserved > 0) '$unobserved with accel coverage <50% (unmeasured)', if (unverifiable > 0) '$unverifiable unverifiable (HR coverage <50%)', if (offWrist > 0) '$offWrist off-wrist/excluded', if (awakeStill > 0) '$awakeStill still but no HR dip', diff --git a/lib/src/onehz/sleep/night_hrv_shape.dart b/lib/src/onehz/sleep/night_hrv_shape.dart new file mode 100644 index 0000000..27e6f22 --- /dev/null +++ b/lib/src/onehz/sleep/night_hrv_shape.dart @@ -0,0 +1,217 @@ +// CV-06 — THE SHAPE OF YOUR NIGHT. Per-bin RMSSD across the sleep window. +// +// Pure description. No model, no schema, no cause. The edge already bins the +// night's cleaned NN into ~30-min windows for respiration (`_respPerWindow`); +// this is that same binning with RMSSD in place of the spectral estimator. +// +// WHAT IT MAY SAY: "variability was low for the first third and climbed after +// that". WHAT IT MAY NEVER SAY: why. A suppressed first third is consistent +// with alcohol, a late meal, late training, a hot room, illness onset, or +// nothing at all — and this function cannot tell those apart, so no caller may +// attach one. +// +// DROPPED ON PURPOSE (the IDEAS entry's own cut): "time to the first bin within +// 10% of the night's max". It is anchored on the noisiest statistic in the +// series — a single high bin moves the anchor — and it jumps by hours between +// adjacent nights on identical physiology. It is not implemented here and must +// not be added. +// +// TWO HONESTY RULES ARE STRUCTURAL, not styling: +// * a bin under the beat floor ABSTAINS and stays in the series as a hole, +// so a curve breaks across a charging gap instead of drawing through it; +// * every bin ships lo/hi as well as a point, because RMSSD from a few +// hundred beats has real sampling spread and a single line drawn through +// the points claims a precision the beats do not carry. Render a BAND. + +import 'dart:math' as math; +import '../types.dart'; +import '../util.dart'; +import '../clinical/hrv_time.dart'; + +/// One ~30-min bin of the night. +class NightHrvBin { + /// Bin start, seconds from the first beat of the night. + final int startSec; + + /// Beats that landed in this bin (published even when the bin abstains — + /// "we had 40 beats here" is the reason, and it is not a bare dash). + final int nBeats; + + /// Bin RMSSD (ms), or NULL when the bin is under the beat floor. + final double? rmssdMs; + + /// Sampling band around [rmssdMs] (ms). Null exactly when [rmssdMs] is. + final double? loMs; + final double? hiMs; + + const NightHrvBin({ + required this.startSec, + required this.nBeats, + required this.rmssdMs, + required this.loMs, + required this.hiMs, + }); + + bool get present => rmssdMs != null; + + Map toJson() => { + 't': startSec, + 'n_beats': nBeats, + 'rmssd_ms': rmssdMs == null ? null : round6(rmssdMs!), + 'lo_ms': loMs == null ? null : round6(loMs!), + 'hi_ms': hiMs == null ? null : round6(hiMs!), + }; +} + +class NightHrvShape { + final List bins; + + /// Mean RMSSD over the PRESENT bins of the first / last third of the night. + /// Both null unless each third holds at least [_minBinsPerThird] present + /// bins — one bin is not a third. + final double? firstThirdMs; + final double? lastThirdMs; + + /// [lastThirdMs] / [firstThirdMs]. Null whenever either side is. + final double? lastOverFirst; + + const NightHrvShape({ + required this.bins, + required this.firstThirdMs, + required this.lastThirdMs, + required this.lastOverFirst, + }); + + Map toJson() => { + 'bins': [for (final b in bins) b.toJson()], + 'first_third_ms': firstThirdMs == null ? null : round6(firstThirdMs!), + 'last_third_ms': lastThirdMs == null ? null : round6(lastThirdMs!), + 'last_over_first': + lastOverFirst == null ? null : round6(lastOverFirst!), + }; +} + +/// A third needs this many PRESENT bins before it gets a mean. +const int _minBinsPerThird = 2; + +/// Beats a bin needs before it publishes an RMSSD. +/// +/// A CHOICE. A 30-min bin at a nocturnal HR of 50 holds ~900 beats, so 300 is +/// roughly a third of a well-covered bin — enough that the estimate is not +/// dominated by one noisy stretch, low enough that a partly-gapped bin still +/// counts. It travels in the signature so a caller can widen the bins and +/// raise it together. +const int kMinBeatsPerHrvBin = 300; + +/// CV-06 — per-bin RMSSD across the night. +/// +/// [nnMs] cleaned NN intervals (ms) and [nnTimesMs] their beat times (ms), the +/// same pair everything else in this package consumes — run `correctRr` first. +/// [binMin] is the bin width; the IDEAS entry says to widen to 45–60 min if +/// 30-min bins look wobbly on real nights, so it is a parameter, not a +/// constant. +Metric nightHrvShape( + List nnMs, + List nnTimesMs, { + double binMin = 30, + int minBeatsPerBin = kMinBeatsPerHrvBin, +}) { + const inputs = ['rr_cleaned', 'beat_times']; + if (nnMs.length != nnTimesMs.length || nnMs.length < minBeatsPerBin) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'too few beats for a nightly shape ' + '(${nnMs.length}, need ≥$minBeatsPerBin)', + ); + } + final binMs = binMin * 60000.0; + final t0 = nnTimesMs.first; + final span = nnTimesMs.last - t0; + final nBins = math.max(1, (span / binMs).ceil()); + if (nBins < 3) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: + 'night spans ${(span / 3600000).toStringAsFixed(1)} h — under three ' + '${binMin.round()}-min bins there is no shape to describe', + ); + } + + final bins = []; + var k = 0; + for (var b = 0; b < nBins; b++) { + final hi = t0 + (b + 1) * binMs; + final start = k; + while (k < nnTimesMs.length && nnTimesMs[k] < hi) { + k++; + } + final n = k - start; + if (n < minBeatsPerBin) { + // A HOLE, kept in the series. Dropping it would let a caller draw a + // straight line across a charging gap and call it flat variability. + bins.add(NightHrvBin( + startSec: (b * binMs / 1000).round(), + nBeats: n, + rmssdMs: null, + loMs: null, + hiMs: null, + )); + continue; + } + final m = hrvTime( + nnMs.sublist(start, k), + nnTimesMs: nnTimesMs.sublist(start, k), + ); + final rmssd = m.value?.rmssd; + // Sampling spread of an RMSSD from `n` successive differences. RMSSD is a + // root-mean-square, so its relative standard error is ~1/sqrt(2n); ±1.96 of + // those is the band. It is the estimator's own noise, NOT a physiological + // range and NOT a confidence interval on anything the user did. + final se = rmssd == null ? null : rmssd / math.sqrt(2 * n); + bins.add(NightHrvBin( + startSec: (b * binMs / 1000).round(), + nBeats: n, + rmssdMs: rmssd, + loMs: se == null ? null : math.max(0.0, rmssd! - 1.96 * se), + hiMs: se == null ? null : rmssd! + 1.96 * se, + )); + } + + final third = bins.length ~/ 3; + double? meanOf(Iterable xs) { + final v = [ + for (final b in xs) + if (b.present) b.rmssdMs! + ]; + return v.length < _minBinsPerThird ? null : mean(v); + } + + final first = meanOf(bins.take(third)); + final last = meanOf(bins.skip(bins.length - third)); + final ratio = + (first == null || last == null || first == 0) ? null : last / first; + + return Metric( + value: NightHrvShape( + bins: bins, + firstThirdMs: first, + lastThirdMs: last, + lastOverFirst: ratio, + ), + // Bounded by how many bins actually carried beats — a night that abstained + // on half its bins is not a shape you can read. + confidence: clamp( + 0.8 * bins.where((b) => b.present).length / bins.length, + 0.0, + 0.8, + ), + tier: Tier.high, + inputs_used: inputs, + note: 'per-bin RMSSD (${binMin.round()}-min bins, PRV not ECG-HRV) — a ' + 'DESCRIPTION of the night, never a cause. A low first third is equally ' + 'consistent with alcohol, a late meal, late training, a warm room, ' + 'illness onset, or nothing. Render each bin as a band, not a point.', + ); +} diff --git a/lib/src/onehz/sleep/segment.dart b/lib/src/onehz/sleep/segment.dart index 6c7ff3b..6905d62 100644 --- a/lib/src/onehz/sleep/segment.dart +++ b/lib/src/onehz/sleep/segment.dart @@ -7,25 +7,36 @@ // estimator — this is THE source. // // Pipeline: -// 1. vanHeesSleepWindow(accel) with the published 30-min bridge gap → the -// in-bed REST window [onsetIdx, offsetIdx) + the per-second immobility mask. -// 2. (optional) nocturnal-HR-dip consensus: when a daytime [hrBaseline] is -// supplied, confirm/refine onset & offset to where HR sustains below the -// baseline (the cardiac signature of sleep), tightening — never widening — -// the accel window. This is a CONSENSUS refinement, not a second detector. -// 3. AdvancedSleepStager.detectSleep/stageWindow, staged via +// 1. vanHeesSleepWindow(accel) → the per-second immobility mask + z-angle +// series ONLY. Its `onsetIdx`/`offsetIdx` are NOT the published window and +// never have been — this file used to claim they were. The window comes +// from step 2; van Hees survives here as the immobility/arm-angle +// primitive that `stages4` and the nap detector share, and as the +// `fallbackWindow` whose masks are forwarded. +// 2. AdvancedSleepStager.detectSleep/stageWindow — THE window +// ([_pickMainSleepGroup] over the detected sessions, or the caller's +// forced window) — staged via // StagingMethod.cardio (the default — delegates to cardioStager, the // transparent motion+HR+RMSSD rule stager; see advanced_stager.dart's // file header for the 2026-07 comparison behind this default) → 4-class // wake/light/deep/rem per epoch, over the WINDOW SLICE ONLY. -// 4. Expand the per-epoch stages to PER-SECOND labels over the window, and +// 3. Expand the per-epoch stages to PER-SECOND labels over the window, mark +// every second we did not actually observe as 'unobserved' (step 4), and // derive TST/WASO/efficiency/stage-seconds ALL from those labels // (asleep = stage != wake), so they are mutually consistent and consistent // with any hypnogram built from `stages`. +// 4. UNOBSERVED time. The window is wall-clock; the substrate is a POSITIONAL +// array with holes, and HR is intermittent. A second with no sample, or +// with no heart rate anywhere near it, is labelled 'unobserved' and counts +// into NOTHING — not TST, not WASO, not wake, not the efficiency +// denominator. It is published as [SleepSegmentation.unobservedSec] so the +// UI can state the reason instead of implying a measurement. // -// HONESTY: if no qualifying sleep (no window, or in-bed < ~3 h, or staging -// cannot run) we return SleepSegmentation.absent — everything null, confidence -// 0. Never fabricated. Stages are tier ESTIMATE (3-class autonomic, not PSG). +// HONESTY: if no qualifying sleep (no window, in-bed < ~3 h, staging cannot +// run, or too little of the window was observed) we return +// SleepSegmentation.absent — everything null, confidence 0, with +// [SleepSegmentation.absenceReason] set where the cause is known. Never +// fabricated. Stages are tier ESTIMATE (3-class autonomic, not PSG). import 'dart:math' as math; import '../types.dart'; @@ -37,6 +48,136 @@ import 'advanced_stager.dart'; /// Minimum in-bed duration to qualify as the main sleep (ARCHITECTURE_V2: ~3 h). const int _minQualifyingSleepSec = 3 * 3600; +/// Fraction of the in-bed window that must be OBSERVED (a sample present, and a +/// heart rate within half an epoch of it) before any accounting figure is +/// published. Below it the night is [SleepSegmentation.absent] with a reason, +/// not a plausible number over a window nobody watched. Same 0.5 floor the edge +/// applies to the nocturnal accel path (`kMinAccelCoverageForVanHees`). +const double kMinObservedFractionForSleep = 0.5; + +/// A second is credited with cardiac evidence when a valid HR sample lands +/// within ±[_hrEvidenceHalfWinSec] of it — half a staging epoch either side. +/// +/// The stager decides a 30-s epoch on ANY valid HR inside it, so this is the +/// closest per-second equivalent available here (segmentation does not see the +/// stager's epoch grid, which starts at each usable run, not at a fixed +/// offset). Worst-case misalignment is under one epoch in either direction. +const int _hrEvidenceHalfWinSec = 15; + +/// Ceiling on [SleepSegmentation.confidence]. +/// +/// WHAT THE CONFIDENCE IS: a COVERAGE score — how much of the window we +/// actually watched (HR seconds, RR seconds, window length), capped here. It is +/// NOT an accuracy, a probability that a stage label is right, or a κ. Nothing +/// in this file measures accuracy; only PSG can, and we do not have it. +/// +/// WHAT 0.6 IS: a CHOICE of cap, and the number is load-bearing in exactly one +/// place — [stageIntervals] scales its half-widths by `confidence / this`, so a +/// well-observed night gets a narrower interval than one scraping the observed +/// floor. Lowering the cap below the typical score would pin every night at 1.0 +/// and DESTROY that contrast, which is why it is not simply set to our own κ. +/// +/// WHAT THE LITERATURE SAYS, stated the right way round. Four-class staging +/// from RR and wrist motion sits at κ 0.21–0.53 across six consumer devices vs +/// PSG (SLEEP Advances 6(2):zpaf021, 2025: Apple Watch S8 0.53, Fitbit Sense +/// 0.42, Charge 5 0.41, WHOOP 4.0 0.37, Withings ScanWatch 0.22, Garmin +/// Vivosmart 4 0.21). That range is the one WE ARE BELOW, not one we sit in: +/// our own held-out four-class κ is 0.132 (`cardio_stager.dart`), under every +/// device in it, and the vendor firmware for the band we decode scores 0.37 — +/// 2.8× ours. The docstring used to quote the range as if it justified the +/// ceiling; it justifies nothing except that no amount of coverage makes a +/// night certain. +const double kMaxSleepConfidence = 0.6; + +/// SLP-13 — the narrowest half-width a stage interval may ever have (s). +/// +/// A CHOICE, and the same five minutes [kSustainedAwakeningSec] already uses: +/// without it a night with `deep_sec == 0` would publish the interval 0–0, +/// which is the exact false precision the intervals exist to remove. A stage +/// the overlay did not fire on is "under five minutes", not "none". +const int kStageIntervalFloorSec = 300; + +/// SLP-13 — relative half-widths at the two ends of the confidence range. +/// +/// These are OUR numbers for OUR segmenter, scaled by each night's own +/// [SleepSegmentation.confidence]. They are deliberately NOT a published κ +/// converted into minutes: one literature figure applied uniformly to every +/// night is itself a fabricated precision, it hides the difference between a +/// fully-covered night and one scraping the observed floor, and it is the kind +/// of number that gets quoted back at us as if we had measured it. +/// +/// Deep is wider than REM at both ends because it is not the same class of +/// estimate: REM comes out of the stager's own four-class decision, while the +/// Light/Deep boundary is the UNVALIDATED HR-depth overlay (see [stages4]). +const double _remRelHalfBest = 0.30; +const double _remRelHalfWorst = 0.55; +const double _deepRelHalfBest = 0.45; +const double _deepRelHalfWorst = 0.75; + +/// SLP-13 — a stage duration as the INTERVAL it actually is. +/// +/// [pointSec] is the exact figure the stager counted. It is kept so Investigate +/// (density 3) can show what was counted, and so nothing downstream has to +/// re-derive it — but a normal screen renders [loSec]–[hiSec] and never +/// [pointSec] on its own. +class StageInterval { + final int pointSec; + final int loSec; + final int hiSec; + const StageInterval(this.pointSec, this.loSec, this.hiSec); + Map toJson() => + {'point_sec': pointSec, 'lo_sec': loSec, 'hi_sec': hiSec}; +} + +/// SLP-13 — the intervals a screen may publish for tonight's stage minutes. +/// +/// [confidence] is that night's own [SleepSegmentation.confidence] (0 .. +/// [kMaxSleepConfidence]); a better-observed night gets a narrower interval and +/// a night at the observed floor gets a wide one. Intervals clamp to [0, +/// tstSec] — no stage can exceed the sleep it is a part of. +({StageInterval light, StageInterval deep, StageInterval rem}) stageIntervals({ + required int lightSec, + required int deepSec, + required int remSec, + required int tstSec, + required double confidence, +}) { + final t = clamp(confidence / kMaxSleepConfidence, 0.0, 1.0); + double rel(double best, double worst) => worst + (best - worst) * t; + final deepHalf = math.max( + (rel(_deepRelHalfBest, _deepRelHalfWorst) * deepSec).round(), + kStageIntervalFloorSec, + ); + final remHalf = math.max( + (rel(_remRelHalfBest, _remRelHalfWorst) * remSec).round(), + kStageIntervalFloorSec, + ); + // Light and Deep are the two sides of ONE binary decision inside NREM: a + // second the overlay wrongly called deep is a second of light, and vice + // versa. So light inherits deep's ABSOLUTE half-width — applying deep's + // RELATIVE width to the (usually much larger) light figure would claim the + // overlay's error grows with the stage it did not misclassify. + return ( + light: _interval(lightSec, deepHalf, tstSec), + deep: _interval(deepSec, deepHalf, tstSec), + rem: _interval(remSec, remHalf, tstSec), + ); +} + +StageInterval _interval(int sec, int halfSec, int tstSec) => StageInterval( + sec, + math.max(0, sec - halfSec), + math.min(tstSec, sec + halfSec), + ); + +/// How long a wake run must last to be counted as an awakening (SLP-03). +/// +/// Five minutes is a CHOICE — it is the bar Webster's rescoring already uses +/// elsewhere in this package for a genuine sustained awakening, and it is not +/// a physiological boundary. Anything shorter is inside the noise of a 1 Hz +/// wrist. Published in the JSON so the number on screen can state its own bar. +const int kSustainedAwakeningSec = 300; + /// Single-source sleep segmentation result. Every accounting figure is derived /// from the per-second [stages] labels (asleep = stage != wake), so the parts /// are mutually consistent. When no sleep qualifies, see [SleepSegmentation.absent]. @@ -47,14 +188,24 @@ class SleepSegmentation { /// Per-second 3-class labels over the window [onsetIdx, offsetIdx). /// Empty when absent. `stages.length == inBedSec`. (Back-compat: wake/nrem/rem; /// NREM here is light+deep combined — the Light/Deep split lives in [stages4].) + /// + /// LEGACY VIEW, LOSSY: [SleepStage] has no 'unobserved' member, so a second we + /// never observed reads here as `wake`. It is NOT counted as wake in any figure + /// on this class — see [stages4] and [unobservedSec] — but a reader that + /// tallies this list itself will over-count wake. Prefer [stages4]. final List stages; - /// Per-second 4-class hypnogram-ready labels over the SAME window, aligned 1:1 - /// with [stages]: 'wake' | 'light' | 'deep' | 'rem'. 'light' and 'deep' are the + /// Per-second labels over the SAME window, aligned 1:1 with [stages]: + /// 'wake' | 'light' | 'deep' | 'rem' | 'unobserved'. 'light' and 'deep' are the /// two halves of NREM — 'deep' marks the NREM seconds the LOW-CONFIDENCE /// HR-depth overlay flags as deep (see walch_stager STEP 2); 'light' is the /// remaining NREM. The Light/Deep split is an UNVALIDATED estimate; surface it - /// badged low-confidence. Empty when absent. + /// badged low-confidence. + /// + /// 'unobserved' is not a stage and not a measurement: it marks a second the + /// record does not contain, or one with no heart rate within half an epoch. A + /// hypnogram must break at those seconds rather than draw across them. + /// Empty when absent. final List stages4; /// Total sleep time (s): seconds where stage != wake. Null when absent. @@ -65,9 +216,19 @@ class SleepSegmentation { final int? wasoSec; /// In-bed time (s) = window length (offsetIdx − onsetIdx). Null when absent. + /// This is WALL-CLOCK span and includes [unobservedSec]. final int? inBedSec; - /// Sleep efficiency (%) = 100 · TST / in-bed. Null when absent. + /// Seconds inside the window we did not observe: no sample in the record, or + /// no heart rate within half a staging epoch. They count into NOTHING else on + /// this class. Observed in-bed time is `inBedSec - unobservedSec`. Null when + /// absent. + final int? unobservedSec; + + /// Sleep efficiency (%) = 100 · TST / OBSERVED in-bed seconds. Null when + /// absent. The denominator excludes [unobservedSec] on purpose: dividing by + /// wall-clock time published 62.7 % for a night where three of the eight hours + /// were never recorded, which reads as measured wakefulness. final double? efficiencyPct; /// NREM seconds (stage == nrem) = [lightSec] + [deepSec]. Null when absent. @@ -87,9 +248,37 @@ class SleepSegmentation { /// Wake seconds within the window (stage == wake). Null when absent. final int? wakeSec; - /// 0..1 confidence (van Hees window × staging × HR-consensus). 0 when absent. + /// Sustained awakenings AT LEAST this long: runs of wake between sleep onset + /// and final offset lasting ≥ [kSustainedAwakeningSec]. Null when absent. + /// + /// AT LEAST is not hedging. A 1 Hz wrist cannot see the 3–15 s cortical + /// arousals PSG counts at all, and our wake specificity is 29–52 %, so the + /// true number of awakenings is HIGHER than this — never lower. This is not + /// an arousal index and must never be rendered as one. + /// + /// A run TERMINATES at an unobserved second; it never merges across one. Two + /// two-minute wakes with a charging gap between them are two short wakes we + /// did not count, not one sustained awakening we invented. + final int? sustainedAwakenings; + + /// Longest unbroken stretch of sleep (s) inside the window. Null when absent. + /// + /// Same rule, and this is where it matters most: the run ends at an + /// unobserved second as surely as it ends at a wake one. A naive longest-run + /// that draws through a three-hour hole prints a 5 h stretch nobody watched. + final int? longestSleepRunSec; + + /// 0..1 confidence — the mean of a van Hees window term and a staging term. + /// There is no HR-consensus term; no such step exists (see [segmentSleep]). + /// 0 when absent. final double confidence; + /// Machine-readable cause when [present] is false and the cause is known + /// (currently: too little of the window was observed). Null when a caller + /// should say nothing more than "no qualifying sleep". Never render a bare + /// dash for an absence that carries one of these. + final String? absenceReason; + const SleepSegmentation({ required this.window, required this.stages, @@ -97,13 +286,17 @@ class SleepSegmentation { required this.tstSec, required this.wasoSec, required this.inBedSec, + required this.unobservedSec, required this.efficiencyPct, required this.nremSec, required this.lightSec, required this.deepSec, required this.remSec, required this.wakeSec, + required this.sustainedAwakenings, + required this.longestSleepRunSec, required this.confidence, + this.absenceReason, }); /// Honest "no qualifying sleep" result — all figures null, confidence 0. @@ -114,43 +307,105 @@ class SleepSegmentation { tstSec: null, wasoSec: null, inBedSec: null, + unobservedSec: null, efficiencyPct: null, nremSec: null, lightSec: null, deepSec: null, remSec: null, wakeSec: null, + sustainedAwakenings: null, + longestSleepRunSec: null, confidence: 0, ); + /// Absent BECAUSE the window was too thinly observed to stand behind — the + /// caller has a reason to show, not a hole. + static SleepSegmentation unobservedWindow(String reason) => SleepSegmentation( + window: null, + stages: const [], + stages4: const [], + tstSec: null, + wasoSec: null, + inBedSec: null, + unobservedSec: null, + efficiencyPct: null, + nremSec: null, + lightSec: null, + deepSec: null, + remSec: null, + wakeSec: null, + sustainedAwakenings: null, + longestSleepRunSec: null, + confidence: 0, + absenceReason: reason, + ); + bool get present => window != null; + /// SLP-13 — tonight's stage minutes as INTERVALS, from this night's own + /// [confidence]. Null when the night is absent. The exact seconds stay on + /// [lightSec]/[deepSec]/[remSec] for Investigate; everything user-facing + /// should be reading this instead. + ({StageInterval light, StageInterval deep, StageInterval rem})? + get stageRanges { + final l = lightSec, d = deepSec, r = remSec, t = tstSec; + if (l == null || d == null || r == null || t == null) return null; + return stageIntervals( + lightSec: l, + deepSec: d, + remSec: r, + tstSec: t, + confidence: confidence, + ); + } + Map toJson() => { 'window': window?.toJson(), 'tst_sec': tstSec, 'waso_sec': wasoSec, 'in_bed_sec': inBedSec, - 'efficiency_pct': - efficiencyPct == null ? null : round6(efficiencyPct!), + 'unobserved_sec': unobservedSec, + 'efficiency_pct': efficiencyPct == null ? null : round6(efficiencyPct!), 'nrem_sec': nremSec, 'light_sec': lightSec, 'deep_sec': deepSec, 'rem_sec': remSec, 'wake_sec': wakeSec, + // SLP-03. The 5-minute bar is a CHOICE, not physiology — it ships with + // the count so a screen can say what it counted. + 'sustained_awakenings': sustainedAwakenings, + 'awakening_min_sec': kSustainedAwakeningSec, + 'longest_sleep_run_sec': longestSleepRunSec, 'epochs': stages.length, 'confidence': round6(confidence), + if (absenceReason != null) 'absence_reason': absenceReason, // Deep is a LOW-CONFIDENCE, unvalidated HR-depth overlay (see // walch_stager STEP 2). Carry the flag so the UI badges it honestly. 'deep_low_confidence': true, + // SLP-13 — the interval each stage figure should actually be shown as, + // derived from THIS night's confidence. `*_sec` above stays the exact + // count for Investigate; these are what a normal screen renders. + 'light_range_sec': _rangeJson(stageRanges?.light), + 'deep_range_sec': _rangeJson(stageRanges?.deep), + 'rem_range_sec': _rangeJson(stageRanges?.rem), }; + + static List? _rangeJson(StageInterval? r) => + r == null ? null : [r.loSec, r.hiSec]; } /// THE single-source sleep segmentation. /// /// [accel] 1 Hz gravity vectors. [hr1hz] 1 Hz HR (bpm; 0 = off-skin), same time -/// base / length as [accel]. [hrBaseline] optional daytime HR samples (bpm) used -/// for a nocturnal-HR-dip consensus that refines onset/offset; when omitted the -/// accel window stands alone. +/// base / length as [accel]. +/// +/// [hrBaseline] is CURRENTLY UNUSED. The file header describes a pipeline step +/// that confirms/refines onset and offset against a nocturnal-HR dip; no such +/// code exists, and the parameter is read nowhere in this file. It is kept in +/// the signature (all three edge call sites pass a real baseline) so wiring the +/// step up later is a one-file change — but until then, onset and offset are +/// NOT HR-refined, whatever the header says. SleepSegmentation segmentSleep( List accel, List hr1hz, { @@ -229,7 +484,6 @@ SleepSegmentation segmentSleep( start: onsetSec, end: offsetSec, asleepMin: AdvancedSleepStager.hypnogramMetrics(session).tstS / 60.0, - inBedSec: offsetSec - onsetSec, ); } else { final sessions = AdvancedSleepStager.detectSleep( @@ -260,6 +514,34 @@ SleepSegmentation segmentSleep( return SleepSegmentation.absent; } + // OBSERVED mask over the wall-clock window. `inBed` is wall-clock length but + // `trimmedAccel` is a POSITIONAL array with holes (pruning, sync gaps), and HR + // is intermittent — so a second is observed only when BOTH hold: + // * the record contains a sample for it, and + // * a valid HR landed within half a staging epoch of it. + // Neither used to be checked. Every unwritten second defaulted to 'wake' and + // was then counted as measured WASO, as wake_sec and into the efficiency + // denominator: an 8 h confirmed window over a 5 h record published waso + // 10740 s and 62.7 % efficiency for three hours nobody watched. And every + // no-HR epoch could only ever fall through to NREM (see cardio_stager's + // asymmetric gates), so it was credited to TST as Light on zero cardiac + // evidence, one-way, inflating TST and efficiency and never deflating them. + final sampled = List.filled(inBed, false); + final hrNear = List.filled(inBed, false); + for (var k = onset; k < tsSec.length && tsSec[k] < chosen.end; k++) { + final off = tsSec[k] - chosen.start; + if (off < 0 || off >= inBed) continue; + sampled[off] = true; + if (trimmedHr[k] <= 0) continue; + final lo = math.max(0, off - _hrEvidenceHalfWinSec); + final hi = math.min(inBed - 1, off + _hrEvidenceHalfWinSec); + for (var m = lo; m <= hi; m++) { + hrNear[m] = true; + } + } + final observed = List.generate(inBed, (i) => sampled[i] && hrNear[i], + growable: false); + final stages4 = List.filled(inBed, 'wake'); for (final session in chosen.sessions) { for (final seg in session.stages) { @@ -270,6 +552,23 @@ SleepSegmentation segmentSleep( } } } + // Stamped LAST: an unobserved second has no stage, whatever a staging segment + // spanning the hole happened to claim. + var unobservedSec = 0; + for (var i = 0; i < inBed; i++) { + if (observed[i]) continue; + stages4[i] = 'unobserved'; + unobservedSec++; + } + + final observedSec = inBed - unobservedSec; + if (observedSec < inBed * kMinObservedFractionForSleep) { + return SleepSegmentation.unobservedWindow( + 'only ${observedSec}s of a ${inBed}s window were observed ' + '(sample + heart rate); below the ' + '${(kMinObservedFractionForSleep * 100).round()}% floor', + ); + } final perSec = List.generate( inBed, @@ -279,6 +578,7 @@ SleepSegmentation segmentSleep( var tst = 0, waso = 0, nrem = 0, light = 0, deep = 0, rem = 0, wake = 0; var firstSleep = -1, lastSleep = -1; for (var i = 0; i < perSec.length; i++) { + if (!observed[i]) continue; switch (perSec[i]) { case SleepStage.wake: wake++; @@ -304,11 +604,76 @@ SleepSegmentation segmentSleep( } if (firstSleep >= 0) { for (var i = firstSleep; i <= lastSleep; i++) { - if (perSec[i] == SleepStage.wake) waso++; + if (observed[i] && perSec[i] == SleepStage.wake) waso++; + } + } + + // SLP-03 — the SHAPE of the night, off the same per-second labels everything + // else here comes from, so there is one definition of where a run ends. + // + // Three states, not two: asleep / wake / UNOBSERVED. Both runs terminate at + // an unobserved second. Merging across one is how a naive longest-run bridges + // a three-hour charging hole and prints a 5 h unbroken stretch, and how two + // short wakes either side of a gap become one "sustained awakening". + var sustainedAwakenings = 0; + var longestSleepRun = 0; + var sleepRun = 0; + var wakeRun = 0; + for (var i = 0; i < perSec.length; i++) { + final asleep = observed[i] && perSec[i] != SleepStage.wake; + final awake = observed[i] && perSec[i] == SleepStage.wake; + sleepRun = asleep ? sleepRun + 1 : 0; + if (sleepRun > longestSleepRun) longestSleepRun = sleepRun; + // Awakenings are WASO only — the settling before onset and the lie-in after + // final offset are not awakenings. + final inWaso = firstSleep >= 0 && i > firstSleep && i < lastSleep; + if (awake && inWaso) { + wakeRun++; + if (wakeRun == kSustainedAwakeningSec) sustainedAwakenings++; + } else { + wakeRun = 0; } } - final efficiency = inBed > 0 ? 100.0 * tst / inBed : 0.0; - final conf = clamp(((wm.confidence > 0 ? wm.confidence : 0.45) + 0.5) / 2.0, 0.0, 0.6); + if (tst == 0) { + // Nothing in the window staged as sleep. A forced window deliberately skips + // the 3 h and index gates, so this used to publish tst 0 / efficiency 0 % as + // a PRESENT metric — "0 h 0 m slept, 0 % efficiency" for a night we simply + // did not observe. That is a fabricated measurement, not an abstention. + return SleepSegmentation.absent; + } + // Denominator is OBSERVED in-bed time, not wall clock — see [efficiencyPct]. + final efficiency = observedSec > 0 ? 100.0 * tst / observedSec : 0.0; + + // Confidence = window quality x STAGING quality. The staging term used to be + // the literal 0.5, so every forced-window night (where `wm` is absent) came + // out at exactly 0.475 no matter how much data was behind it. cardioStager + // computes a real staging confidence (0.35 + 0.25*rrCov) and throws it away + // on the way out, and threading it back through SleepSession is a wider + // change than this — so recompute the same quantity from the HR and beats + // that actually landed INSIDE the chosen window. + var hrCovered = 0; + final rrSeconds = {}; + for (var i = 0; i < n; i++) { + final ts = trimmedAccel[i].tsMs ~/ 1000; + if (ts < chosen.start || ts >= chosen.end) continue; + if (trimmedHr[i] > 0) hrCovered++; + } + for (final b in rr) { + if (b.ts >= chosen.start && b.ts < chosen.end) rrSeconds.add(b.ts); + } + final hrCov = clamp(hrCovered / inBed, 0.0, 1.0); + final rrCov = clamp(rrSeconds.length / inBed, 0.0, 1.0); + final stagingConf = (0.35 + 0.25 * rrCov) * hrCov; + // Confidence in the window WE PUBLISHED, on van Hees' own construction + // (length up to a typical 7 h night, clamped) — but measured on `chosen`, the + // window that actually ships. It used to be `wm.confidence`: literally how + // long the DISCARDED van Hees block was, from a detector whose window this + // file never reads. Half of every night's published confidence came from the + // detector that lost, and that confidence sets the SLP-13 stage-interval + // widths. + final windowConf = clamp(inBed / (7 * 3600), 0.3, 0.95); + final conf = + clamp((windowConf + stagingConf) / 2.0, 0.0, kMaxSleepConfidence); return SleepSegmentation( window: SleepWindow( @@ -316,15 +681,17 @@ SleepSegmentation segmentSleep( offsetIdx: offset, onsetMs: chosen.start * 1000.0, offsetMs: chosen.end * 1000.0, - immobile: - fallbackWindow?.immobile ?? List.filled(trimmedAccel.length, false), + // EMPTY when van Hees never ran (every forced-window path), as + // `immobileUnknown` already is. A full-length all-false immobility mask + // and an all-0.0 deg arm-angle series are fabricated per-second + // measurements, not "we did not look". + immobile: fallbackWindow?.immobile ?? const [], // Forward the undecidable-second mask too. Dropping it here silently // downgraded "we could not tell" into "not immobile", which is the - // conservative direction but costs the caller `unresolvedTailSec` — the + // conservative direction but costs the caller `undecidableSec` — the // one signal that says a night ran past the end of the record. immobileUnknown: fallbackWindow?.immobileUnknown ?? const [], - zAngleDeg: - fallbackWindow?.zAngleDeg ?? List.filled(trimmedAccel.length, 0.0), + zAngleDeg: fallbackWindow?.zAngleDeg ?? const [], sptSec: inBed, ), stages: perSec, @@ -332,12 +699,15 @@ SleepSegmentation segmentSleep( tstSec: tst, wasoSec: waso, inBedSec: inBed, + unobservedSec: unobservedSec, efficiencyPct: efficiency, nremSec: nrem, lightSec: light, deepSec: deep, remSec: rem, wakeSec: wake, + sustainedAwakenings: sustainedAwakenings, + longestSleepRunSec: longestSleepRun, confidence: conf, ); } @@ -348,12 +718,6 @@ class _SleepGroup { final int end; final double asleepMin; - /// SUM of the bridged sessions' durations — the bridge GAPS are excluded, so - /// this is NOT `end - start` for a multi-session group. Do not use it for - /// time-of-day math: `start + inBedSec ~/ 2` lands half the total gap EARLY - /// (a single 50-min bridge ⇒ 25 min early). Use [midsleepSec] for that. - final int inBedSec; - /// The group's circadian centre: the midpoint of its ACTUAL SPAN. This is /// what a midsleep anchor is defined against (the middle of the sleep period, /// gaps included — Roenneberg's MSF/mid-sleep convention), and it is what @@ -365,7 +729,6 @@ class _SleepGroup { required this.start, required this.end, required this.asleepMin, - required this.inBedSec, }); } @@ -383,7 +746,6 @@ List<_SleepGroup> _bridgeAdjacentSessions(List sessions) { start: session.start, end: session.end, asleepMin: asleepMin, - inBedSec: session.end - session.start, ), ); continue; @@ -397,7 +759,6 @@ List<_SleepGroup> _bridgeAdjacentSessions(List sessions) { start: last.start, end: math.max(last.end, session.end), asleepMin: last.asleepMin + asleepMin, - inBedSec: last.inBedSec + (session.end - session.start), ), ); } else { @@ -408,7 +769,6 @@ List<_SleepGroup> _bridgeAdjacentSessions(List sessions) { start: session.start, end: session.end, asleepMin: asleepMin, - inBedSec: session.end - session.start, ), ); } @@ -448,8 +808,7 @@ _SleepGroup? _pickMainSleepGroup( } double alignmentBonusFor(_SleepGroup g) { - // Midsleep = the middle of the SPAN, never `start + inBedSec/2` — see - // [_SleepGroup.inBedSec]/[_SleepGroup.midsleepSec]. + // Midsleep = the middle of the SPAN — see [_SleepGroup.midsleepSec]. final dist = circularDistanceSec(localSecOfDay(g.midsleepSec), targetMidsleepSec); if (dist <= fullWindowSec) return alignmentBonusMin; @@ -462,9 +821,7 @@ _SleepGroup? _pickMainSleepGroup( var bestScore = winner.asleepMin + alignmentBonusFor(winner); for (final g in groups.skip(1)) { final score = g.asleepMin + alignmentBonusFor(g); - if (score > bestScore || - (score == bestScore && - g.start < winner.start)) { + if (score > bestScore || (score == bestScore && g.start < winner.start)) { winner = g; bestScore = score; } @@ -552,6 +909,9 @@ int? _circularMeanSec(List secs) { return ((sec % secondsPerDay) + secondsPerDay) % secondsPerDay; } +/// 4-class label → the legacy 3-class enum. 'unobserved' has no member and maps +/// to `wake` — see [SleepSegmentation.stages] for why that view is lossy and why +/// no figure on this class is derived from it. SleepStage _sleepStageFor(String label) { switch (label) { case 'rem': diff --git a/lib/src/onehz/sleep/sleep.dart b/lib/src/onehz/sleep/sleep.dart index d0b5f3b..e18982d 100644 --- a/lib/src/onehz/sleep/sleep.dart +++ b/lib/src/onehz/sleep/sleep.dart @@ -35,3 +35,4 @@ export 'cardio_stager.dart'; export 'cpc.dart'; export 'circadian_np.dart'; export 'cycles.dart'; +export 'night_hrv_shape.dart'; diff --git a/lib/src/onehz/sleep/sri.dart b/lib/src/onehz/sleep/sri.dart index e8cd6af..552b337 100644 --- a/lib/src/onehz/sleep/sri.dart +++ b/lib/src/onehz/sleep/sri.dart @@ -19,15 +19,50 @@ import '../types.dart'; import '../util.dart'; +/// One adjacent-day comparison, broken out of the total. +/// +/// [dayIndex] is the LATER day of the pair, indexing the same day sequence the +/// caller laid out (so the pair is `dayIndex-1` vs `dayIndex`) — this class +/// never sees a date, and the caller that built the vector is the only thing +/// that can name the two nights. +/// +/// [cases]/[agreement] count ONLY the epochs the validity mask accepted, so +/// [sri] is the concordance over what was actually observed on both days, never +/// over a hole. A pair the mask left too thin is not emitted at all — see +/// [phillipsSri]'s `minPairCases`. +class SriPair { + final int dayIndex; + final int agreement; + final int cases; + const SriPair(this.dayIndex, this.agreement, this.cases); + + /// This pair's SRI on the same 200·p−100 scale as the whole-record number. + double get sri => 200.0 * agreement / cases - 100.0; + + Map toJson() => { + 'day_index': dayIndex, + 'sri': round6(sri), + 'agreement': agreement, + 'cases': cases, + }; +} + class SriResult { final double sri; // −100..100 final int days; // number of adjacent-day comparisons + 1 final int cases; // number of epoch comparisons made - const SriResult(this.sri, this.days, this.cases); + /// Per-adjacent-day-pair decomposition of the SAME arithmetic, in day order. + /// Empty when no pair cleared the coverage floor. Purely a breakdown of a + /// number already published — it licenses "these two nights differed most", + /// never a judgement about which night should have been which. + final List pairs; + const SriResult(this.sri, this.days, this.cases, + [this.pairs = const []]); Map toJson() => { 'sri': round6(sri), 'days': days, 'cases': cases, + 'pairs': [for (final p in pairs) p.toJson()], }; } @@ -37,10 +72,19 @@ class SriResult { /// of exactly [epochsPerDay] epochs each, aligned so index `d*epochsPerDay + e` /// is epoch `e` of day `d`. [valid] optional same-length mask (false epochs are /// excluded from the agreement count, so gaps don't fabricate concordance). +/// +/// The result also carries a per-pair decomposition ([SriResult.pairs]) of the +/// same sum. A pair is only emitted when the mask accepted at least +/// [minPairCases] epochs on BOTH days (default: half the clock day) — a pair +/// compared on 40 observed minutes is a coverage hole, not an irregular night, +/// and without the floor a half-unobserved weekend tops the "worst pair" list +/// for having no data. The floor does NOT change the published [SriResult.sri]: +/// every accepted epoch still counts toward the total, thin pair or not. Metric phillipsSri( List sleepWake, int epochsPerDay, { List? valid, + int? minPairCases, }) { const inputs = ['sleep_wake_epochs']; final n = sleepWake.length; @@ -60,16 +104,27 @@ Metric phillipsSri( ); } + final pairFloor = minPairCases ?? (epochsPerDay ~/ 2); var agreement = 0; var cases = 0; + final pairs = []; for (var d = 1; d < days; d++) { + var pairAgreement = 0; + var pairCases = 0; for (var e = 0; e < epochsPerDay; e++) { final iPrev = (d - 1) * epochsPerDay + e; final iCur = d * epochsPerDay + e; if (iCur >= n) break; if (valid != null && (!valid[iPrev] || !valid[iCur])) continue; - cases++; - if (sleepWake[iPrev] == sleepWake[iCur]) agreement++; + pairCases++; + if (sleepWake[iPrev] == sleepWake[iCur]) pairAgreement++; + } + agreement += pairAgreement; + cases += pairCases; + // `> 0` is not redundant with the floor: a caller may pass + // `minPairCases: 0`, and a zero-case pair would make SriPair.sri NaN. + if (pairCases > 0 && pairCases >= pairFloor) { + pairs.add(SriPair(d, pairAgreement, pairCases)); } } if (cases == 0) { @@ -85,7 +140,7 @@ Metric phillipsSri( // stable). Saturates around a typical 7-day record. final conf = clamp((days - 1) / 7.0, 0.3, 0.95); return Metric( - value: SriResult(sri, days, cases), + value: SriResult(sri, days, cases, pairs), confidence: conf, tier: Tier.high, inputs_used: inputs, diff --git a/lib/src/onehz/sleep/stager.dart b/lib/src/onehz/sleep/stager.dart index e75d49b..0117a9f 100644 --- a/lib/src/onehz/sleep/stager.dart +++ b/lib/src/onehz/sleep/stager.dart @@ -168,7 +168,8 @@ Metric autonomicStager( // classic Webster rules (durations in minutes, converted to epochs): // after ≥ 4 min sleep, ≤ 1 min wake → sleep // after ≥10 min sleep, ≤ 3 min wake → sleep - // after ≥15 min sleep, ≤ 4 min wake → sleep + // after ≥15 min sleep, ≤ 5 min wake → sleep (Webster says 4; the extra + // minute is deliberate here — see the rules table for why) // (Symmetric in both directions: "surrounded by" sleep on either side.) _websterRescore(sm, epochSec); @@ -229,7 +230,11 @@ void websterRescoreAutonomic(List sm, int epochSec) => /// genuine, sustained awakening). Rules (Webster, in epochs): /// sleep-before ≥ 4 min AND wake-run ≤ 1 min → sleep /// sleep-before ≥10 min AND wake-run ≤ 3 min → sleep -/// sleep-before ≥15 min AND wake-run ≤ 4 min → sleep +/// sleep-before ≥15 min AND wake-run ≤ 5 min → sleep +/// The top row is 5, not Webster's 4, and the widening is deliberate: it +/// absorbs the van Hees 5-min mask-smear artifact. See the rules table below. +/// `cardio_stager`'s copy runs the published 4 because van Hees does not run +/// on its forced-window paths at all. /// "sleep-before" is satisfied by sustained sleep on EITHER bracketing side, /// so an arousal sandwiched between two long sleep bouts is bridged. /// diff --git a/lib/src/onehz/sleep/van_hees.dart b/lib/src/onehz/sleep/van_hees.dart index 3455e8a..7c2c1b6 100644 --- a/lib/src/onehz/sleep/van_hees.dart +++ b/lib/src/onehz/sleep/van_hees.dart @@ -45,11 +45,12 @@ class SleepWindow { final List immobile; /// Per-second "undecidable" mask, full length and index-aligned with - /// [immobile]. `true` marks a second whose forward sustained-inactivity - /// window is truncated by the end of the record AND which shows no movement - /// in the data we do have — i.e. it could still resolve either way once the - /// next samples arrive. Such seconds are `false` in [immobile] (not claimed - /// as rest) and are excluded from the detected sleep period. + /// [immobile]. `true` marks a second that shows no movement in the data we do + /// have but whose forward sustained-inactivity window cannot be certified — + /// either it is truncated by the end of the record (it could still resolve + /// either way once the next samples arrive) or it spans seconds the device + /// never measured. Such seconds are `false` in [immobile] (not claimed as + /// rest) and are excluded from the detected sleep period. /// /// Consumers that only read [immobile] therefore degrade conservatively (an /// unresolved tail second reads as "not asserted immobile"), never @@ -73,9 +74,9 @@ class SleepWindow { this.immobileUnknown = const [], }); - /// Seconds at the end of the record whose immobility is genuinely - /// undecidable (see [immobileUnknown]). - int get unresolvedTailSec { + /// Seconds whose immobility is genuinely undecidable — an unresolved record + /// tail, or a stretch the device never measured (see [immobileUnknown]). + int get undecidableSec { var c = 0; for (final u in immobileUnknown) { if (u) c++; @@ -89,7 +90,7 @@ class SleepWindow { if (onsetMs != null) 'onset_ms': onsetMs, if (offsetMs != null) 'offset_ms': offsetMs, 'spt_sec': sptSec, - if (unresolvedTailSec > 0) 'unresolved_tail_sec': unresolvedTailSec, + if (undecidableSec > 0) 'undecidable_sec': undecidableSec, }; } @@ -112,14 +113,21 @@ class ImmobilityMask { /// ASSERTED immobile (see [SleepWindow.immobile]). final List immobile; - /// Undecidable — forward window truncated by the record end (see + /// Undecidable — either the forward window is truncated by the record end, or + /// it contains seconds the device never measured (see /// [SleepWindow.immobileUnknown]). final List immobileUnknown; - /// Smoothed per-second z-angle (deg). + /// Smoothed per-second z-angle (deg). NaN where [AccelSample.valid] is false: + /// absent gravity is stored as exact (0,0,0), whose z-angle is a perfectly + /// well-formed 0.0° that is not a measurement of anything. final List zAngleDeg; - /// |Δ z-angle| vs the previous second (index 0 is 0 by definition). + /// |Δ z-angle| vs the previous second (index 0 is 0 by definition). NaN when + /// either endpoint second is invalid — the arm may have moved across the gap + /// and the record cannot say. Every comparison against NaN is false, so a + /// consumer testing `deltaDeg[k] < threshold` degrades to "not still", never + /// to "perfectly still". final List deltaDeg; /// The sustained-inactivity window actually used, in seconds. @@ -161,8 +169,18 @@ ImmobilityMask immobilityMask( } // 1–2. z-angle + rolling-median smoothing. + // + // A second whose gravity vector was never decoded ([AccelSample.valid] false) + // is handed over as exact (0,0,0). That has a z-angle — 0.0°, constant — so a + // run of them reads as a wrist held perfectly still, which is how a decode gap + // became a "nap" at the confidence cap. Absence is not a measurement: those + // seconds carry NaN from here on, and every rule below asks about them + // explicitly instead of comparing against a number that isn't one. final raw = List.generate( - n, (i) => zAngle(accel[i].x, accel[i].y, accel[i].z)); + n, + (i) => accel[i].valid + ? zAngle(accel[i].x, accel[i].y, accel[i].z) + : double.nan); final ang = _rollingMedian(raw, smoothSec); // 3. per-second immobility: |Δ z-angle| < threshold sustained for ≥ window. @@ -188,10 +206,20 @@ ImmobilityMask immobilityMask( // * no movement seen, window short ⇒ UNDECIDABLE. We do not assert rest // (that would extrapolate stillness past the end of the record) and we // record it in `immobileUnknown` rather than guessing either way. + // + // A window containing an UNMEASURED second lands in the same two cases, for + // the same reason: movement already seen decides it (the sustained rule has + // failed on the data in hand), otherwise the arm could have moved where we + // were not looking ⇒ undecidable, never asserted rest. for (var i = 0; i < n; i++) { final hi = math.min(n, i + win); var maxd = 0.0; + var gap = !accel[i].valid; for (var k = i + 1; k < hi; k++) { + if (!accel[k].valid) { + gap = true; + break; + } if (dAng[k] > maxd) { maxd = dAng[k]; if (maxd >= angleThresholdDeg) break; @@ -199,8 +227,8 @@ ImmobilityMask immobilityMask( } final still = maxd < angleThresholdDeg; final fullWindow = hi - i >= win; - immobile[i] = still && fullWindow; - immobileUnknown[i] = still && !fullWindow; + immobile[i] = still && fullWindow && !gap; + immobileUnknown[i] = still && (!fullWindow || gap); } return ImmobilityMask( @@ -326,21 +354,31 @@ Metric vanHeesSleepWindow( inputs_used: inputs, note: 'van Hees angle-based REST window (5°/${sustainedMin}min); ' 'a rest period, not PSG sleep' - '${unresolved > 0 ? '; last ${unresolved}s of the record are ' - 'undecidable (forward window truncated) and are excluded' : ''}', + '${unresolved > 0 ? '; ${unresolved}s are undecidable (forward window ' + 'truncated by the record end, or unmeasured gravity) and are ' + 'excluded' : ''}', ); } /// Centered rolling median (window `w`, odd-ish), edge-clamped. +/// +/// NaN in = NaN out AT THAT INDEX: an unmeasured second must not inherit an +/// angle smoothed out of its measured neighbours, or the whole point of marking +/// it absent is undone one line later. Neighbouring NaNs are simply skipped +/// when smoothing a second that WAS measured. List _rollingMedian(List x, int w) { final n = x.length; if (w <= 1) return [...x]; final half = w ~/ 2; - final out = List.filled(n, 0); + final out = List.filled(n, double.nan); for (var i = 0; i < n; i++) { + if (x[i].isNaN) continue; final lo = math.max(0, i - half); final hi = math.min(n - 1, i + half); - final seg = x.sublist(lo, hi + 1); + final seg = [ + for (var k = lo; k <= hi; k++) + if (!x[k].isNaN) x[k] + ]; out[i] = median(seg)!; } return out; diff --git a/lib/src/onehz/types.dart b/lib/src/onehz/types.dart index 424d4fd..c0c7f3b 100644 --- a/lib/src/onehz/types.dart +++ b/lib/src/onehz/types.dart @@ -103,12 +103,13 @@ class Metric { /// numerics/maps). When absent, emits the honest `"—"` placeholder. Map toJson([Object? Function(T v)? encode]) { final m = {}; - if (value == null) { - m['value'] = '—'; - } else { - m['value'] = encode != null ? encode(value as T) : value; - } - m['confidence'] = round6(confidence); + final enc = value == null ? null : (encode != null ? encode(value as T) : value); + // A non-finite scalar is not a measurement, and jsonEncode throws on it — + // one such field used to discard the caller's entire bundle. Treat it as + // absent (last-resort backstop; producers should abstain properly instead). + final bad = enc is double && !enc.isFinite; + m['value'] = (enc == null || bad) ? '—' : enc; + m['confidence'] = round6(bad ? 0.0 : confidence); m['tier'] = tier; m['inputs_used'] = inputs_used; if (drivers != null) m['drivers'] = drivers!.map((d) => d.toJson()).toList(); diff --git a/lib/src/onehz/util.dart b/lib/src/onehz/util.dart index f90c596..ff584fc 100644 --- a/lib/src/onehz/util.dart +++ b/lib/src/onehz/util.dart @@ -156,8 +156,12 @@ double? theilSen(List y, [List? x]) { } /// 6-dp rounding used for stable JSON output. -double round6(double x) { - if (x.isNaN || x.isInfinite) return x; +/// +/// Returns `null` for a non-finite input. NaN/Infinity is not a measurement, and +/// `jsonEncode` THROWS on it — one bad field used to take down a whole +/// serialised bundle. Absence is the only honest encoding of "not a number". +double? round6(double x) { + if (!x.isFinite) return null; return (x * 1e6).roundToDouble() / 1e6; } @@ -166,25 +170,43 @@ double roundTo(double x, int decimals) { return (x * f).roundToDouble() / f; } -/// One spectral estimate (power at an angular/ordinary frequency). +/// One spectral estimate: power spectral DENSITY at an ordinary frequency. +/// +/// UNITS — this is the whole contract, and getting it wrong has shipped a +/// "Total power 0.1 ms²" for a night whose true total was 1260 ms²: +/// [psd] is in (unit of y)² per Hz. For an RR series in ms sampled on beat +/// times in SECONDS, that is **ms²/Hz**. Integrating it over a band (see +/// [LombScargle.bandPower]) therefore yields **ms²**, the unit the HRV +/// literature reports LF/HF/VLF/total in. class LsPoint { final double freqHz; - final double power; - const LsPoint(this.freqHz, this.power); + final double psd; + const LsPoint(this.freqHz, this.psd); } class LombScargle { final List spectrum; const LombScargle(this.spectrum); - /// Total power = Σ P(f)·Δf (trapezoid-free rectangular sum over the grid). + /// Band power = ∫ PSD(f)·df over [loHz, hiHz), rectangular sum over the grid. + /// + /// Returns (unit of y)² — **ms² for an RR series in ms** (see [LsPoint.psd]). + /// Integrated over the full resolvable band it approximates the series + /// variance, i.e. SDNN²; that identity is the unit test for this scaling. double bandPower(double loHz, double hiHz) { var p = 0.0; - for (var i = 1; i < spectrum.length; i++) { + for (var i = 0; i < spectrum.length; i++) { final f = spectrum[i].freqHz; if (f < loHz || f >= hiHz) continue; - final df = spectrum[i].freqHz - spectrum[i - 1].freqHz; - p += spectrum[i].power * df; + // Bin width: the step to the previous grid point, or (for the first grid + // point) the step to the next. Starting the loop at i=1 used to drop the + // grid's lowest point entirely, which zeroed narrow low-frequency bands. + final df = i > 0 + ? spectrum[i].freqHz - spectrum[i - 1].freqHz + : (spectrum.length > 1 + ? spectrum[1].freqHz - spectrum[0].freqHz + : 0.0); + p += spectrum[i].psd * df; } return p; } @@ -195,8 +217,8 @@ class LombScargle { var bestP = double.negativeInfinity; for (final pt in spectrum) { if (pt.freqHz < loHz || pt.freqHz > hiHz) continue; - if (pt.power > bestP) { - bestP = pt.power; + if (pt.psd > bestP) { + bestP = pt.psd; best = pt.freqHz; } } @@ -204,13 +226,24 @@ class LombScargle { } } -/// Lomb–Scargle periodogram for UNEVENLY-sampled data (Press & Rybicki 1989, -/// classic form with Horne–Baliunas normalization by data variance). +/// Lomb–Scargle PSD for UNEVENLY-sampled data (Press & Rybicki 1989). /// -/// [t] sample times (any consistent unit — pass SECONDS for Hz output), +/// [t] sample times — pass SECONDS, so the grid and the output are in Hz. /// [y] sample values. Computed on the supplied [freqsHz] grid. The series is /// mean-subtracted; the τ phase-offset makes the estimate time-shift -/// invariant. Returns null if <4 points or zero variance. +/// invariant. Returns null if <4 points, zero variance, or zero time span. +/// +/// UNITS: the returned [LsPoint.psd] is a PHYSICAL density in (unit of y)²/Hz — +/// NOT the dimensionless Horne–Baliunas variance-normalised power. The classic +/// normalisation divides the periodogram by the data variance, which cancels +/// the very unit the caller wants: `bandPower` then returns Hz, and labelling +/// it ms² understates a real 1260 ms² night as 0.1. So we keep the unnormalised +/// periodogram `0.5·(term1+term2)` and apply the standard one-sided +/// periodogram→density scaling `2·Δt`, with Δt the MEAN sample interval. The +/// identity that pins it: ∫PSD df over the resolvable band ≈ var(y). +/// +/// Ratios of band powers (LF/HF, nu_lf, coherence) are unaffected by this +/// scaling — it is a single positive constant across the whole spectrum. /// /// This operates on NATIVE sample times — no resampling — which is exactly why /// it's the correct PSD for unevenly-sampled beat-time RR. @@ -226,6 +259,16 @@ LombScargle? lombScargle(List t, List y, List freqsHz) { variance /= (n - 1); if (variance <= 0) return null; + var tMin = t[0], tMax = t[0]; + for (final ti in t) { + if (ti < tMin) tMin = ti; + if (ti > tMax) tMax = ti; + } + final span = tMax - tMin; + if (span <= 0) return null; + // One-sided periodogram → density: 2·Δt, with Δt the MEAN sample interval. + final psdScale = 2.0 * span / (n - 1); + final out = []; for (final fHz in freqsHz) { final w = 2 * math.pi * fHz; // angular frequency (rad / time-unit) @@ -253,8 +296,7 @@ LombScargle? lombScargle(List t, List y, List freqsHz) { } final term1 = cDen == 0 ? 0.0 : (cNum * cNum) / cDen; final term2 = sDen == 0 ? 0.0 : (sNum * sNum) / sDen; - final power = 0.5 * (term1 + term2) / variance; - out.add(LsPoint(fHz, power)); + out.add(LsPoint(fHz, 0.5 * (term1 + term2) * psdScale)); } return LombScargle(out); } @@ -269,3 +311,115 @@ List freqGrid(double loHz, double hiHz, int n) { /// Natural log guard: ln of a positive value, else null. double? safeLn(double x) => x > 0 ? math.log(x) : null; + +/// Calendar-day index (days since epoch) for each `yyyy-MM-dd` label. +/// +/// Trailing windows and "N nights running" counters in this package used to be +/// POSITIONAL — N rows, not N days — and the callers feed them only the days +/// that produced a derived row. After a wear gap a "28-day baseline" spanned +/// months, and `persistDays: 2` meant two RECORDED nights, so an elevated +/// Monday plus an elevated night three weeks later escalated an illness CUSUM +/// to red "sustained elevation". Index by these instead. +/// +/// An unparseable label falls back to its position — that is exactly the old +/// behaviour, so a caller with non-ISO labels is no worse off than before. +List calendarDays(List dates) { + final out = List.filled(dates.length, 0); + for (var i = 0; i < dates.length; i++) { + final d = DateTime.tryParse(dates[i]); + out[i] = d == null + ? i + : DateTime.utc(d.year, d.month, d.day).millisecondsSinceEpoch ~/ + 86400000; + } + return out; +} + +/// Average ranks, 1-based, ties sharing their mean rank. +/// +/// Tie handling is not a detail wherever this is used: journal fields are full +/// of ties (mood is 1–5, most people log the same 2 coffees most days) and so +/// are quantised daily metrics. Ranking ties arbitrarily invents an ordering +/// nobody reported. +List averageRanks(List xs) { + final idx = List.generate(xs.length, (i) => i) + ..sort((a, b) => xs[a].compareTo(xs[b])); + final ranks = List.filled(xs.length, 0); + var i = 0; + while (i < idx.length) { + var j = i; + while (j + 1 < idx.length && xs[idx[j + 1]] == xs[idx[i]]) { + j++; + } + // Ranks are 1-based, so positions i..j map to ranks i+1..j+1. + final shared = (i + 1 + j + 1) / 2.0; + for (var k = i; k <= j; k++) { + ranks[idx[k]] = shared; + } + i = j + 1; + } + return ranks; +} + +/// Two-sided p for a standard-normal z. `2·(1 − Φ(|z|))`. +/// +/// Numerical Recipes' `erfcc` — a Chebyshev fit to erfc with fractional error +/// under 1.2e-7 everywhere, which is four orders of magnitude finer than any +/// gate that reads it. No special-function dependency, no table. +double normalTwoSidedP(double zScore) { + if (!zScore.isFinite) return 1.0; + final x = zScore.abs() / math.sqrt2; + final t = 1.0 / (1.0 + 0.5 * x); + final ans = t * + math.exp(-x * x - + 1.26551223 + + t * + (1.00002368 + + t * + (0.37409196 + + t * + (0.09678418 + + t * + (-0.18628806 + + t * + (0.27886807 + + t * + (-1.13520398 + + t * + (1.48851587 + + t * + (-0.82215223 + + t * 0.17087277))))))))); + return clamp(ans, 0.0, 1.0); +} + +/// Benjamini-Hochberg (1995) step-up FDR adjustment over a family of p-values. +/// +/// Returns q-values POSITIONALLY ALIGNED to [ps]; a null p (a test that could +/// not be run) stays null and does not enter the family size — a test we +/// abstained from is not a test we performed. +/// +/// WHY THIS IS NOT OPTIONAL where it is used: a per-test 0.05 gate over a grid +/// of 36 simultaneous tests produces ~2 "findings" from pure noise, every time, +/// for every user. BH controls the expected FRACTION of the published findings +/// that are false, which is the quantity a screen full of "this moves your +/// recovery" rows is actually promising. +/// +/// The step-up enforcement (running minimum from the largest p downwards) is +/// the part everyone drops: without it the q-values are not monotone in p and a +/// weaker test can be published while a stronger one is refused. +List benjaminiHochberg(List ps) { + final idx = [ + for (var i = 0; i < ps.length; i++) + if (ps[i] != null) i + ]..sort((a, b) => ps[a]!.compareTo(ps[b]!)); + final m = idx.length; + final out = List.filled(ps.length, null); + var running = 1.0; + for (var k = m - 1; k >= 0; k--) { + final q = ps[idx[k]]! * m / (k + 1); + running = math.min(running, q); + out[idx[k]] = running; + } + return out; +} diff --git a/lib/src/onehz/wellness/anomaly.dart b/lib/src/onehz/wellness/anomaly.dart index ee9c31b..41ff445 100644 --- a/lib/src/onehz/wellness/anomaly.dart +++ b/lib/src/onehz/wellness/anomaly.dart @@ -8,7 +8,9 @@ // surfaces — so a single noisy night never cries wolf. // // Robustness: covariance is estimated from a trailing window via median/MAD -// (diagonal robust scale) + Spearman-style rank correlation off-diagonal, then +// (diagonal robust scale) + a Pearson correlation off-diagonal taken on the +// already-robustly-standardized columns (NOT a rank correlation — one outlier +// night in the window still moves it), then // regularized (ridge) so a near-singular small-sample covariance can't blow up // the distance. Features are sign-ORIENTED so "bad" is always positive // (HRV is negated: a DROP in HRV is the illness direction). @@ -29,7 +31,11 @@ import '../util.dart'; class AnomalyFeatures { final double? rhr; // RHR (↑ worse) final double? hrv; // lnRMSSD or RMSSD (↓ worse -> we negate internally) - final double? temp; // relative skin-temp (↑ worse) + /// RAW nightly-mean skin-temp in the device's own unit (`nightlySkinTemp`), + /// ↑ worse. NOT a z — this detector standardises every column itself against + /// the trailing window below, so a pre-standardised series is standardised + /// twice and `minBaseline`/the χ² gate stop meaning what they say. + final double? temp; final double? resp; // respiration rate (↑ worse) const AnomalyFeatures({this.rhr, this.hrv, this.temp, this.resp}); } @@ -73,15 +79,16 @@ const _featLabels = ['RHR', 'HRV(↓)', 'temp', 'resp']; /// [dates] labels; [feats] per-night features (same length). [baselineDays] /// trailing window for the covariance cloud; [minBaseline] min valid nights /// before any distance is computed; [chiSqGate] threshold on Mahalanobis² -/// (default 9.21 ≈ χ²_{0.99,2}, conservative); [persistDays] consecutive -/// candidate nights required to flag; [ridge] covariance regularizer fraction. +/// (see below — no fixed default); [persistDays] consecutive candidate nights +/// required to flag; [ridge] covariance regularizer fraction. /// /// [chiSqGate] OPTIONAL fixed Mahalanobis² threshold. When null (default) the -/// gate is DIMENSION-AWARE: a conservative χ²_{0.999, dof} upper quantile keyed -/// by the number of features actually present that night — so the false-alarm -/// rate stays honest as the vector dimension changes. (Robust MAD modestly -/// under-estimates SD on small samples, which inflates z; the conservative -/// 0.999 level absorbs that so normal nights don't trip the alarm.) +/// gate is DIMENSION- AND SAMPLE-AWARE: χ²_{0.999, dof} for the number of +/// features present that night, widened by [_madInflation] for how few nights +/// the MAD scale was estimated from. The bare χ² quantile assumes a KNOWN +/// covariance; ours is a MAD over as few as [minBaseline] nights, and against +/// the bare gate the measured false-candidate rate is 0.197/night at n = 10 and +/// dof = 4 — 196× nominal. See [_madInflationTable]. List multivariateAnomaly( List dates, List feats, { @@ -93,16 +100,19 @@ List multivariateAnomaly( }) { final n = feats.length; final out = []; + // CALENDAR days, not rows — see [calendarDays]. + final day = calendarDays(dates); var run = 0; + var lastScoredDay = -1 << 20; for (var i = 0; i < n; i++) { // Orient: HRV negated so a drop is positive ("worse" direction). final cur = _orient(feats[i]); - final lo = i - baselineDays < 0 ? 0 : i - baselineDays; // Build per-feature baseline columns (valid only) from the trailing window. final cols = List.generate(4, (_) => []); // Aligned rows (all 4 features present) for covariance off-diagonals. final rows = >[]; - for (var j = lo; j < i; j++) { + for (var j = i - 1; j >= 0; j--) { + if (day[i] - day[j] > baselineDays) break; final o = _orient(feats[j]); for (var f = 0; f < 4; f++) { if (o[f] != null) cols[f].add(o[f]!); @@ -186,12 +196,22 @@ List multivariateAnomaly( // Per-feature contribution to d² (diagonal share), for the "why". final drivers = []; for (var a = 0; a < keep.length; a++) { - drivers.add(Driver(_featLabels[keep[a]], round6(zc[a]), + drivers.add(Driver(_featLabels[keep[a]], roundTo(zc[a], 6), detail: 'standardized deviation')); } drivers.sort((x, y) => y.contribution.abs().compareTo(x.contribution.abs())); - final gate = chiSqGate ?? _chiSq999(keep.length); + // "N nights running" means CONSECUTIVE NIGHTS, not consecutive rows. + if (day[i] - lastScoredDay > 1) run = 0; + lastScoredDay = day[i]; + + // Smallest surviving per-feature baseline drives the gate: the noisiest + // scale estimate in the vector is the one inflating d². + var baseN = 1 << 30; + for (final f in keep) { + if (cols[f].length < baseN) baseN = cols[f].length; + } + final gate = chiSqGate ?? _chiSq999(keep.length) * _madInflation(baseN); final candidate = d2 > gate; if (candidate) { run++; @@ -204,9 +224,63 @@ List multivariateAnomaly( return out; } +/// How much the χ² gate must be widened because the scale is ESTIMATED, not +/// known. +/// +/// χ²(0.999) is the quantile of a Mahalanobis distance taken against a KNOWN +/// covariance. Ours is a median + MAD over `minBaseline` nights, and a MAD on +/// ten values is so noisy that d² lives in a far heavier-tailed distribution +/// than χ². Measured (200 k trials/cell, iid N(0,1) features, replicating this +/// function exactly — `_robustCorr`'s Pearson-on-standardised-columns, the 0.1 +/// ridge, the stddev fallback), the false-candidate rate against the bare χ² +/// gate was: +/// +/// baseline n | dof 2 | dof 3 | dof 4 +/// -----------|--------|--------|------- +/// 10 | 0.0796 | 0.1298 | 0.1973 ← 196× nominal at dof 4 +/// 14 | 0.0451 | 0.0709 | 0.1048 +/// 20 | 0.0244 | 0.0376 | 0.0532 +/// 28 | 0.0144 | 0.0202 | 0.0272 +/// 60 | 0.0050 | 0.0059 | 0.0075 +/// +/// That is user-facing: a flagged night drives a quiet-hours-overriding +/// "Unusual overnight physiology" notification, and the exposure is concentrated +/// in a new user's first month while the per-feature baseline grows from 10 to +/// `baselineDays`. +/// +/// The factor below is the empirical 99.9 %ile of d² divided by χ²(0.999) at +/// that dof, taken as the MAX over dof 2..4 at each n (conservative by +/// construction — a shared curve must not under-widen the widest case). +/// Interpolated log-linearly in n; below the first anchor it holds, above the +/// last it decays to 1.0 as MAD → σ. `test/onehz/wellness_test.dart` pins the +/// anchors. +const List<(int, double)> _madInflationTable = [ + (10, 13.23), + (14, 5.65), + (20, 3.17), + (28, 2.24), + (45, 1.65), + (90, 1.25), + (200, 1.0), +]; + +double _madInflation(int n) { + if (n <= _madInflationTable.first.$1) return _madInflationTable.first.$2; + if (n >= _madInflationTable.last.$1) return 1.0; + for (var i = 1; i < _madInflationTable.length; i++) { + final (n1, f1) = _madInflationTable[i]; + if (n > n1) continue; + final (n0, f0) = _madInflationTable[i - 1]; + final t = (math.log(n) - math.log(n0)) / (math.log(n1) - math.log(n0)); + return math.exp(math.log(f0) + t * (math.log(f1) - math.log(f0))); + } + return 1.0; +} + /// Conservative χ² 0.999 upper-quantile by degrees of freedom (1..4 — the four /// illness features). A single noisy night must clear this to even become a /// candidate, so the persistence gate then needs TWO such nights to flag. +/// Widened by [_madInflation] for the small-sample scale estimate. double _chiSq999(int dof) { switch (dof) { case 1: diff --git a/lib/src/onehz/wellness/changepoint.dart b/lib/src/onehz/wellness/changepoint.dart index 5a4feea..eb40b4d 100644 --- a/lib/src/onehz/wellness/changepoint.dart +++ b/lib/src/onehz/wellness/changepoint.dart @@ -4,14 +4,21 @@ // min-seg ≥7 d) + BOCPD online ... on smoothed daily aggregates." // // Two methods: -// 1. ONLINE two-sided CUSUM (Page 1954) — running detector that fires when the -// accumulated standardized deviation crosses a threshold; resets after a -// detection. Cheap, streaming, for "something just shifted". -// 2. OFFLINE exact change-point search via binary segmentation with an -// MBIC/BIC penalty (Killick 2012 cost = Gaussian change-in-mean SSE), with -// a min-segment length ≥7. (Exact PELT and binary segmentation give the -// same segmentation for the change-in-mean cost; we use the simpler -// recursive binary search guarded by the same penalty.) +// 1. ONLINE two-sided CUSUM (Page 1954) — running detector that standardizes +// each point against the data BEFORE it (an expanding pre-change baseline, +// restarted at every detection) and fires when the accumulated deviation +// crosses a threshold. Cheap, streaming, for "something just shifted". +// 2. OFFLINE GREEDY binary segmentation with a BIC penalty on the Gaussian +// change-in-mean cost (Killick 2012), min-segment length ≥7. This is an +// APPROXIMATION to the exact partition, NOT PELT: Killick, Fearnhead & +// Eckley 2012 (JASA 107(500):1590-1598) is built on exactly that contrast +// — PELT is exact while binary segmentation "only provides an approximate +// solution and can lead to poor estimation of the number and position of +// changepoints", because a greedy first split is not in general the +// optimal single split of an optimally-partitioned series. We take the +// approximation on purpose (it is retrospective, gated by a penalty AND an +// MDC effect-size floor); the header used to claim the two agree, which is +// the paper's counter-example, not its result. // // HONESTY: change-points are reported on SMOOTHED aggregates only and gated by // the penalty so we don't celebrate regression-to-the-mean noise. min-seg @@ -20,6 +27,7 @@ import 'dart:math' as math; import '../types.dart'; import '../util.dart'; +import '../foundations/baseline.dart'; // --------------------------------------------------------------------------- // 1. Online CUSUM change detector (two-sided) @@ -34,39 +42,92 @@ class CusumDetection { {'index': index, 'direction': direction, 'stat': round6(stat)}; } -/// Two-sided CUSUM over a series, standardized by a robust scale. +/// Two-sided CUSUM over a series, standardized against a PRE-change baseline. /// -/// [x] the (smoothed) series. [k] slack in scale-units (reference value), -/// [h] decision threshold in scale-units. Center/scale are the robust -/// median/MAD of the whole series (or a supplied [center]/[scale]). Returns -/// the indices where an upward/downward shift was detected (accumulator reset -/// after each detection). +/// [x] the (smoothed) series, oldest first. [k] slack in scale-units (reference +/// value), [h] decision threshold in scale-units. [minBaseline] points required +/// before any point can be scored. +/// +/// [dates] are the `yyyy-MM-dd` labels of [x], POSITIONALLY ALIGNED. Pass them. +/// Callers feed this a COMPACTED series — only the days that produced a value — +/// so without them the pre-change window and the accumulator are positional and +/// silently span wear gaps: "10 points of baseline" becomes ten scattered days +/// over four months, and evidence accumulated before a two-month gap still fires +/// a critical-priority "your resting HR shifted" on the first day back. A CUSUM +/// is evidence accumulated over CONSECUTIVE observations; across a gap there are +/// no observations, so there is no evidence to carry. This is the same fix +/// `clinical/illness_cusum.dart` already carries, and the failure mode +/// `util.dart`'s [calendarDays] documents. +/// +/// Omitting [dates] (or passing a mismatched length) keeps the old positional +/// behaviour — no caller is made worse off — but it is not the honest reading. +/// +/// Each point is standardized by the robust median/MAD of the points BEFORE it, +/// back to the start of the current regime — never by the whole series. That is +/// the difference between a detector and a description: standardizing by the +/// whole series folds the post-change data into the scale, which pins |z| at +/// MAD's 1.4826 reciprocal (≈0.675) for ANY balanced two-regime series whatever +/// the step size. A 145 bpm permanent step in a 30-day window was invisible, and +/// detection depended only on how LONG the regimes were. At h = 5 with k = 0.5 +/// that also meant the accumulator crept 0.1745/day and re-crossed the threshold +/// on ten separate later days, so the caller's "only when the shift lands on the +/// latest day" guard re-announced one shift as fresh over and over. +/// +/// After a detection the baseline restarts at the detection index (the new +/// regime is the new normal) and [minBaseline] points must accrue again, so one +/// shift is announced once. +/// +/// NO detection is emitted while a robust scale cannot be estimated — a +/// perfectly flat baseline has no dispersion to standardize against, and an +/// absent change-point beats a fabricated one. List cusumChangePoints( List x, { + List? dates, double k = 0.5, double h = 5.0, - double? center, - double? scale, + int minBaseline = 10, }) { final out = []; - if (x.length < 2) return out; - final c = center ?? median(x)!; - var s = scale ?? (mad(x) ?? 0); - if (s <= 0) s = (stddev(x) ?? 1.0); - if (s <= 0) s = 1.0; + if (x.length <= minBaseline) return out; + final day = + (dates != null && dates.length == x.length) ? calendarDays(dates) : null; + var regimeStart = 0; // first index of the current (pre-change) regime var up = 0.0, dn = 0.0; for (var i = 0; i < x.length; i++) { + // A break in the calendar breaks the regime AND the accumulator: the + // pre-change window restarts here and [minBaseline] consecutive days must + // accrue again before anything can be scored. + if (day != null && i > 0 && day[i] - day[i - 1] > 1) { + up = 0; + dn = 0; + regimeStart = i; + } + final baseline = x.sublist(regimeStart, i); + if (baseline.length < minBaseline) continue; + // ponytail: median/MAD recomputed over the expanding window each step — + // O(n² log n), and n is a 90-day series. Make it incremental if it ever + // runs on something longer. + final c = median(baseline)!; + final s = mad(baseline) ?? 0; + if (s <= 0 || !s.isFinite) { + // Degenerate (constant) baseline: there is no robust dispersion to + // standardize against. Hold, don't accumulate. Deliberately NO stddev + // fallback here (unlike illness_cusum, whose baseline window is a fixed + // 28 calendar days of the user's OWN quiet nights): this window grows + // until a detection, so on a flat history the SD is driven entirely by + // the very point under test — sqrt(n)-ish z for a 0.5 bpm move — and a + // critical-priority notification would fire on noise-free arithmetic. + continue; + } final z = (x[i] - c) / s; up = math.max(0, up + z - k); dn = math.max(0, dn - z - k); - if (up > h) { - out.add(CusumDetection(i, 1, up)); - up = 0; - dn = 0; - } else if (dn > h) { - out.add(CusumDetection(i, -1, dn)); + if (up > h || dn > h) { + final upward = up > h; + out.add(CusumDetection(i, upward ? 1 : -1, upward ? up : dn)); up = 0; dn = 0; + regimeStart = i; // the shifted level becomes the new baseline } } return out; @@ -102,20 +163,36 @@ double _segSse(List x, List prefix, List prefixSq, return sumSq - sum * sum / n; } -/// Offline change-point detection by binary segmentation with a BIC/MBIC +/// Offline change-point detection by greedy binary segmentation with a BIC /// penalty on the Gaussian change-in-mean cost. /// /// [x] the (smoothed) daily-aggregate series. [minSeg] minimum segment length -/// (≥7 per catalog). The per-change penalty defaults to MBIC-style -/// `penaltyK · σ̂² · ln(n)` where σ̂² is the variance of the full series; a -/// split is accepted only if it reduces SSE by more than the penalty. +/// (≥7 per catalog). The per-change penalty is `penaltyK · σ̂² · ln(n)` where +/// σ̂² is the variance of the full series; a split is accepted only if it +/// reduces SSE by more than the penalty. +/// +/// [penaltyK] DEFAULTS TO 2.0 = BIC, and that 2 is not a taste setting. The +/// reference implementation (`changepoint::penalty_decision`) gives +/// BIC = (diffparam + 1)·log(n); for the Normal change-in-mean with known +/// variance diffparam = 1, so BIC = 2·log(n) on the STANDARDISED cost, which is +/// 2·σ̂²·ln(n) on the raw-SSE scale [_binSeg] compares against. This shipped at +/// 1.0 — exactly HALF the criterion the docstring named — so the detector split +/// more eagerly than the method it claimed to be. Use 3.0 for a flat MBIC-like +/// approximation (the real MBIC also carries Zhang & Siegmund's +/// Σ log(τᵢ − τᵢ₋₁) segment-length term, which this does not implement — so do +/// not call it MBIC). +/// +/// It does NOT cancel against the σ̂² inflation the note below describes: on a +/// series containing a real shift the full-series σ̂² already over-penalises, +/// and halving the constant was not a correction for that, it was a second +/// error pointing the other way with no reason to match in size. /// /// Returns the change-point indices (start of each new segment), segment means, /// and segment spans. Metric segmentChangePoints( List x, { int minSeg = 7, - double penaltyK = 1.0, + double penaltyK = 2.0, double? penaltyOverride, }) { const inputs = ['daily_aggregate']; @@ -143,6 +220,31 @@ Metric segmentChangePoints( _binSeg(x, prefix, prefixSq, 0, n, minSeg, penalty, cps); cps.sort(); + // EFFECT-SIZE GATE (MT-10). Statistical significance is not a finding: a + // step the instrument cannot resolve is not a step. Each surviving boundary + // must move the mean by at least the MDC of the segment BEFORE it — the + // quiet regime is the only honest noise estimate, since the full series + // contains the shift itself. No dispersion in the pre-segment ⇒ no MDC ⇒ the + // boundary is dropped, never waved through. + var dropped = 0; + var changed = true; + while (changed && cps.isNotEmpty) { + changed = false; + final edges = [0, ...cps, n]; + for (var i = 1; i < edges.length - 1; i++) { + final lo = edges[i - 1], cut = edges[i], hi = edges[i + 1]; + final before = x.sublist(lo, cut); + final step = (mean(x.sublist(cut, hi))! - mean(before)!).abs(); + final gate = mdc(robustBaseline(before, minValid: minSeg)); + if (gate == null || step < gate) { + cps.remove(cut); + dropped++; + changed = true; + break; // means around the merge moved; re-evaluate from scratch + } + } + } + // Build segments + means. final bounds = [0, ...cps, n]; final segments = >[]; @@ -157,8 +259,17 @@ Metric segmentChangePoints( confidence: 0.6, tier: Tier.estimate, inputs_used: inputs, - note: 'binary segmentation w/ BIC-penalized change-in-mean; min-seg=$minSeg. ' - 'Run on SMOOTHED aggregates only — do not celebrate regression-to-mean.', + note: + 'GREEDY binary segmentation (an approximation to the exact partition, ' + 'Killick 2012) w/ BIC-penalized change-in-mean; min-seg=$minSeg; ' + 'below-MDC boundaries dropped=$dropped. Run on SMOOTHED aggregates ' + 'only — do not celebrate regression-to-mean. RETROSPECTIVE ONLY: no ' + 'forward alerting, and a boundary is a date that can MOVE when more ' + 'data arrives. The penalty is σ̂² of the FULL series, so a series ' + 'containing one big real shift inflates its own penalty and the ' + 'detector UNDER-SPLITS by construction — an empty result is not ' + 'evidence of stability. It cannot separate infection from a training ' + 'block, a heat wave, a new medication, altitude or a cycle phase.', ); } diff --git a/lib/src/onehz/wellness/cycle_lengths.dart b/lib/src/onehz/wellness/cycle_lengths.dart new file mode 100644 index 0000000..7fda577 --- /dev/null +++ b/lib/src/onehz/wellness/cycle_lengths.dart @@ -0,0 +1,145 @@ +// WELLNESS — logged cycle lengths, and the two published criterion numbers +// (WH-08). +// +// Arithmetic on LOGGED ONSET DATES ONLY. No PPG, no temperature, no HRV, +// nothing that can be wrong for sensor reasons. Consecutive-onset differencing +// and one absolute value. +// +// WHAT IT PRODUCES: her own cycle lengths, the difference between each +// consecutive pair, and the two numbers the published criteria use (Harlow +// 2012 / STRAW+10): a persistent 7-day-or-more difference between consecutive +// cycles, and a 60-day-or-longer interval. The screen renders the lengths as a +// bar series with those two lines on it and NO VERDICT TEXT AT ALL. +// +// NAMING. There is no stage vocabulary in this file and there must not be one +// added — not in a string, not in a field, not in a local variable. An +// identifier leaks into logs, into exports and eventually into a sentence on a +// screen, which is precisely how a description of arithmetic turns into a +// diagnosis nobody wrote. Everything here is named after what it computes. +// +// REFUSING ON HOLES IS THE FEATURE. This is entirely hostage to logging +// discipline: one missed start manufactures a single 60-day "cycle" that meets +// a criterion on its own. So an interval longer than [implausibleGapDays] is +// classed UNUSABLE — we cannot tell a long interval from a forgotten log — and +// its presence blocks the whole series rather than being quietly charted. +// +// This never says anyone is anything, never predicts an age, and never +// headlines a label. The card carries a non-dismissible line that cycle change +// has many causes — thyroid, stress, weight change, contraception, PCOS — and +// that this is a prompt to ask a clinician, not an answer. + +import '../types.dart'; +import '../util.dart'; + +/// Consecutive-cycle difference used by the published criteria, in days. +const int cycleLengthDifferenceCriterionDays = 7; + +/// Long-interval criterion, in days. +const int cycleLongIntervalCriterionDays = 60; + +/// Logged onsets required before anything is computed. Roughly a year: the +/// criteria are about PERSISTENT change, and a handful of cycles cannot show +/// persistence. +const int cycleLengthMinCycles = 12; + +/// Above this, an interval is more likely a missed log than a cycle. +const int cycleLengthImplausibleGapDays = 90; + +class CycleLengthSeries { + /// Days between consecutive logged onsets, oldest first. + final List lengthsDays; + + /// |lengths[i+1] − lengths[i]|, one shorter than [lengthsDays]. + final List consecutiveDifferencesDays; + + /// Largest value in [consecutiveDifferencesDays]. + final int maxConsecutiveDifferenceDays; + + /// Largest value in [lengthsDays]. + final int longestIntervalDays; + + const CycleLengthSeries({ + required this.lengthsDays, + required this.consecutiveDifferencesDays, + required this.maxConsecutiveDifferenceDays, + required this.longestIntervalDays, + }); + + Map toJson() => { + 'lengths_days': lengthsDays, + 'consecutive_differences_days': consecutiveDifferencesDays, + 'max_consecutive_difference_days': maxConsecutiveDifferenceDays, + 'longest_interval_days': longestIntervalDays, + 'difference_criterion_days': cycleLengthDifferenceCriterionDays, + 'long_interval_criterion_days': cycleLongIntervalCriterionDays, + }; +} + +/// Cycle lengths from logged onset day labels (`YYYY-MM-DD`, any order). +/// +/// Day labels, differenced as UTC calendar dates so a DST boundary cannot turn +/// a 28-day cycle into 27. Nothing here touches an epoch timestamp. +Metric cycleLengthSeries( + List onsetDates, { + int minCycles = cycleLengthMinCycles, + int implausibleGapDays = cycleLengthImplausibleGapDays, +}) { + const inputs = ['cycle_log_onsets']; + const caveat = 'Logged dates only — no sensor input. Cycle change has many ' + 'causes (thyroid, stress, weight change, contraception, PCOS); this is a ' + 'prompt to ask a clinician, not an answer. No verdict, no label, no ' + 'prediction.'; + + final days = []; + for (final d in onsetDates) { + final parsed = DateTime.tryParse('${d}T00:00:00Z'); + if (parsed != null) days.add(parsed.millisecondsSinceEpoch ~/ 86400000); + } + days.sort(); + // Same day logged twice is one onset. + final unique = []; + for (final d in days) { + if (unique.isEmpty || unique.last != d) unique.add(d); + } + + if (unique.length < minCycles + 1) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: '${needBaselineNote(have: unique.length, need: minCycles + 1)} ' + '$caveat', + ); + } + + final lengths = [ + for (var i = 1; i < unique.length; i++) unique[i] - unique[i - 1] + ]; + final unusable = lengths.where((l) => l > implausibleGapDays).length; + if (unusable > 0) { + // A hole is indistinguishable from a long interval, and a long interval is + // exactly what one of the criteria is about. Refuse the series rather than + // chart a manufactured one. + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'log_has_holes:intervals_over_${implausibleGapDays}d=$unusable — ' + 'a missed start and a long interval look identical here. $caveat', + ); + } + + final diffs = [ + for (var i = 1; i < lengths.length; i++) (lengths[i] - lengths[i - 1]).abs() + ]; + return Metric( + value: CycleLengthSeries( + lengthsDays: lengths, + consecutiveDifferencesDays: diffs, + maxConsecutiveDifferenceDays: diffs.reduce((a, b) => a > b ? a : b), + longestIntervalDays: lengths.reduce((a, b) => a > b ? a : b), + ), + confidence: clamp(lengths.length / 24.0, 0.3, 0.8), + tier: Tier.high, + inputs_used: inputs, + note: caveat, + ); +} diff --git a/lib/src/onehz/wellness/readiness_composite.dart b/lib/src/onehz/wellness/readiness_composite.dart index 6337ec9..72ba7eb 100644 --- a/lib/src/onehz/wellness/readiness_composite.dart +++ b/lib/src/onehz/wellness/readiness_composite.dart @@ -3,8 +3,11 @@ // ★ CANONICAL recovery/readiness (ARCHITECTURE_V2 `recovery.score`). ★ // Per the "one source per concept" invariant there is exactly ONE headline // readiness, and this is it: disclosed weights (HRV>RHR>RR>temp) + ranked -// drivers + personal-baseline robust z-scores (median+MAD `robustZ`) + an -// SWC/TE gate that will say "no meaningful change". The other readiness +// drivers + personal-baseline robust z-scores (median+MAD `robustZ`). The +// SWC/TE "no meaningful change" flag is GONE — it was computed and serialized +// for months with no reader, so a flat night rendered exactly like a real one +// while the code claimed otherwise. If a surface wants to say "flat", add the +// gate back WITH the surface. The other readiness // function — `glassBoxReadiness` in human/readiness_glassbox.dart — is the // DEPRECATED duplicate (ARCHITECTURE_V2 "DROP: the duplicate readiness // composite"); it is kept exported for back-compat but is INTERNAL and must not @@ -25,10 +28,7 @@ // dropped, never zero-imputed). // 4. The composite z is mapped to a 0..100 score via a logistic so typical // days land near 50. -// 5. SWC/TE gate: if |composite z| is below the smallest-worthwhile-change of -// the dominant input, we report "no meaningful change" (flat) — the -// credibility signal is the willingness to say nothing. -// 6. ALWAYS attach the per-input contribution breakdown |w_i·z_i| ranked. +// 5. ALWAYS attach the per-input contribution breakdown |w_i·z_i| ranked. // // HONESTY: glass-box index (weights disclosed); "—" when no inputs present; // every score carries its drivers; never names a driver below its MDC. @@ -36,6 +36,7 @@ import 'dart:math' as math; import '../types.dart'; import '../util.dart'; +import 'temp_circadian.dart' show kMinSettledFraction; /// One readiness input: its current value + a trailing baseline window + the /// sign of "good" (+1 if higher is better, -1 if higher is worse) + a weight. @@ -45,13 +46,20 @@ class ReadinessInput { final List baseline; // trailing personal-baseline window final int goodSign; // +1 higher-is-better, -1 higher-is-worse final double weight; // relative importance (HRV>RHR>RR>temp) + + /// Set when a value WAS measured but this input refuses to let readiness use + /// it tonight (see [tempInput]). [value] is null in that case, so the input + /// drops out and the weights renormalise — but the reason is named in the + /// composite's note instead of vanishing. + final String? refusal; const ReadinessInput( this.label, this.value, this.baseline, this.goodSign, - this.weight, - ); + this.weight, { + this.refusal, + }); } /// Canonical default inputs (caller supplies values + baselines). Weights encode @@ -62,37 +70,91 @@ ReadinessInput rhrInput(double? v, List base) => ReadinessInput('RHR', v, base, -1, 0.30); ReadinessInput respInput(double? v, List base) => ReadinessInput('RR', v, base, -1, 0.20); -ReadinessInput tempInput(double? v, List base) => - ReadinessInput('temp', v, base, -1, 0.10); + +/// The skin-temp driver — GATED ON SETTLEDNESS, and the gate has no safe +/// default. +/// +/// The channel stays; what is gated is readiness's USE of it. A nightly mean +/// taken over a window the strap spent partly warming up (or off the body) is +/// displaced downwards, and `goodSign = -1` reads downwards as GOOD: on the one +/// real gen4 night carrying a two-hour cold segment the ungated input pushed +/// readiness UP by 7.6 points on the shipped baseline (and by ~26 against the +/// clean-night baseline the audit measured against). A cold strap must not read +/// as a recovered person. +/// +/// [settledFraction] is `nightlySkinTemp`'s scalar for this night. NULL means +/// "nobody measured it", which is not the same as "it was fine" — the input is +/// refused, exactly as it is when the fraction is below [minSettledFraction], +/// and the composite renormalises over the drivers that are present. Pass the +/// fraction to get the driver back. +/// +/// The ORIENTATION is inherited, not endorsed: Kräuchi's distal-proximal +/// gradient work says nocturnal distal skin temperature BELOW baseline is +/// vasoconstriction, not recovery, and `glassBoxReadiness` already disagrees +/// with this file by using |z|. Deciding the sign is a separate, deliberate +/// change (RD-05); this one only stops a sensor artifact from paying out. +ReadinessInput tempInput( + double? v, + List base, { + double? settledFraction, + double minSettledFraction = kMinSettledFraction, +}) { + if (v == null) return ReadinessInput('temp', null, base, -1, 0.10); + if (settledFraction == null) { + return ReadinessInput('temp', null, base, -1, 0.10, + refusal: 'temp: no settled fraction measured for this night — an ' + 'ungated nightly mean cannot tell skin from a warming strap'); + } + if (settledFraction < minSettledFraction) { + return ReadinessInput('temp', null, base, -1, 0.10, + refusal: 'temp: unsettled_skin_temp:settled=' + '${round6(settledFraction)},need=${round6(minSettledFraction)}'); + } + return ReadinessInput('temp', v, base, -1, 0.10); +} class Readiness { final double score; // 0..100 glass-box readiness final double compositeZ; // weighted, sign-oriented composite z - final bool meaningful; // passed the SWC gate (else "flat") - const Readiness(this.score, this.compositeZ, this.meaningful); + const Readiness(this.score, this.compositeZ); Map toJson() => { 'score': round6(score), 'composite_z': round6(compositeZ), - 'meaningful': meaningful, }; } /// Compute the honest readiness composite. /// /// Each present input with a usable robust baseline contributes a sign-oriented -/// robust z. Weights are renormalized over present inputs. [swcMultiplier] sets -/// the smallest-worthwhile-change gate (Hopkins 0.2 of the composite scale, -/// i.e. of unit SD here since z is standardized). +/// robust z. Weights are renormalized over present inputs. /// Required minimum baseline points (per input) before readiness can compute. const int readinessCompositeMinBaseline = 3; +/// Minimum number of inputs, and minimum surviving weight, before a composite +/// is a composite at all. +/// +/// Renormalising over present inputs is the frozen catalog's own rule +/// ("Reweight on missing inputs, don't zero") and stays. What it must not do is +/// hand the WHOLE 0..100 score to one input: with only `temp` present, its +/// disclosed 0.10 becomes an effective 1.0 and a single sensor artifact IS the +/// headline. Reachable — each input needs its own ≥3-night baseline and the four +/// series fill at different rates. On the real gen4 corpus two of seventeen days +/// scored off exactly one input (readiness 44.7 and 0.8); both now read "—". +const int readinessCompositeMinInputs = 2; +const double readinessCompositeMinWeight = 0.5; + Metric readinessComposite( List inputs, { - double swcMultiplier = 0.2, int minBaseline = readinessCompositeMinBaseline, + int minInputs = readinessCompositeMinInputs, + double minWeightSum = readinessCompositeMinWeight, }) { final used = []; final drivers = []; + final refusals = [ + for (final inp in inputs) + if (inp.refusal != null) inp.refusal! + ]; var weightSum = 0.0; var weightedZ = 0.0; // Track the best-covered input that has a value but a too-short baseline, so @@ -133,20 +195,33 @@ Metric readinessComposite( drivers.add(Driver(inp.label, inp.weight * oriented, detail: 'oriented $method=${round6(oriented)}')); } - if (used.isEmpty || weightSum == 0) { + final suffix = refusals.isEmpty ? '' : ' Refused: ${refusals.join('; ')}.'; + if (used.length < minInputs || weightSum < minWeightSum) { // If inputs HAD values but their baselines were too short, say so in the // machine-readable need_baseline convention (don't fabricate a score). - if (anyValuePresent && bestShortHave >= 0) { + if (used.isEmpty && anyValuePresent && bestShortHave >= 0) { return Metric.absent( tier: Tier.estimate, inputs_used: const [], - note: needBaselineNote(have: bestShortHave, need: minBaseline), + note: needBaselineNote(have: bestShortHave, need: minBaseline) + suffix, ); } - return const Metric.absent( + if (used.isEmpty) { + return Metric.absent( + tier: Tier.estimate, + inputs_used: const [], + note: 'no readiness inputs present — "—" (never imputed).$suffix', + ); + } + // Present but too thin to be a COMPOSITE. Renormalising here would promote + // one disclosed weight to 1.0; see [readinessCompositeMinInputs]. + return Metric.absent( tier: Tier.estimate, - inputs_used: [], - note: 'no readiness inputs present — "—" (never imputed)', + inputs_used: used, + note: 'need_inputs:have=${used.length},need=$minInputs,' + 'weight=${round6(weightSum)},need_weight=${round6(minWeightSum)} — ' + 'renormalising this few would hand the whole score to one input.' + '$suffix', ); } // Renormalize weights over present inputs. @@ -155,15 +230,11 @@ Metric readinessComposite( // composite z (glass-box: contributions are definitional within the formula). final normDrivers = [ for (final d in drivers) - Driver(d.label, round6(d.contribution / weightSum), detail: d.detail) + Driver(d.label, roundTo(d.contribution / weightSum, 6), detail: d.detail) ]; // Rank by |contribution| (the deterministic-narrative driver ordering). normDrivers.sort((a, b) => b.contribution.abs().compareTo(a.contribution.abs())); - // SWC gate: standardized composite z has unit SD by construction, so the SWC - // is swcMultiplier (×1). Below it => not a meaningful change ("flat"). - final meaningful = composite.abs() > swcMultiplier; - // Map composite z -> 0..100 via logistic; ~50 at z=0, scale so ±2 z ~ 12/88. final score = 100 / (1 + math.exp(-composite)); @@ -171,13 +242,13 @@ Metric readinessComposite( final conf = clamp(0.3 + 0.15 * used.length, 0.3, 0.9); return Metric( - value: Readiness(score, composite, meaningful), + value: Readiness(score, composite), confidence: conf, tier: Tier.estimate, inputs_used: used, drivers: normDrivers, note: 'GLASS-BOX readiness: disclosed weights HRV>RHR>RR>temp, renormalized ' - 'over present inputs; SWC-gated (meaningful=$meaningful). Drivers are ' - 'definitional within the formula (correction, not inferred cause).', + 'over present inputs. Drivers are definitional within the formula ' + '(correction, not inferred cause).$suffix', ); } diff --git a/lib/src/onehz/wellness/temp_circadian.dart b/lib/src/onehz/wellness/temp_circadian.dart index 751dc16..2393e52 100644 --- a/lib/src/onehz/wellness/temp_circadian.dart +++ b/lib/src/onehz/wellness/temp_circadian.dart @@ -11,16 +11,33 @@ // 2. Nonparametric circadian statistics (Witting/van Someren): // IS interdaily stability (rhythm strength vs population of days) // IV intradaily variability (fragmentation) -// M10 most-active 10 h mean + onset; L5 least-active 5 h mean + onset -// RA relative amplitude = (M10-L5)/(M10+L5) -// (Computed on temp; "active" = warmest for temp, since wrist temp is -// ANTIPHASE to core — high distal skin temp marks rest/sleep.) // -// HONESTY: phase only. No absolute °C, no fever. We optionally de-mask by -// dropping high-motion epochs (activity raises distal skin temp via vasomotor -// confound) before computing the rhythm, and report how many epochs survived. +// M10, L5 AND RA ARE WITHHELD — not null-when-unobserved, absent from the type. +// The series this runs on is median-centred, so it is SIGNED, and +// RA = (M10−L5)/(M10+L5) has a denominator that crosses zero: the "relative +// amplitude" flips sign and blows up on a series whose warm and cool windows +// straddle the centre. M10/L5 go with it because their only published use is +// RA and because a "warmest 10 h" of a centred series is not a level. IS/IV +// need no such level and stay. +// +// PER-DEVICE (device.dart contract). gen4 reports skin temperature as a raw +// ADC count and gen5 as centi-°C; they arrive in ONE array with no unit. A +// cosinor amplitude is unit-bearing, so an amplitude computed here is only +// comparable to another of the SAME family — hence [TempCircadian.unit], which +// exists so that no screen and no aggregate ever puts the two side by side. +// The de-mask motion gate is also per-family: on our own captures the resting +// accel noise floor differs ~4x between the families (gen4 |‖a‖−1| median +// 0.034 g / p90 0.058 g; gen5+MG median 0.007–0.012 g / p90 0.018–0.024 g), so +// one shared gate either de-masks half a gen4 night or nothing on a gen5 one. +// Unknown family REFUSES. +// +// HONESTY: phase only. No absolute °C, no fever, no core temperature, and the +// antiphase relationship to core is not to be quietly inverted for a nicer +// chart. Charging happens at a consistent time for most people, so the off-wrist +// gap is PHASE-LOCKED and biases the rhythm — the de-masking exists for that. import 'dart:math' as math; +import '../device.dart'; import '../types.dart'; import '../util.dart'; import '../clinical/cosinor.dart'; @@ -28,32 +45,17 @@ import '../clinical/cosinor.dart'; class CircadianNonparam { final double interdailyStability; // IS, 0..1 (higher = more stable) final double intradailyVariability; // IV, ~0..2 (higher = more fragmented) - final double m10; // mean of most-active (warmest) 10 h - final double m10OnsetHour; // start hour-of-day of the M10 window - final double l5; // mean of least-active (coolest) 5 h - final double l5OnsetHour; // start hour-of-day of the L5 window - final double relativeAmplitude; // RA = (M10-L5)/(M10+L5) final int epochsPerDay; final int nDays; const CircadianNonparam({ required this.interdailyStability, required this.intradailyVariability, - required this.m10, - required this.m10OnsetHour, - required this.l5, - required this.l5OnsetHour, - required this.relativeAmplitude, required this.epochsPerDay, required this.nDays, }); Map toJson() => { 'interdaily_stability': round6(interdailyStability), 'intradaily_variability': round6(intradailyVariability), - 'm10': round6(m10), - 'm10_onset_hour': round6(m10OnsetHour), - 'l5': round6(l5), - 'l5_onset_hour': round6(l5OnsetHour), - 'relative_amplitude': round6(relativeAmplitude), 'epochs_per_day': epochsPerDay, 'n_days': nDays, }; @@ -62,31 +64,186 @@ class CircadianNonparam { class TempCircadian { final CosinorFit? cosinorFit; final CircadianNonparam? nonparam; - const TempCircadian(this.cosinorFit, this.nonparam); + + /// The unit the input series was in — `adc_counts` (gen4) or `centi_c` + /// (gen5). The cosinor AMPLITUDE is in this unit. Two families' amplitudes + /// are not comparable and must never be pooled, plotted together or + /// converted; nothing here is °C on screen. + final String unit; + const TempCircadian(this.cosinorFit, this.nonparam, this.unit); Map toJson() => { if (cosinorFit != null) 'cosinor': cosinorFit!.toJson(), if (nonparam != null) 'nonparam': nonparam!.toJson(), + 'unit': unit, + }; +} + +/// What each family's skin-temp column actually holds, how still "still" is on +/// its accelerometer, and how far BELOW a night's own level a reading can fall +/// and still be skin. All three are sensor properties, not physiology. +class _TempCal { + final String unit; + final double motionGate; // g of |‖a‖ − 1| above which an epoch is masked + /// One-sided settle band, in [unit]: a sample more than this far BELOW the + /// night's median is not settled skin (see [nightlySkinTemp]). Null = this + /// family has no measured band, so the settled mean REFUSES for it rather + /// than borrow gen4's counts. + final double? settleBandLow; + const _TempCal(this.unit, this.motionGate, this.settleBandLow); +} + +const Map _tempCal = { + // gen4's 40 counts: on the 8 real gen4 nights in whoop-4.db the seven clean + // nights keep 94.4–100.0 % of their sleep-window samples above + // (night median − 40), while the one night carrying a two-hour cold segment + // keeps 78.7 %. The band is ~6× the 6.47-count between-night SD and ~1.4× the + // 29.3-count corpus-wide circadian range, so ordinary rhythm survives it. + DeviceFamily.gen4: _TempCal('adc_counts', 0.10, 40.0), + // gen5 has NO measured band. The exports carry 106 (W5) and 933 (MG) non-zero + // skin-temp rows in total — not one night clears the 60-sample floor — so + // there is nothing to calibrate against. Scaling gen4's 40 counts by a + // counts-per-°C guess would be gen4's number wearing a gen5 badge, which is + // exactly what device.dart's contract forbids. Fill this in from gen5 nights, + // not from arithmetic. + DeviceFamily.gen5: _TempCal('centi_c', 0.04, null), +}; + +/// A nightly skin-temp mean that knows how much of the night it is made of. +class SettledSkinTemp { + /// Mean of the SETTLED samples only (see [nightlySkinTemp]), in [unit]. + final double mean; + + /// Fraction of the night's valid samples that were settled, 0..1. This is the + /// scalar RD-15 asks for (`skin_temp_settled_fraction`) — persist it. + final double settledFraction; + + /// `adc_counts` (gen4) or `centi_c` (gen5). Never °C on screen. + final String unit; + const SettledSkinTemp(this.mean, this.settledFraction, this.unit); + Map toJson() => { + 'mean': round6(mean), + 'settled_fraction': round6(settledFraction), + 'unit': unit, }; } +/// Minimum settled fraction a night must reach before its skin-temp mean may be +/// USED as a measurement. 0.80 is the audit's floor and costs one night in +/// eight on the real gen4 corpus. +const double kMinSettledFraction = 0.80; + +/// Nightly skin-temp mean over the SETTLED portion of a sleep window. +/// +/// THE PROBLEM THIS EXISTS FOR. The plain mean of a sleep window's skin-temp +/// samples cannot tell a fever from a strap that was just put on. On the real +/// gen4 corpus one night carries a two-hour segment ~100 counts below the rest +/// of that same night — a cold strap, HR present throughout, so not off-wrist, +/// just not yet at skin temperature — and it displaces that night's mean by +/// more than the entire 29.3-count daily rhythm. Every consumer of the nightly +/// mean (readiness's temp driver, `tempIllnessFlag`, `multivariateAnomaly`'s +/// temp column, `tempCircadian`) inherits that displacement. +/// +/// THE RULE. A sample is SETTLED when it sits no more than the family's +/// [_TempCal.settleBandLow] BELOW the night's own median. One-sided on purpose: +/// warm-up and off-wrist both read LOW, while a fever reads HIGH and must pass +/// through untouched — a symmetric band would delete the signal the channel +/// exists for. +/// +/// Returns the settled mean plus the settled fraction. ABSENT when the family +/// has no measured band, when there are fewer than [minSamples] valid samples, +/// or when the settled fraction is below [minSettledFraction] — in that last +/// case the night has no usable temperature and the consumer must drop the +/// input, not substitute one. +/// +/// The channel itself is NEVER removed or nulled by this: it gates USE. +Metric nightlySkinTemp( + List samples, { + required String? deviceFamily, + double minSettledFraction = kMinSettledFraction, + int minSamples = 60, +}) { + const inputs = ['skin_temp_adc', 'device_family']; + final cal = calibrationFor(_tempCal, deviceFamily); + if (cal == null || cal.settleBandLow == null) { + return Metric.absent( + tier: Tier.relative, + inputs_used: inputs, + note: unknownFamilyNote(deviceFamily), + ); + } + final v = [ + for (final s in samples) + if (s.valid && s.adc > 0) s.adc + ]; + if (v.length < minSamples) { + return Metric.absent( + tier: Tier.relative, + inputs_used: inputs, + note: needBaselineNote(have: v.length, need: minSamples), + ); + } + final floor = median(v)! - cal.settleBandLow!; + final kept = [ + for (final x in v) + if (x >= floor) x + ]; + final frac = kept.length / v.length; + if (frac < minSettledFraction) { + return Metric.absent( + tier: Tier.relative, + inputs_used: inputs, + note: 'unsettled_skin_temp:settled=${round6(frac)},' + 'need=${round6(minSettledFraction)} — the strap spent part of this ' + 'night below skin temperature (warm-up or off-body), so the nightly ' + 'mean is not a measurement of this person', + ); + } + return Metric( + value: SettledSkinTemp(mean(kept)!, frac, cal.unit), + confidence: clamp(frac, 0.0, 1.0), + tier: Tier.relative, + inputs_used: inputs, + note: 'RELATIVE nightly skin-temp mean over the SETTLED portion only ' + '(${round6(frac)} of valid samples, band ${cal.settleBandLow} ' + '${cal.unit} below the night median; one-sided so a fever passes). ' + 'Unit is ${cal.unit} — never °C, never compared across families.', + ); +} + /// Relative skin-temp circadian analysis on a (de-masked) time-series of the /// relative temp ADC. /// -/// [samples] consecutive AdcSamples of the relative skin-temp channel (raw -/// counts). [accel] OPTIONAL co-sampled accel for activity de-masking; when -/// supplied, epochs whose accel motion exceeds [motionGate] (in g of deviation -/// from 1 g rest) are dropped before fitting (vasomotor confound). [epochMin] -/// the binning interval in minutes for the nonparametric statistics +/// [samples] consecutive AdcSamples of the relative skin-temp channel, in +/// whatever unit [deviceFamily] reports it in. [accel] OPTIONAL co-sampled +/// accel for activity de-masking; when supplied, epochs whose accel motion +/// exceeds the family's gate (in g of deviation from 1 g rest) are dropped +/// before fitting (vasomotor confound). [motionGate] overrides that gate for +/// tests and tuning only — production passes the family and nothing else. +/// [epochMin] the binning interval in minutes for the nonparametric statistics /// (van Someren standard is hourly; we allow finer). /// -/// Returns a RELATIVE-tier metric (phase only). Null/absent if too little data. +/// [deviceFamily] is the ingest-stamped id. NULL — every pre-schema-41 row, +/// every import, every raw replay — REFUSES, because a gen4 ADC count and a +/// gen5 centi-°C in the same array cannot be told apart afterwards. +/// +/// Returns a RELATIVE-tier metric (phase only). Absent if too little data. Metric tempCircadian( List samples, { + required String? deviceFamily, List? accel, - double motionGate = 0.08, + double? motionGate, int epochMin = 60, }) { - const inputs = ['skin_temp_adc']; + const inputs = ['skin_temp_adc', 'device_family']; + final cal = calibrationFor(_tempCal, deviceFamily); + if (cal == null) { + return Metric.absent( + tier: Tier.relative, + inputs_used: inputs, + note: unknownFamilyNote(deviceFamily), + ); + } + final gate = motionGate ?? cal.motionGate; // De-mask: drop invalid + high-motion epochs. final ts = []; final adc = []; @@ -98,7 +255,7 @@ Metric tempCircadian( final a = accel[i]; if (a.valid) { final mag = math.sqrt(a.x * a.x + a.y * a.y + a.z * a.z); - if ((mag - 1.0).abs() > motionGate) { + if ((mag - 1.0).abs() > gate) { deMasked++; continue; } @@ -132,17 +289,17 @@ Metric tempCircadian( } // Confidence: tie to cosinor R² when present, else a modest np-only value. - final conf = cos.value != null - ? clamp(cos.value!.r2, 0.1, 0.9) - : 0.3; + final conf = cos.value != null ? clamp(cos.value!.r2, 0.1, 0.9) : 0.3; return Metric( - value: TempCircadian(cos.value, np), + value: TempCircadian(cos.value, np, cal.unit), confidence: conf, tier: Tier.relative, inputs_used: accel == null ? inputs : [...inputs, 'accel'], - note: 'RELATIVE skin-temp phase only (no °C/fever). Wrist temp is ANTIPHASE ' - 'to core; activity-demasked epochs dropped=$deMasked. ' - 'M10/L5 are warmest/coolest windows.', + note: 'RELATIVE skin-temp phase only (no °C/fever/core). Wrist temp is ' + 'ANTIPHASE to core; activity-demasked epochs dropped=$deMasked ' + '(gate=${gate}g). Amplitude is in ${cal.unit} — never compare it ' + 'across device families. M10/L5/RA are WITHHELD: the series is ' + 'median-centred, so RA divides by a quantity that crosses zero.', ); } @@ -175,7 +332,10 @@ CircadianNonparam? _nonparam( for (var b = minBin; b <= maxBin; b++) { series.add(counts.containsKey(b) ? sums[b]! / counts[b]! : null); } - final present = [for (final v in series) if (v != null) v]; + final present = [ + for (final v in series) + if (v != null) v + ]; if (present.length < epochsPerDay) return null; // <1 day of epochs final nDays = (series.length / epochsPerDay).ceil(); @@ -197,9 +357,7 @@ CircadianNonparam? _nonparam( varTot += d * d; } final p = present.length; - final iv = (diffN > 0 && varTot > 0) - ? (diffSq / diffN) / (varTot / p) - : 0.0; + final iv = (diffN > 0 && varTot > 0) ? (diffSq / diffN) / (varTot / p) : 0.0; // IS: between-day stability. Average each within-day epoch-of-day across days, // then variance-of-the-24h-profile / total variance. @@ -219,56 +377,26 @@ CircadianNonparam? _nonparam( } var profVar = 0.0; if (profile.isNotEmpty) { - final pm = mean(profile)!; + // van Someren's IS takes BOTH variances about the GRAND mean. Using the + // profile's own mean here (which it did until 2026-08-17) minimises profVar + // by construction, so IS came out biased low whenever epoch-of-day coverage + // was unbalanced across days — the normal case (per-day bin coverage on real + // gen4 data runs 14/24 to 24/24). for (final v in profile) { - profVar += (v - pm) * (v - pm); + profVar += (v - grand) * (v - grand); } profVar /= profile.length; } final is_ = varTot > 0 ? clamp(profVar / (varTot / p), 0, 1) : 0.0; - // M10 / L5 on the averaged 24-h profile (warmest 10 h, coolest 5 h windows). - // Use the epoch-of-day profile, circularly. epochs in 10h / 5h windows: - final per = profile.length == epochsPerDay ? profile : null; - double m10 = grand, l5 = grand, m10On = 0, l5On = 0; - if (per != null) { - final w10 = (epochsPerDay * 10 / 24).round(); - final w5 = (epochsPerDay * 5 / 24).round(); - var bestHi = double.negativeInfinity, bestLo = double.infinity; - for (var start = 0; start < epochsPerDay; start++) { - var s10 = 0.0; - for (var k = 0; k < w10; k++) { - s10 += per[(start + k) % epochsPerDay]; - } - final m = s10 / w10; - if (m > bestHi) { - bestHi = m; - m10 = m; - m10On = start * (24.0 / epochsPerDay); - } - var s5 = 0.0; - for (var k = 0; k < w5; k++) { - s5 += per[(start + k) % epochsPerDay]; - } - final ml = s5 / w5; - if (ml < bestLo) { - bestLo = ml; - l5 = ml; - l5On = start * (24.0 / epochsPerDay); - } - } - } - final denom = m10 + l5; - final ra = denom != 0 ? (m10 - l5) / denom : 0.0; + // M10 / L5 / RA are DELIBERATELY NOT COMPUTED. See the file header: the temp + // series that survives retention is median-centred, so it is signed, and + // RA's (M10+L5) denominator crosses zero. Restoring them needs an UNCENTRED + // multi-day series first, not a null guard here. return CircadianNonparam( interdailyStability: is_, intradailyVariability: iv, - m10: m10, - m10OnsetHour: m10On, - l5: l5, - l5OnsetHour: l5On, - relativeAmplitude: ra, epochsPerDay: epochsPerDay, nDays: nDays, ); diff --git a/lib/src/onehz/wellness/temp_health.dart b/lib/src/onehz/wellness/temp_health.dart index b2272d1..d3cae0e 100644 --- a/lib/src/onehz/wellness/temp_health.dart +++ b/lib/src/onehz/wellness/temp_health.dart @@ -1,7 +1,11 @@ // WELLNESS — relative skin-temp health signals (illness flag + menstrual). // -// Both operate on the NIGHTLY-MEAN RELATIVE skin-temp (z-scored or raw ADC). -// NEVER absolute °C / fever. +// Both operate on the RAW NIGHTLY-MEAN skin-temp in the device's own unit — +// NOT a z-score. Each standardises once, internally, against its own window; +// handing them an already-standardised series standardises it twice and the +// thresholds below stop meaning what they say. `nightlySkinTemp` (temp_circadian +// .dart) is the canonical producer and also gates out nights the strap spent +// warming up. NEVER absolute °C / fever. // // 1. Skin-temp z-score illness flag (Smarr 2020) — nightly relative-temp z vs a // trailing personal baseline; flags a sustained elevation. MUST be @@ -52,10 +56,22 @@ class TempIllnessDay { /// Nightly relative skin-temp z-score illness flag, cycle-aware. /// -/// [dates] labels; [nightlyTemp] nightly-mean RELATIVE temp ADC (null = missing -/// night); [luteal] OPTIONAL per-night luteal-phase flag (true => suppress). -/// [baselineDays] trailing robust-baseline window; [zThresh] flag elevation -/// above this robust z; [persistDays] consecutive elevated nights required. +/// [dates] labels; [nightlyTemp] the RAW nightly-mean skin-temp in the device's +/// own unit (gen4 ADC counts, gen5 centi-°C) — `nightlySkinTemp(...).value.mean` +/// is the canonical producer. Null = missing night. [luteal] OPTIONAL per-night +/// luteal-phase flag (true => suppress). [baselineDays] trailing robust-baseline +/// window; [zThresh] flag elevation above this robust z; [persistDays] +/// consecutive elevated nights required. +/// +/// DO NOT PASS A Z-SCORE. This function standardises internally against its own +/// trailing window (`robustBaseline(...).modZ`), so a pre-standardised series +/// gets standardised twice and `zThresh = 2.0` stops meaning anything: an +/// already-z-scored value whose own denominator came from as few as 3 nights +/// carries ~50 % relative error before this ever sees it. The function cannot +/// detect the unit — the contract is the only guard there is, which is why it +/// is written here in capitals rather than left in a header line. (Smarr et al., +/// Sci Rep 2020;10:21640 reports +0.63 °C against a per-subject baseline and +/// supplies NO z threshold; 2.0 is ours, on an uncalibrated relative channel.) /// /// HONESTY: when MAD baseline is degenerate (quantized/flat) we report null z /// and stay `normal` — we never invent a deviation. In luteal nights an @@ -72,13 +88,16 @@ List tempIllnessFlag( }) { final n = nightlyTemp.length; final out = []; + // CALENDAR days, not rows — see [calendarDays]. + final day = calendarDays(dates); var elevatedRun = 0; + var lastScoredDay = -1 << 20; for (var i = 0; i < n; i++) { final t = nightlyTemp[i]; final lut = (luteal != null && i < luteal.length) ? luteal[i] : false; - final lo = i - baselineDays < 0 ? 0 : i - baselineDays; final window = []; - for (var j = lo; j < i; j++) { + for (var j = i - 1; j >= 0; j--) { + if (day[i] - day[j] > baselineDays) break; final v = nightlyTemp[j]; if (v != null) window.add(v); } @@ -101,6 +120,9 @@ List tempIllnessFlag( elevatedRun = 0; continue; } + // "N nights running" means CONSECUTIVE NIGHTS, not consecutive rows. + if (day[i] - lastScoredDay > 1) elevatedRun = 0; + lastScoredDay = day[i]; final elevated = zz >= zThresh; if (elevated) { elevatedRun++; @@ -149,19 +171,43 @@ class OvulationEvent { /// /// For each candidate night i: the coverline = max of the prior [lookback] /// nights; if night i and the next ([confirm]-1) nights all exceed -/// (coverline + [threshold]) ADC counts, ovulation is confirmed, estimated at -/// the night just before the rise (i-1). +/// (coverline + [threshold]), ovulation is confirmed, estimated at the night +/// just before the rise (i-1). +/// +/// UNITS: [threshold] is in WHATEVER UNIT [nightlyTemp] carries — this function +/// is unit-agnostic and cannot check, so there is NO default: it used to be 1.0 +/// (documented as ADC counts) while the caller passed z-scores, which silently +/// turned the classic ~0.2 °F rule into "one whole SD above the prior 6 nights" +/// and confirmed ovulation essentially never. Pass the threshold in the unit of +/// the series you hand in, and set [inputLabel] so the envelope names it. /// /// HONESTY: confirmation ONLY — this NEVER predicts a future ovulation. Returns -/// all detected events over the series. `relative` tier (ADC counts, no °C). +/// all detected events over the series. `relative` tier — never an absolute +/// temperature. +/// +/// UNCALLED TODAY, AND THE CONDITION FOR CALLING IT IS A THRESHOLD. edge +/// unwired it after passing z-scores against a 1.0 default (see the note at the +/// old call site in local_repository_impl.dart). restoring it needs a threshold +/// defensible IN THE UNIT PASSED, ON THIS SENSOR, with a stated basis — not a +/// number chosen because it makes events appear. nobody has one yet. +/// +/// AN EMPTY EVENT LIST IS NOT AN ANSWER ABOUT HER BODY. the temperature channel +/// is populated at ~1.5% of a sleep window on real data, so a null coverline +/// result is dominated by measurement failure, not physiology. the app may make +/// a statement about its own detector and never about her ovulation. the word +/// "anovulatory", and every paraphrase of it, stays out of this codebase and +/// out of the copy. whatever renders this either says nothing in the no-event +/// state or states the coverage it had IN THE SAME SENTENCE, with no +/// cycle-level interpretation — not in a footnote, not on a second screen. Metric> menstrualCoverline( List dates, List nightlyTemp, { + required double threshold, int lookback = 6, int confirm = 3, - double threshold = 1.0, + String inputLabel = 'nightly_skin_temp', }) { - const inputs = ['nightly_skin_temp_adc']; + final inputs = [inputLabel]; final n = nightlyTemp.length; if (n < lookback + confirm) { return Metric>.absent( diff --git a/lib/src/onehz/wellness/wellness.dart b/lib/src/onehz/wellness/wellness.dart index 79631ed..95590f9 100644 --- a/lib/src/onehz/wellness/wellness.dart +++ b/lib/src/onehz/wellness/wellness.dart @@ -2,7 +2,12 @@ // // Built on the foundations (types/util/baseline/fusion) and clinical (cosinor, // illness_cusum) layers. MED-tier, RELATIVE-honest methods: -// - tempCircadian : relative skin-temp cosinor + IS/IV/RA/L5/M10 +// - nightlySkinTemp : the nightly skin-temp mean over the SETTLED part +// of the window + the settled fraction. THE producer +// every temp consumer should read; a warming strap +// is not a cold person. PER-FAMILY. +// - tempCircadian : relative skin-temp cosinor + IS/IV, PER-FAMILY +// (gen4 ADC vs gen5 centi-°C); M10/L5/RA withheld // - tempIllnessFlag : Smarr nightly relative-temp z, CYCLE-AWARE // - menstrualCoverline : 3-over-6 retrospective ovulation CONFIRMATION // - multivariateAnomaly : robust Mahalanobis {RHR↑,HRV↓,temp↑,resp↑} + gates @@ -22,9 +27,12 @@ library wellness; // depend on just this barrel. export '../types.dart'; export '../util.dart'; +// tempCircadian dispatches on device family, so its callers need the seam. +export '../device.dart'; export 'temp_circadian.dart'; export 'temp_health.dart'; +export 'cycle_lengths.dart'; export 'anomaly.dart'; export 'changepoint.dart'; export 'readiness_composite.dart'; diff --git a/lib/src/onehz/workout/auto_detect.dart b/lib/src/onehz/workout/auto_detect.dart index b7cc441..3ad37a8 100644 --- a/lib/src/onehz/workout/auto_detect.dart +++ b/lib/src/onehz/workout/auto_detect.dart @@ -1,13 +1,15 @@ // auto_detect.dart — opt-in "did you just work out?" SUGGESTION detector. // // Opt-in workout suggestion detector (ported from AutoWorkoutDetector.swift). -// DELIBERATELY conservative (low sensitivity): a sustained ≥12-min -// elevation of HR ≥ resting+30 bpm, brief (≤90 s) dips tolerated, near windows -// merged, optional motion confirmation, overlap-excluded against saved spans. +// DELIBERATELY conservative (low sensitivity): a sustained ≥12-min elevation of +// HR ≥ max(RHR+[elevatedMarginBPM], RHR+[hrrFloorFraction]·HRR) — RHR+59 at +// RHR 60 / HRmax 190 — brief (≤90 s) dips tolerated, near windows merged, +// optional motion confirmation, overlap-excluded against saved spans. +// (The header and the note used to quote "RHR+30", the margin this detector +// deliberately abandoned: see [hrrFloorFraction] for the 30 false windows on a +// sedentary week that motivated replacing it.) // -// This NEVER writes a row — it only ever SUGGESTS. It is separate from the -// persistent per-day [WorkoutDetector] (workout_detect.dart), which computes -// calories/zones/strain for the durable "detected" rows. +// This NEVER writes a row — it only ever SUGGESTS. // // HYBRID SEAM: every surviving bout is run through a [SportClassifier]; the // default returns "detected" (no sport typing). OpenStrap's motion-based HAR typer @@ -102,6 +104,16 @@ class AutoWorkoutDetector { /// A dip below the gate no longer than this does NOT break the span. static const int maxDipS = 90; + /// A GAP IN THE DATA longer than this breaks the span. + /// + /// A dip is "HR present and low"; a gap is "no HR at all", and only the dip + /// used to be time-bounded — so an off-wrist stretch produced no samples, the + /// loop never saw it, and the span simply resumed at the next elevated + /// sample. Two 12-minute efforts 40 minutes apart came out as one 64-minute + /// workout whose mean HR (a mean over present samples only) still read + /// normal. Absence of data is not evidence of continuity. + static const int maxGapS = 90; + /// Two windows whose gap is strictly < this are merged (5 min). static const int mergeGapS = 5 * 60; @@ -219,6 +231,8 @@ class AutoWorkoutDetector { } for (var k = 0; k < n; k++) { + // Break on a hole in the record before anything else — see [maxGapS]. + if (k > 0 && spanStart != null && ts[k] - ts[k - 1] > maxGapS) closeSpan(); if (bpm[k] >= floor) { spanStart ??= ts[k]; spanEnd = ts[k]; @@ -382,7 +396,9 @@ Metric> autoDetectWorkouts({ confidence: list.isEmpty ? 0.0 : 0.6, tier: Tier.estimate, inputs_used: const ['hr_1hz', 'resting_hr', 'motion_1hz'], - note: 'opt-in workout suggestion (HR ≥ RHR+30 sustained ≥12 min); ' + note: 'opt-in workout suggestion (HR ≥ max(RHR+' + '${AutoWorkoutDetector.elevatedMarginBPM}, RHR+' + '${AutoWorkoutDetector.hrrFloorFraction}·HRR) sustained ≥12 min); ' 'wrist-HR ESTIMATE, not medical advice', ); } diff --git a/lib/src/onehz/workout/calories.dart b/lib/src/onehz/workout/calories.dart index b5855b7..e9bd746 100644 --- a/lib/src/onehz/workout/calories.dart +++ b/lib/src/onehz/workout/calories.dart @@ -1,8 +1,10 @@ // Per-bout calorie estimation (calories.py / WorkoutDetector.swift). // // HR-based: Keytel et al. 2005 active EE (kJ/min from HR, weight, age, sex) + -// revised Harris–Benedict BMR for the resting floor. Sex-specific coefficients -// (male / female / nonbinary). APPROXIMATE — not laboratory calorimetry, not +// revised Harris–Benedict BMR for the resting floor. Keytel publishes TWO +// coefficient sets, male and female; the third set here (`nonbinary`) is OUR +// element-wise INTERPOLATION of those two, not a third published equation — +// see [Calories.nonbinary]. APPROXIMATE — not laboratory calorimetry, not // medical advice. // // Faithfulness note: the coefficients, the 86_400 BMR/s divisor, the 251.04 @@ -73,6 +75,17 @@ class Calories { workoutAge: 0.0740, workoutAlpha: -20.4022, ); + /// INTERPOLATED, NOT PUBLISHED (audit MOT-10). Keytel 2005 and revised + /// Harris–Benedict each publish exactly two sex sets; every constant below is + /// the element-wise ARITHMETIC MEAN of [male] and [female] + /// ((88.362+447.593)/2 = 267.9775, (0.1988−0.1263)/2 = 0.03625, …). The + /// weight term is the one to be careful with: male +0.1988 and female −0.1263 + /// have OPPOSITE SIGNS, so the mean 0.03625 kJ/min/kg belongs to neither + /// regression and is not a fitted quantity at all. It is used deliberately — + /// `workoutSex` routes null/'other'/unrecognised here by design, and the mean + /// of the two is a smaller lie than guessing one of them — but nothing about + /// it is Keytel's. Same convention, same wording, as [mifflinBmrKcalDay]'s + /// nonbinary constant. static const CalorieCoeffs nonbinary = CalorieCoeffs( restingAlpha: 267.9775, restingWeight: 11.322, @@ -88,6 +101,35 @@ class Calories { /// resting + this fraction of HRR, else the resting BMR rate. static const double activeHRRFraction = 0.30; + /// THE HR-FLEX POINT for [dailyEnergy], as a fraction of HRmax. + /// + /// RAISED 0.50 → 0.65 on 2026-08-17 (audit MOT-02). This number decides which + /// waking minutes get billed the Keytel rate at all, and 0.50 put it at + /// 0.50 × 187 = 93.5 bpm for a 30 y/o — a heart rate a person reaches by + /// standing up. Keytel's regression is fitted on STEADY-STATE SUBMAXIMAL + /// EXERCISE with chest/ECG HR; below the exercise domain there is no HR↔EE + /// slope to extrapolate along (that absence is the entire premise of the + /// HR-FLEX method this function cites — Spurr 1988 / Ceesay 1989 assign + /// resting EE below flex and an INDIVIDUALLY CALIBRATED line above it). Ours + /// is a population line, so the gate has to sit where that line is at least + /// approximately true. 0.65 × HRmax is the ACSM moderate-intensity floor + /// (64 % HRmax ≈ 40 % HRR) — the published boundary below which movement is + /// not counted as exercise at all, and the region Keytel's submaximal + /// exercise protocol does not sample. It is a choice, not a number Keytel + /// prints; what Keytel prints is a regression with no resting arm. + /// + /// MEASURED CONSEQUENCE, whoop-4.db, 9 days, 70 kg/170 cm/30 y male stand-in, + /// Tanaka HRmax 187: billed minutes 39.4 % → 4.9 % of the wake window; daily + /// ACTIVE energy min/median/max 769/1,955/4,062 → 9/48/1,917 kcal; daily + /// TOTAL 1,544–5,582 → 793–3,437 kcal. Quiet days lose essentially all their + /// "active" energy, which is the correction: they never earned it. Real + /// sessions keep most of theirs (a 103-min-above-50 %-HRR day: 3,462 → 1,202). + /// + /// This is still a point estimate with no error band, deliberately: Keytel + /// publishes a POPULATION MAPE, which is not this user's error distribution, + /// and drawing bounds around the number would claim we computed them. + static const double defaultActiveFraction = 0.65; + /// 60 s/min × 4.184 kJ/kcal. static const double workoutDivisor = 251.04; @@ -171,33 +213,42 @@ class Calories { /// filling any minute that has no HR sample. /// /// [hrPerMin] is per-minute mean HR (bpm); 0/absent minutes fall back to BMR. - /// [activeFraction] is the HR-flex point as a fraction of HRmax (default 0.50, - /// matching the edge pipeline): minutes below it burn BMR only, so a quiet day - /// reads ≈ basal and Keytel's low-HR over-estimate can't inflate "active". + /// [activeFraction] is the HR-flex point as a fraction of HRmax + /// ([defaultActiveFraction] = 0.65 — read its doc before moving it, it is the + /// single number that decides what "active" means): minutes below it burn BMR + /// only, so a quiet day reads ≈ basal and Keytel's low-HR extrapolation can't + /// inflate "active". /// [dayMinutes] lets a partial day pro-rate basal (default 1440 = full day). - static ({double total, double active, double basal, bool usedDefaultHrmax}) - dailyEnergy( + /// + /// [hrmax] IS REQUIRED, and there is deliberately no fallback for it. It used + /// to default to `220 − age` here, which is a universal formula — a ceiling + /// invented in this package, applied to whatever strap measured the HR, and + /// then used as the flex gate that decides which minutes count as active at + /// all. Whether a wrist can be banded on that number is a property of the + /// sensor package, so the ceiling is DISPATCHED PER DEVICE FAMILY by the + /// caller (edge: `hr_max.dart`'s `estimatedMaxHr(age, deviceFamily)`, the one + /// definition all zone/TRIMP/calorie call sites route through). An unknown + /// age or an uncalibrated strap yields no ceiling there, and a caller with no + /// ceiling has no energy figure: it abstains rather than call this. That is + /// why the old `usedDefaultHrmax` flag is gone — the case it flagged can no + /// longer reach this function. + static ({double total, double active, double basal}) dailyEnergy( List hrPerMin, { required WorkoutUserProfile profile, - double? hrmax, - double activeFraction = 0.50, + required double hrmax, + double activeFraction = defaultActiveFraction, int dayMinutes = 1440, }) { - // the 220-age hrmax fallback used to just silently apply with nothing - // telling the caller it wasnt a real anchor. usedDefaultHrmax lets the - // UI caveat the number instead of showing it as if it were solid. - // (note: this can only catch hrmax - WorkoutUserProfile's own - // constructor already defaults weight/height/age to 70/170/30, so by - // the time a profile object gets here there's no way left to tell "the - // caller passed 70kg" from "nobody set a weight so it's 70kg". if that - // ever needs catching too, WorkoutUserProfile's fields need to become - // nullable at the source - bigger change, not doing it here.) + // (weight/height/age still can't be told apart from their defaults here: + // WorkoutUserProfile's own constructor bakes 70/170/30 in at construction, + // so by the time a profile object arrives there is no way left to tell "the + // caller passed 70kg" from "nobody set a weight". Making those three + // nullable at the source is the real fix and a bigger change.) final weightKg = profile.weightKg > 0 ? profile.weightKg : 70.0; final heightCm = profile.heightCm > 0 ? profile.heightCm : 170.0; final age = profile.age > 0 ? profile.age : 30.0; final coeffs = resolveCoeffs(profile.sex); - final effHRmax = hrmax ?? (220.0 - age); - final flexHr = activeFraction * effHRmax; + final flexHr = activeFraction * hrmax; final bmrDay = mifflinBmrKcalDay(weightKg, heightCm, age, profile.sex); final basalPerMin = bmrDay / 1440.0; @@ -206,17 +257,12 @@ class Calories { for (final hr in hrPerMin) { if (hr < flexHr) continue; // below flex point → basal only final activePerMin = - activeKcalPerS(coeffs, hr, effHRmax, weightKg, age) * 60.0; + activeKcalPerS(coeffs, hr, hrmax, weightKg, age) * 60.0; final surplus = activePerMin - basalPerMin; if (surplus > 0) active += surplus; } final basal = basalPerMin * dayMinutes; - return ( - total: basal + active, - active: active, - basal: basal, - usedDefaultHrmax: hrmax == null, - ); + return (total: basal + active, active: active, basal: basal); } /// Estimate (kcal, kJ) for a workout bout. Each sample is weighted by the @@ -228,7 +274,8 @@ class Calories { /// length). [hrmax]/[restingHr] anchors (null → 220 / 60 fallback, flagged /// via [usedDefaultAnchors] on the result so a fabricated-anchor calorie /// number can be caveated instead of shown as if it were real). - static ({double kcal, double kj, bool usedDefaultAnchors}) estimateBoutCalories( + static ({double kcal, double kj, bool usedDefaultAnchors}) + estimateBoutCalories( List hrTsSec, List hrBpm, { required WorkoutUserProfile profile, @@ -272,8 +319,7 @@ class Calories { if (b < activeThreshold) { totalKcal += restingRate * dur; } else { - totalKcal += - activeKcalPerS(coeffs, b, effHRmax, weightKg, age) * dur; + totalKcal += activeKcalPerS(coeffs, b, effHRmax, weightKg, age) * dur; } } return ( diff --git a/lib/src/onehz/workout/hr_recovery.dart b/lib/src/onehz/workout/hr_recovery.dart index ef12494..7fbf3d9 100644 --- a/lib/src/onehz/workout/hr_recovery.dart +++ b/lib/src/onehz/workout/hr_recovery.dart @@ -10,6 +10,8 @@ // If the wearer kept moving (HR stayed high) or the signal is missing, we return // absent rather than a fabricated drop. Tier ESTIMATE (wrist pulse, not ECG). +import 'dart:math' as math; + import '../types.dart'; import '../util.dart'; @@ -18,20 +20,135 @@ class HrRecovery { final double hrAt60s; // bpm 60 s later final double dropBpm; // peakHr - hrAt60s (≥0) final double dropPct; // drop as % of peak + + /// Time constant (s) of a single exponential fitted to the 0–[tauWindowSec] + /// recovery tail: `hr(t) = asymptote + amp·exp(−t/tau)`. Null whenever the + /// fit did not clear its gates — see [tauAbsenceReason], which is the usual + /// outcome and is meant to be. + /// + /// IT PUBLISHES ALONGSIDE [dropBpm], NEVER INSTEAD OF IT. And it is NOT + /// intensity-invariant: HR recovery is biphasic and the fast-phase constant + /// is itself intensity-dependent, so one exponential over 0–180 s conflates + /// the two phases. Compare a tau only against other taus that started from a + /// similar [tauStartHr] — which is why that number ships with it and is not + /// optional on any surface that shows tau. + final double? tauSec; + + /// Fitted HR at t = 0 (`asymptote + amp`), i.e. the intensity this recovery + /// started from. Null when [tauSec] is. + final double? tauStartHr; + + /// Fitted amplitude (bpm) — how far the exponential falls in total. + final double? tauAmpBpm; + + /// Fitted asymptote (bpm) the curve settles toward. + final double? tauAsymptoteBpm; + + /// Fit RMSE as a FRACTION of [tauAmpBpm]. Dimensionless on purpose: a bpm + /// threshold would be a per-strap constant with nothing behind it, while + /// "the residual is under 15 % of the swing being fitted" means the same + /// thing on any sensor. Null when [tauSec] is. + final double? tauResidualRatio; + + /// Machine-readable reason the tau fit abstained, or null when it did not. + final String? tauAbsenceReason; const HrRecovery({ required this.peakHr, required this.hrAt60s, required this.dropBpm, required this.dropPct, + this.tauSec, + this.tauStartHr, + this.tauAmpBpm, + this.tauAsymptoteBpm, + this.tauResidualRatio, + this.tauAbsenceReason, }); Map toJson() => { 'peak_hr': round6(peakHr), 'hr_at_60s': round6(hrAt60s), 'drop_bpm': round6(dropBpm), 'drop_pct': round6(dropPct), + 'tau_sec': tauSec == null ? null : round6(tauSec!), + 'tau_start_hr': tauStartHr == null ? null : round6(tauStartHr!), + 'tau_amp_bpm': tauAmpBpm == null ? null : round6(tauAmpBpm!), + 'tau_asymptote_bpm': + tauAsymptoteBpm == null ? null : round6(tauAsymptoteBpm!), + 'tau_residual_ratio': + tauResidualRatio == null ? null : round6(tauResidualRatio!), + if (tauAbsenceReason != null) 'tau_absence_reason': tauAbsenceReason, }; } +/// Grid-search a single-exponential recovery `hr(t) = a + b·exp(−t/tau)` over +/// the post-exercise tail, and REFUSE unless the fit is clean. +/// +/// [tSec] seconds since exercise end (0 at end, ascending), [hr] bpm, both +/// already filtered to valid samples. For each candidate tau the basis +/// `exp(−t/tau)` is fixed, so a and b fall out of a two-parameter least +/// squares in closed form — no iteration, no optimiser, no dependency. +/// +/// The gates exist because most bouts will not survive them, which is the +/// correct outcome: a walk-home cooldown, a stop that was not really a stop, +/// or a sparse tail all produce a number this fit would happily print. +({ + double tau, + double amp, + double asymptote, + double residualRatio, +})? _fitTau( + List tSec, + List hr, { + required double minTau, + required double maxTau, + required double maxResidualRatio, +}) { + final m = tSec.length; + if (m < 30) return null; + double? bestSse; + double bestTau = 0, bestA = 0, bestB = 0; + for (var tau = minTau; tau <= maxTau; tau += 1.0) { + var se = 0.0, see = 0.0, sy = 0.0, sey = 0.0; + for (var i = 0; i < m; i++) { + final e = math.exp(-tSec[i] / tau); + se += e; + see += e * e; + sy += hr[i]; + sey += e * hr[i]; + } + final den = m * see - se * se; + if (den.abs() < 1e-12) continue; + final b = (m * sey - se * sy) / den; + final a = (sy - b * se) / m; + var sse = 0.0; + for (var i = 0; i < m; i++) { + final r = hr[i] - (a + b * math.exp(-tSec[i] / tau)); + sse += r * r; + } + if (bestSse == null || sse < bestSse) { + bestSse = sse; + bestTau = tau; + bestA = a; + bestB = b; + } + } + if (bestSse == null) return null; + // Must actually decay. A negative amplitude is HR still climbing — a stop + // that was not a stop. + if (bestB <= 0) return null; + // A tau pinned to either end of the grid is not an estimate, it is the grid + // boundary: the tail did not constrain it. + if (bestTau <= minTau || bestTau >= maxTau) return null; + final ratio = math.sqrt(bestSse / m) / bestB; + if (!ratio.isFinite || ratio > maxResidualRatio) return null; + return ( + tau: bestTau, + amp: bestB, + asymptote: bestA, + residualRatio: ratio, + ); +} + /// Heart-rate recovery from a per-second HR tail bracketing the end of a bout. /// /// [hrTailBpm] is a contiguous per-second HR series (bpm; 0 = off-skin) covering @@ -41,11 +158,36 @@ class HrRecovery { /// exercise end. Peak HR is the max over the [peakWindowSec] before end; recovery /// HR is the median of a small window around end+[recoverySec] (robust to a single /// spike). Returns absent if the tail is too short, off-skin, or doesn't descend. +/// +/// [tsSec] OPTIONAL unix-second timestamps parallel to [hrTailBpm]. SUPPLY THEM. +/// Without them EVERY window here is really ARRAY POSITIONS: the substrate this +/// is sliced from is one row per decoded record, not a dense 1 Hz grid, so on a +/// gappy tail the "HR at +60 s" sample can be many minutes post-exercise and +/// `hrr_bpm` is overstated — read by the user as a fitness marker. With [tsSec] +/// ALL THREE windows are resolved on the clock: the peak window before end, the +/// recovery point, and the ±3 s median around it. Supplying [tsSec] used to +/// convert only the recovery point, leaving the peak window positional — a +/// 30-position window on a 20 s-spaced tail spanned 580 s of clock, so "peak at +/// exercise end" came from nine minutes earlier and inflated the drop by 28 bpm +/// while ALSO collecting the +0.2 timestamped-confidence bonus. +/// +/// [maxGapSec] bounds a hole in either direction. A gap inside the peak window +/// is not fatal — the window simply stops there — but it does forfeit the +/// timestamped bonus, because the peak is then measured over less clock time +/// than was asked for. +/// [tauWindowSec] is the tail the exponential is fitted over (CV-08). The +/// caller has to actually SLICE that much substrate — with a −30/+75 s tail +/// there is nothing to fit and tau abstains with a reason, which is honest but +/// useless. Metric hrRecovery( List hrTailBpm, { int? endIndex, int recoverySec = 60, int peakWindowSec = 30, + List? tsSec, + int maxGapSec = 30, + int tauWindowSec = 180, + double tauMaxResidualRatio = 0.15, }) { const inputs = ['hr_1hz']; final end = endIndex ?? 0; @@ -56,24 +198,84 @@ Metric hrRecovery( note: 'no HR tail for HRR', ); } + final times = + (tsSec != null && tsSec.length == hrTailBpm.length) ? tsSec : null; // Peak HR over the window ending at exercise end (valid samples only). - final peakLo = (end - peakWindowSec).clamp(0, hrTailBpm.length - 1); + // With timestamps the window is peakWindowSec of CLOCK TIME walked backwards + // from end, stopping at a hole; without them it is peakWindowSec positions, + // which is all the caller has given us to go on. + int peakLo; + var peakWindowFull = true; + if (times != null) { + var i = end; + while (i > 0 && + times[end] - times[i - 1] <= peakWindowSec && + times[i] - times[i - 1] <= maxGapSec) { + i--; + } + peakLo = i; + // "Full" = the window covers the requested clock span, OR it was cut short + // by the start of the caller's slice rather than by the DATA. Stopping at + // peakLo > 0 with a short span means the samples themselves are too far + // apart (or a hole intervened) to resolve the window — that is the case + // that forfeits the timestamped confidence bonus below. + peakWindowFull = peakLo == 0 || times[end] - times[peakLo] >= peakWindowSec; + } else { + peakLo = (end - peakWindowSec).clamp(0, hrTailBpm.length - 1); + } double peak = 0; for (var i = peakLo; i <= end; i++) { final h = hrTailBpm[i]; if (h > peak) peak = h.toDouble(); } // Recovery HR: median of a ±3 s window around end + recoverySec. - final target = end + recoverySec; - final lo = (target - 3).clamp(0, hrTailBpm.length - 1); - final hi = (target + 3).clamp(0, hrTailBpm.length - 1); - if (hi >= hrTailBpm.length || target >= hrTailBpm.length) { - return const Metric.absent( + int target; + if (times != null) { + // Locate the recovery point on the CLOCK, and refuse if the tail has a hole + // in it between exercise end and there. + final wantTs = times[end] + recoverySec; + var t = -1; + for (var i = end; i < times.length; i++) { + if (i > end && times[i] - times[i - 1] > maxGapSec) break; + if (times[i] >= wantTs) { + t = i; + break; + } + } + if (t < 0) { + return Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'HR tail does not reach +${recoverySec}s without a gap — no HRR', + ); + } + target = t; + } else { + target = end + recoverySec; + } + if (target >= hrTailBpm.length) { + return Metric.absent( tier: Tier.estimate, inputs_used: inputs, - note: 'HR tail too short to reach +${60}s recovery point', + note: 'HR tail too short to reach +${recoverySec}s recovery point', ); } + // The ±3 s median window, on the clock when we have one. Positionally it was + // ±3 ROWS, which on a sparse tail averages HR from minutes apart. + int lo, hi; + if (times != null) { + lo = target; + while (lo > 0 && times[target] - times[lo - 1] <= 3) { + lo--; + } + hi = target; + while (hi + 1 < times.length && times[hi + 1] - times[target] <= 3) { + hi++; + } + } else { + lo = (target - 3).clamp(0, hrTailBpm.length - 1); + hi = (target + 3).clamp(0, hrTailBpm.length - 1); + } final recWin = [ for (var i = lo; i <= hi; i++) if (hrTailBpm[i] > 0) hrTailBpm[i].toDouble() @@ -95,14 +297,77 @@ Metric hrRecovery( ); } final pct = peak > 0 ? 100.0 * drop / peak : 0.0; - // Confidence: a clearly descending tail from a high peak is more trustworthy. - final conf = clamp(drop / 30.0, 0.3, 0.9); + // Confidence from DATA QUALITY, not from the answer. This used to be + // `clamp(drop / 30, 0.3, 0.9)` — confidence derived from the metric's own + // magnitude, so a motion-artifact spike inflated `peak`, inflated `drop`, and + // RAISED the confidence: the least trustworthy readings scored highest. + var seen = 0, valid = 0; + for (var i = peakLo; i <= end; i++) { + seen++; + if (hrTailBpm[i] > 0) valid++; + } + for (var i = lo; i <= hi; i++) { + seen++; + if (hrTailBpm[i] > 0) valid++; + } + final validFrac = seen == 0 ? 0.0 : valid / seen; + // Timestamped tails earn the top of the band; positional ones cannot, because + // we have no way to know the recovery sample is really 60 s later. A + // timestamped tail too sparse to fill the peak window doesn't earn it either + // — the timestamps proved the window was short rather than fixing it. + final conf = clamp( + 0.3 + 0.4 * validFrac + (times != null && peakWindowFull ? 0.2 : 0.0), + 0.2, + 0.9, + ); + + // TAU (CV-08). Same tail, same slice, no second pass over the substrate. + // Timestamps are REQUIRED: a time constant fitted to array positions of + // unknown spacing is a number in seconds that was never measured in seconds. + String? tauWhy; + ({double tau, double amp, double asymptote, double residualRatio})? fit; + if (times == null) { + tauWhy = 'tau_needs_timestamps'; + } else { + final t0 = times[end]; + final tt = []; + final yy = []; + for (var i = end; i < times.length; i++) { + if (i > end && times[i] - times[i - 1] > maxGapSec) break; + final dt = times[i] - t0; + if (dt > tauWindowSec) break; + if (hrTailBpm[i] <= 0) continue; + tt.add(dt.toDouble()); + yy.add(hrTailBpm[i].toDouble()); + } + // Half the window, on the clock, or there is not enough curve to separate + // the decay from the asymptote. + if (tt.isEmpty || tt.last < tauWindowSec / 2) { + tauWhy = 'tau_tail_short:span=${tt.isEmpty ? 0 : tt.last.round()}s'; + } else { + fit = _fitTau( + tt, + yy, + minTau: 5, + maxTau: 200, + maxResidualRatio: tauMaxResidualRatio, + ); + if (fit == null) tauWhy = 'tau_fit_rejected'; + } + } + return Metric( value: HrRecovery( peakHr: peak, hrAt60s: hrAt60, dropBpm: drop, dropPct: pct, + tauSec: fit?.tau, + tauStartHr: fit == null ? null : fit.asymptote + fit.amp, + tauAmpBpm: fit?.amp, + tauAsymptoteBpm: fit?.asymptote, + tauResidualRatio: fit?.residualRatio, + tauAbsenceReason: tauWhy, ), confidence: conf, tier: Tier.estimate, diff --git a/lib/src/onehz/workout/hr_zones.dart b/lib/src/onehz/workout/hr_zones.dart index 5bd2e7c..b6beb78 100644 --- a/lib/src/onehz/workout/hr_zones.dart +++ b/lib/src/onehz/workout/hr_zones.dart @@ -7,8 +7,10 @@ class HeartRateZone { final int number; // 1..5 final double lower; // inclusive bpm final double upper; // exclusive except zone 5 - final double lowerPct; // fraction of HRmax - final double upperPct; // fraction of HRmax + // Fraction of the anchor this zone set was built on: of HRmax for the + // %HRmax sets, of HEART-RATE RESERVE for a Karvonen set (`source` says which). + final double lowerPct; + final double upperPct; const HeartRateZone({ required this.number, @@ -89,7 +91,8 @@ class HeartRateZones { } /// Build zones directly from a known max HR. - static HeartRateZoneSet zonesFromMaxHr(double maxHr, {String source = 'manual'}) { + static HeartRateZoneSet zonesFromMaxHr(double maxHr, + {String source = 'manual'}) { final built = []; for (var i = 0; i < 5; i++) { final loPct = zoneEdges[i]; @@ -105,6 +108,65 @@ class HeartRateZones { return HeartRateZoneSet(zones: built, maxHr: maxHr, source: source); } + /// Minimum trailing daily resting-HR values [reserveZones] will accept. + /// + /// The anchor is the 28-day MEDIAN resting HR. One night is a measurement of + /// that night — a poor night moves every zone boundary in the app, and the + /// user has no way to see why. Half the window is the floor. + static const int reserveMinDays = 14; + + /// Karvonen %HRR zones: the same 50/60/70/80/90 convention, in RESERVE units. + /// + /// `lower = rhr + pct · (maxHr − rhr)`, so a zone is a band between two heart + /// rates the band actually measured instead of one guessed one — and Banister + /// TRIMP already works in reserve units, so strain and zones finally agree. + /// + /// STILL A MODEL. %HRR bands are a convention, not a measurement of your + /// thresholds: never "your aerobic threshold", never "fat burning zone". What + /// changes is that the app can name the two numbers it anchored on. + /// + /// [restingHrHistory] is the trailing daily resting HR (up to 28 days, any + /// order); the median of the valid values is the anchor. It is taken as a + /// series rather than a scalar on purpose: a call site that has only tonight + /// cannot pass tonight and have it read as the baseline. + /// + /// Returns null when there is not enough history, or when [maxHr] is not + /// above the anchor — an inverted reserve is not a zone set, and there is no + /// substitute number to fall back to. + static HeartRateZoneSet? reserveZones({ + required List restingHrHistory, + required double maxHr, + int minDays = reserveMinDays, + String source = 'karvonen', + }) { + final valid = [ + for (final v in restingHrHistory) + if (v.isFinite && v > 0) v + ]; + if (valid.length < minDays) return null; + valid.sort(); + final rhr = valid.length.isOdd + ? valid[valid.length ~/ 2] + : (valid[valid.length ~/ 2 - 1] + valid[valid.length ~/ 2]) / 2.0; + if (!(maxHr > rhr)) return null; + final reserve = maxHr - rhr; + final built = []; + for (var i = 0; i < 5; i++) { + final loPct = zoneEdges[i]; + final hiPct = zoneEdges[i + 1]; + built.add(HeartRateZone( + number: i + 1, + lower: rhr + loPct * reserve, + upper: rhr + hiPct * reserve, + lowerPct: loPct, + upperPct: hiPct, + )); + } + // A very low resting HR widens zone 1 a long way. That is the arithmetic + // working, not a bug, and it will be reported as one. + return HeartRateZoneSet(zones: built, maxHr: maxHr, source: source); + } + /// Time-in-zone from a time-ordered HR stream. /// /// Each sample is credited with the duration until the next sample. The tail @@ -140,7 +202,9 @@ class HeartRateZones { static double _boundedGapSeconds(double gapMs, double fallbackSeconds) { final gapSeconds = gapMs / 1000.0; - return gapSeconds > 0 ? math.min(gapSeconds, fallbackSeconds) : fallbackSeconds; + return gapSeconds > 0 + ? math.min(gapSeconds, fallbackSeconds) + : fallbackSeconds; } static double _medianIntervalSeconds(List sorted) { diff --git a/lib/src/onehz/workout/observed_max_hr.dart b/lib/src/onehz/workout/observed_max_hr.dart new file mode 100644 index 0000000..3d7a997 --- /dev/null +++ b/lib/src/onehz/workout/observed_max_hr.dart @@ -0,0 +1,190 @@ +// WORKOUT — the OBSERVED heart-rate ceiling (TS-03). +// +// Not a physiological HRmax. This is the highest heart rate the band actually +// held on you, with the date it happened. If you have never gone truly hard it +// is an underestimate and it will keep creeping up — that is the honest shape +// of the number, and the copy must say "highest we've seen: 184, on 3 Aug", +// never "your max HR is 184", and must never invite a max-effort test. +// +// WHY THE HOLD GUARD IS THE WHOLE FEATURE. Every zone boundary in the app is a +// fraction of this number, so a single PPG artifact — one 210 bpm sample from a +// sleeve dragging across the optical window — would inflate the ceiling forever +// and silently drag every boundary up, with no way for the user to see why +// their easy runs stopped being easy. So a candidate counts only when it was +// HELD for [holdSeconds] continuous seconds AND the accelerometer says the body +// was moving while it happened. A held-but-still high HR is a stress response, +// a fever or an artifact, and it is not a ceiling. +// +// PER-DEVICE (device.dart contract). The motion gate is NOT universal: on our +// own captures the two families' resting accelerometer noise floors differ by +// ~4x (gen4 |‖a‖−1| median 0.034 g / p90 0.058 g over 672k rows; gen5+MG median +// 0.007–0.012 g / p90 0.018–0.024 g over 260k rows). One shared gate is either +// "moving" through half of a gen4 night or unreachable on a gen5 walk, and +// either way it is a fabricated corroboration. Unknown family REFUSES; it does +// not silently borrow gen4's floor. +// +// WHAT THE GUARD DOES NOT CATCH. Run over the real gen4 capture's own labelled +// sessions it trims 4-8 bpm off each session's raw maximum (a 193 bpm run peak +// becomes 189; a walk's 188 becomes 180) — so it removes single-sample spikes, +// which is what it is for. It does NOT remove CADENCE LOCK: a walk that reports +// 180 bpm for twenty seconds while the wrist is swinging satisfies both the +// hold and the motion test, because the artifact is generated BY the motion. +// That is why the number is labelled with its session and its date, so a wrong +// one is visible and attributable rather than an anonymous ceiling. +// +// SKIN TONE. Wrist reflectance PPG error at intensity is larger on darker skin +// (published bias up to ~11.8 bpm), and that bias rides straight into this +// number and into every zone built on it. Nothing here can correct it; the +// screen has to say it. + +import 'dart:math' as math; + +import '../device.dart'; +import '../types.dart'; +import '../util.dart'; + +/// A heart rate that was held long enough, while moving, to count. +class HrCeiling { + /// The bpm that was met or exceeded for the whole hold. + final double bpm; + + /// Start of the qualifying hold (epoch ms) — the "on 3 Aug" in the copy. + final double tsMs; + + /// How long the hold actually ran (>= the required hold). + final int heldSeconds; + + /// Mean |‖a‖ − 1| in g over the hold — the corroborating motion. + final double motionG; + + const HrCeiling({ + required this.bpm, + required this.tsMs, + required this.heldSeconds, + required this.motionG, + }); + + Map toJson() => { + 'bpm': round6(bpm), + 'ts_ms': tsMs, + 'held_seconds': heldSeconds, + 'motion_g': round6(motionG), + }; +} + +/// Mean resting-motion floor per family, in g of |‖a‖ − 1|, measured on our own +/// captures (see the header). The gate sits above each family's own p90 and +/// well below its p99, so ordinary wear does not corroborate and real movement +/// does. +const Map _motionGateG = { + DeviceFamily.gen4: 0.10, + DeviceFamily.gen5: 0.04, +}; + +/// Above this, a HELD heart rate is a sensor fault rather than a heart — no +/// per-family split, this is a physiological bound and not a sensor property. +const double _absurdBpm = 240.0; + +/// The highest heart rate held for [holdSeconds] continuous seconds with +/// corroborating motion, over one session's co-sampled 1 Hz streams. +/// +/// [hr] and [accel] must be the SAME session and index-aligned (sample i of one +/// is sample i of the other), exactly as the pipeline emits them; they are +/// sorted together here, so caller order does not matter but pairing does. +/// [deviceFamily] is the ingest-stamped id — NULL (every pre-schema-41 row, +/// every import, every raw replay) is a refusal, not gen4. +/// +/// Returns ABSENT when the session simply contains no qualifying hold. That is +/// the common case and it is correct: most sessions do not test your ceiling. +/// The app-wide ceiling is the max over sessions of the present values, which +/// is the caller's one-line reduce — deliberately not a function here, so that +/// the guard cannot be bypassed by feeding it un-guarded numbers. +Metric sessionHrCeiling( + List hr, + List accel, { + String? deviceFamily, + int holdSeconds = 15, + double maxGapSeconds = 2.0, +}) { + const inputs = ['hr_1hz', 'accel_1hz', 'device_family']; + final gate = calibrationFor(_motionGateG, deviceFamily); + if (gate == null) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: unknownFamilyNote(deviceFamily), + ); + } + if (accel.length != hr.length) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'hr and accel must be index-aligned samples of one session', + ); + } + + // Pair, drop unusable samples, sort by time. A sample with no HR or no accel + // can neither set a ceiling nor corroborate one. + final rows = <({double ts, double hr, double motion})>[]; + for (var i = 0; i < hr.length; i++) { + final h = hr[i]; + final a = accel[i]; + if (!h.valid || !a.valid) continue; + if (h.hr >= _absurdBpm) continue; + final mag = math.sqrt(a.x * a.x + a.y * a.y + a.z * a.z); + rows.add((ts: h.tsMs, hr: h.hr, motion: (mag - 1.0).abs())); + } + rows.sort((a, b) => a.ts.compareTo(b.ts)); + + final holdMs = holdSeconds * 1000.0; + final gapMs = maxGapSeconds * 1000.0; + HrCeiling? best; + // ponytail: O(n · holdSeconds) — one session at 1 Hz, so ~15 passes over a + // few thousand samples. A monotonic-deque sliding minimum if it ever runs + // over a whole day. + for (var i = 0; i < rows.length; i++) { + var lo = rows[i].hr; + var motionSum = 0.0; + var count = 0; + for (var j = i; j < rows.length; j++) { + if (j > i && rows[j].ts - rows[j - 1].ts > gapMs) break; // stream broke + lo = math.min(lo, rows[j].hr); + motionSum += rows[j].motion; + count++; + final span = rows[j].ts - rows[i].ts; + if (span < holdMs) continue; + // The window qualifies on duration. `lo` is the bpm sustained across all + // of it; extending further can only lower it, so this is the best this + // start can do and we stop. + final motion = motionSum / count; + if (motion >= gate && (best == null || lo > best.bpm)) { + best = HrCeiling( + bpm: lo, + tsMs: rows[i].ts, + heldSeconds: (span / 1000).round(), + motionG: motion, + ); + } + break; + } + } + + if (best == null) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'no_held_ceiling:hold=${holdSeconds}s,motion_gate_g=$gate', + ); + } + return Metric( + value: best, + confidence: clamp(best.heldSeconds / (holdSeconds * 2.0), 0.3, 0.8), + tier: Tier.high, + inputs_used: inputs, + note: 'OBSERVED ceiling, not physiological HRmax — an underestimate if you ' + 'have never gone truly hard, and it will keep creeping up. Label it ' + 'with its date. Wrist PPG at intensity is biased by skin tone ' + '(published up to ~11.8 bpm) and that rides into every zone built on ' + 'this number.', + ); +} diff --git a/lib/src/onehz/workout/workout.dart b/lib/src/onehz/workout/workout.dart index d4b96cf..542c245 100644 --- a/lib/src/onehz/workout/workout.dart +++ b/lib/src/onehz/workout/workout.dart @@ -4,14 +4,18 @@ /// untouched and adds bout-detection: /// /// * [AutoWorkoutDetector] — opt-in "did you just work out?" SUGGESTION -/// detector (HR ≥ RHR+30 sustained ≥12 min, brief dips tolerated, optional -/// motion confirmation, overlap-excluded). Never writes a row. → auto_detect.dart -/// * [WorkoutDetector] — persistent per-day detector (HR + motion gated, ≥5 min, -/// ≥50% time in zone 2+, #303 HR-bridge) producing per-bout avg/peak HR, -/// Edwards zone time-%, mean %HRR, strain (reuses [StrainScorer]), HRmax, and -/// calories (Keytel + Harris–Benedict via [Calories]). → workout_detect.dart +/// detector (HR ≥ max(RHR+40, RHR+0.45·HRR) sustained ≥12 min, brief dips +/// tolerated, optional motion confirmation, overlap-excluded). Never writes +/// a row, and is the ONLY bout detector here. → auto_detect.dart /// * [Calories] — Keytel 2005 + Harris–Benedict per-bout energy. → calories.dart /// +/// A second, retroactive per-day detector (`WorkoutDetector` / `detectWorkouts` +/// / `ExerciseSession`, workout_detect.dart) lived here for a while and was +/// never called by anything but its own tests. It is deleted rather than kept +/// warm: edge's `derivation_engine` no longer writes the `detected_workouts` +/// stub it would have fed, and auto-detected bouts reach the UI through +/// `workout_suggestions` from [AutoWorkoutDetector] instead. +/// /// HYBRID SEAM: every detected bout is typed through a [SportClassifier] /// (default "detected"). OpenStrap's motion-based HAR typer can be injected /// when high-rate accel features ([MotionFeatures]) are available. → sport.dart @@ -21,5 +25,5 @@ export 'sport.dart'; export 'calories.dart'; export 'auto_detect.dart'; export 'hr_zones.dart'; -export 'workout_detect.dart'; +export 'observed_max_hr.dart'; export 'hr_recovery.dart'; diff --git a/lib/src/onehz/workout/workout_detect.dart b/lib/src/onehz/workout/workout_detect.dart deleted file mode 100644 index 86e969a..0000000 --- a/lib/src/onehz/workout/workout_detect.dart +++ /dev/null @@ -1,551 +0,0 @@ -// workout_detect.dart — retroactive per-day workout detection from the 1 Hz store. -// -// Retroactive per-day workout detector (ported from WorkoutDetector.swift / -// exercise.py / activity.py / calories.py). A workout is a SUSTAINED window (≥ MIN_EXERCISE_MIN) -// of elevated HR (> resting + HR_MARGIN_BPM) AND sustained motion (gravity-derived -// intensity > MOTION_THRESHOLD); both gates must hold for a sample to count. -// -// Per detected bout: avg/peak HR, duration, Edwards zone time-%, mean %HRR, -// strain (the existing [StrainScorer], REUSED — not re-derived), HRmax + source -// (via [StrainScorer.estimateHRmax]), and calories (Keytel + Harris–Benedict via -// the ported [Calories]). Every bout is typed through the [SportClassifier] seam. -// -// Pure / headless: no I/O, no clock, dart:math only. ts/start/end are unix -// SECONDS. All intensity/energy outputs are APPROXIMATE, not medical advice. - -import 'dart:math' as math; - -import '../types.dart'; -import '../util.dart' show round6; -import '../clinical/load_trimp.dart'; -import 'auto_detect.dart' show SavedWorkoutSpan; -import 'calories.dart'; -import 'sport.dart'; - -/// A detected workout window. All intensity fields are APPROXIMATE. -class ExerciseSession { - final int start; - final int end; - final double avgHR; - final int peakHR; - final double? strain; - final double durationS; - - /// Edwards zone (0–5) time breakdown as % of HR samples; sums to ~100. - final Map zoneTimePct; - - /// Mean Karvonen %HRR over the bout, clamped [0,100], or null. - final double? avgHRRPct; - - /// Effective HRmax used for zone math (bpm), or null. - final double? hrmax; - - /// "caller" | "observed" | "tanaka" | "unknown". - final String hrmaxSource; - final double? caloriesKcal; - final double? caloriesKJ; - - /// True when [caloriesKcal]/[caloriesKJ] were computed against a FABRICATED - /// anchor — [Calories.estimateBoutCalories] falls back to a flat - /// `hrmax = 220` / `restingHr = 60` when either is null, and returns - /// `usedDefaultAnchors` precisely so that number can be caveated instead of - /// shown as if it were personal. The flag used to be computed and thrown - /// away; it is now carried through to [toJson]. False when no calories were - /// computed at all. - final bool caloriesUsedDefaultAnchors; - - /// Sport label from the classifier seam ("detected" by default). - final String sport; - - const ExerciseSession({ - required this.start, - required this.end, - required this.avgHR, - required this.peakHR, - required this.strain, - required this.durationS, - required this.zoneTimePct, - required this.avgHRRPct, - required this.hrmax, - required this.hrmaxSource, - required this.caloriesKcal, - required this.caloriesKJ, - this.caloriesUsedDefaultAnchors = false, - this.sport = defaultSportLabel, - }); - - Map toJson() => { - 'start': start, - 'end': end, - 'avg_hr': round6(avgHR), - 'peak_hr': peakHR, - 'strain': strain == null ? null : round6(strain!), - 'duration_s': round6(durationS), - 'zone_time_pct': {for (final e in zoneTimePct.entries) '${e.key}': e.value}, - 'avg_hrr_pct': avgHRRPct, - 'hrmax': hrmax == null ? null : round6(hrmax!), - 'hrmax_source': hrmaxSource, - 'calories_kcal': caloriesKcal == null ? null : round6(caloriesKcal!), - 'calories_kj': caloriesKJ == null ? null : round6(caloriesKJ!), - 'calories_used_default_anchors': caloriesUsedDefaultAnchors, - 'sport': sport, - }; -} - -/// Per-record motion-intensity point. -class ActivityPoint { - final int ts; - final double intensity; - const ActivityPoint(this.ts, this.intensity); -} - -/// Retroactive per-day workout detector. -class WorkoutDetector { - // ── Constants (exercise.py) ──────────────────────────────────────────────── - static const double minExerciseMin = 5.0; - static const double hrMarginBPM = 15.0; - static const double motionThreshold = 0.20; - static const double motionSmoothS = 10.0; - static const double mergeGapS = 150.0; - static const double minIntensityZ2Plus = 0.50; - static const double alignToleranceS = 5.0; - static const double restingPercentile = 10.0; - - /// Second-pass bridge window (#303): two adjacent active runs separated by a - /// below-motion gap ≤ this are stitched IFF HR stays elevated across the gap. - static const double bridgeGapS = 300.0; - - /// Per-record motion-intensity series: L2 magnitude of the gravity change vs - /// the previous record. First row → 0. Empty → []. [gravTs]/[gx]/[gy]/[gz] - /// parallel arrays. - static List activitySeries( - List gravTs, List gx, List gy, List gz) { - final n = gravTs.length; - if (n == 0) return const []; - final idx = List.generate(n, (i) => i) - ..sort((a, b) => gravTs[a].compareTo(gravTs[b])); - final out = []; - double? px, py, pz; - for (var k = 0; k < n; k++) { - final i = idx[k]; - final double intensity; - if (k == 0 || px == null) { - intensity = 0.0; - } else { - final dx = gx[i] - px; - final dy = gy[i] - py!; - final dz = gz[i] - pz!; - intensity = math.sqrt(dx * dx + dy * dy + dz * dz); - } - out.add(ActivityPoint(gravTs[i], intensity)); - px = gx[i]; - py = gy[i]; - pz = gz[i]; - } - return out; - } - - /// Day resting-HR baseline = nearest-rank RESTING_PERCENTILE of the ON-SKIN - /// bpm values. - /// - /// OFF-SKIN FILTER: the package convention (types.dart, `HrSample`) is that - /// `hr == 0` means the sensor was off the wrist, NEVER bradycardia. Taking the - /// 10th percentile of the RAW stream on a day with ≥10 % dropout returned - /// restHR = 0, which dragged hrFloor down to 15 bpm and inflated every - /// downstream %HRR — an ordinary 120 bpm walk read as 63 % HRR (zone 2) - /// instead of 48 % (zone 0), so it cleared the zone-2 workout gate. Zeros are - /// dropped before the percentile; null when nothing on-skin remains (we - /// abstain rather than invent a resting HR). - static double? _deriveRestingHR(List bpm) { - final onSkin = [for (final b in bpm) if (b > 0 && b.isFinite) b]..sort(); - if (onSkin.isEmpty) return null; - final rank = math.max(1, (restingPercentile / 100.0 * onSkin.length).ceil()); - return onSkin[rank - 1]; - } - - /// Value whose ts is nearest [target] within [tol] s, else null. Ties → later - /// timestamp (matches Python <=). - static double? _nearest( - List sortedTs, List values, int target, double tol) { - if (sortedTs.isEmpty) return null; - // bisect_left - var lo = 0, hi = sortedTs.length; - while (lo < hi) { - final mid = (lo + hi) >> 1; - if (sortedTs[mid] < target) { - lo = mid + 1; - } else { - hi = mid; - } - } - final i = lo; - double? bestV; - var bestD = tol; - for (final j in [i - 1, i]) { - if (j >= 0 && j < sortedTs.length) { - final d = (sortedTs[j] - target).abs().toDouble(); - if (d <= bestD) { - bestD = d; - bestV = values[j]; - } - } - } - return bestV; - } - - /// Trailing rolling mean (over [windowS]) of intensities. - static List _smoothedIntensity( - List motion, double windowS) { - final ts = [for (final p in motion) p.ts]; - final raw = [for (final p in motion) p.intensity.isFinite ? p.intensity : 0.0]; - final out = []; - var lo = 0; - var running = 0.0; - for (var i = 0; i < motion.length; i++) { - running += raw[i]; - while ((ts[i] - ts[lo]).toDouble() > windowS) { - running -= raw[lo]; - lo++; - } - out.add(running / (i - lo + 1)); - } - return out; - } - - /// Per-bout Edwards zone breakdown (%) + mean %HRR. Reuses [StrainScorer]. - static (Map, double?) _boutIntensity( - List bpm, double restingHR, double maxHR) { - if (bpm.isEmpty || maxHR <= restingHR) return ({}, null); - final hrReserve = maxHR - restingHR; - final zoneCounts = {for (var z = 0; z <= 5; z++) z: 0}; - final hrrVals = []; - for (final b in bpm) { - final z = StrainScorer.zoneWeight(b, restingHR, hrReserve); - zoneCounts[z] = (zoneCounts[z] ?? 0) + 1; - hrrVals.add(StrainScorer.pctHRR(b, restingHR, hrReserve)); - } - final n = bpm.length.toDouble(); - final zonePct = {}; - zoneCounts.forEach((z, c) { - zonePct[z] = ((c / n * 100.0) * 10).roundToDouble() / 10; - }); - final avgHRR = - ((hrrVals.reduce((a, b) => a + b) / n) * 10).roundToDouble() / 10; - return (zonePct, avgHRR); - } - - /// Second-pass bridge over raw active runs (#303). - static List> _bridgeRuns( - List> runs, List hrTs, List hrBpm, double hrFloor) { - if (runs.length <= 1) return runs; - final merged = >[]; - var curStart = runs[0][0]; - var curEnd = runs[0][1]; - for (var r = 1; r < runs.length; r++) { - final next = runs[r]; - final gap = (next[0] - curEnd).toDouble(); - var bridge = false; - if (gap <= bridgeGapS) { - final gapHR = []; - for (var k = 0; k < hrTs.length; k++) { - if (hrTs[k] > curEnd && hrTs[k] < next[0]) gapHR.add(hrBpm[k]); - } - if (gapHR.isEmpty) { - bridge = true; // sensor dropout mid-effort → same workout - } else { - final meanGapHR = gapHR.reduce((a, b) => a + b) / gapHR.length; - bridge = meanGapHR > hrFloor; // still working → same workout - } - } - if (bridge) { - curEnd = math.max(curEnd, next[1]); - } else { - merged.add([curStart, curEnd]); - curStart = next[0]; - curEnd = next[1]; - } - } - merged.add([curStart, curEnd]); - return merged; - } - - /// Closed-interval overlap (touching endpoints count). - static bool _overlaps(int aStart, int aEnd, int bStart, int bEnd) => - aStart <= bEnd && bStart <= aEnd; - - /// Detect workouts from the 1 Hz HR + gravity store. - /// - /// [hrTs]/[hrBpm] heart-rate stream (parallel arrays; empty → []). - /// [gravTs]/[gx]/[gy]/[gz] gravity stream (parallel; empty → []). - /// [restingHR] day RHR baseline; null → 10th-pct of the day's HR. - /// [maxHR] HRmax; null → [StrainScorer.estimateHRmax] (observed/Tanaka). - /// [age] for the Tanaka fallback. [profile] when provided → per-bout calories. - /// [savedSpans] saved/manual spans — a detected bout overlapping any is DROPPED - /// (overlap-dedup; caller passes saved spans). [classify] the sport seam. - static List detect({ - required List hrTs, - required List hrBpm, - required List gravTs, - required List gx, - required List gy, - required List gz, - double? restingHR, - double? maxHR, - double? age, - WorkoutUserProfile? profile, - List savedSpans = const [], - SportClassifier classify = defaultSportClassifier, - }) { - // Clean + sort HR. - final hn = hrTs.length; - if (hn == 0) return const []; - final ho = List.generate(hn, (i) => i) - ..sort((a, b) => hrTs[a].compareTo(hrTs[b])); - final sTs = [for (final i in ho) hrTs[i]]; - final sBpm = [for (final i in ho) hrBpm[i]]; - - final motion = activitySeries(gravTs, gx, gy, gz); - if (motion.isEmpty) return const []; - - final restHR = restingHR ?? _deriveRestingHR(sBpm); - // No caller RHR and no on-skin sample to derive one from → no baseline, so - // no honest HR gate. Abstain rather than gate against a fabricated floor. - if (restHR == null) return const []; - final hrFloor = restHR + hrMarginBPM; - - final double? effMaxHR; - final String hrmaxSource; - if (maxHR != null) { - effMaxHR = maxHR; - hrmaxSource = 'caller'; - } else { - final (est, src) = StrainScorer.estimateHRmax(sBpm, age); - effMaxHR = est == 0.0 ? null : est; - hrmaxSource = src; - } - - final smooth = _smoothedIntensity(motion, motionSmoothS); - - // Walk the gravity timeline; flag samples where BOTH gates hold. - final activeTs = []; - for (var k = 0; k < motion.length; k++) { - if (smooth[k] <= motionThreshold) continue; - final bpm = _nearest(sTs, sBpm, motion[k].ts, alignToleranceS); - if (bpm == null || bpm <= hrFloor) continue; - activeTs.add(motion[k].ts); - } - if (activeTs.isEmpty) return const []; - - // Group contiguous active samples into runs, merging gaps < MERGE_GAP_S. - var runs = >[]; - var runStart = activeTs[0]; - var prev = activeTs[0]; - for (var k = 1; k < activeTs.length; k++) { - final t = activeTs[k]; - if ((t - prev).toDouble() > mergeGapS) { - runs.add([runStart, prev]); - runStart = t; - } - prev = t; - } - runs.add([runStart, prev]); - - // Second pass (#303): bridge across brief, still-elevated-HR lulls. - runs = _bridgeRuns(runs, sTs, sBpm, hrFloor); - - final minDurS = minExerciseMin * 60.0; - final sessions = []; - for (final run in runs) { - final start = run[0], end = run[1]; - - // Onset latency tolerance equal to the smoothing window. - if ((end - start).toDouble() < minDurS - motionSmoothS) continue; - - // window HR samples - final winTs = []; - final winBpm = []; - for (var k = 0; k < sTs.length; k++) { - if (sTs[k] >= start && sTs[k] <= end) { - winTs.add(sTs[k]); - winBpm.add(sBpm[k]); - } - } - if (winBpm.isEmpty) continue; - - var zonePct = {}; - double? avgHRR; - if (effMaxHR != null && effMaxHR > restHR) { - (zonePct, avgHRR) = _boutIntensity(winBpm, restHR, effMaxHR); - } - - // Intensity qualification: require ≥ MIN_INTENSITY_Z2PLUS in zone 2+. - // - // AN UNEVALUABLE GATE BLOCKS, IT DOES NOT PASS. `zonePct` is empty exactly - // when there is no usable HRmax anchor (no caller HRmax, no age for - // Tanaka, <600 samples for an observed estimate → estimateHRmax returns - // ("unknown", 0)), or when the anchor is at/below resting HR. The old code - // skipped the whole gate in that case, so a 6-minute walk at RHR+16 bpm - // was emitted as a durable workout. With no zone breakdown we cannot know - // whether the bout qualified, so we drop it. - if (zonePct.isEmpty) continue; - var z2plus = 0.0; - for (var z = 2; z <= 5; z++) { - z2plus += zonePct[z] ?? 0.0; - } - z2plus /= 100.0; - if (z2plus < minIntensityZ2Plus) continue; - - // OVERLAP-DEDUP: drop a detected bout overlapping a saved/manual span. - if (savedSpans.any((s) => _overlaps(start, end, s.startSec, s.endSec))) { - continue; - } - - double? kcal, kj; - // Carry [Calories.estimateBoutCalories]'s usedDefaultAnchors through to - // the session — it exists so a calorie number built on the flat - // `hrmax ?? 220` / `restingHr ?? 60` fallback can be caveated, and it used - // to be computed and dropped on the floor here. - var calUsedDefaultAnchors = false; - if (profile != null) { - final winBpmInt = [for (final b in winBpm) b]; - final cal = Calories.estimateBoutCalories( - winTs, - winBpmInt, - profile: profile, - hrmax: effMaxHR, - restingHr: restHR, - // The published cap, not this class's split threshold. They are the - // same number today, but they are not the same quantity, and the cap - // really can bind inside a detected bout: [bridgeGapS] is twice - // [mergeGapS], so _bridgeRuns stitches an HR-free dropout of up to - // 300 s into one bout. A bout straddling a ~252 s dropout measures - // 324.62 kcal capped against 349.44 uncapped — 7.1%. Restating one - // constant as the other would mean a later move of the cap silently - // rescored auto-detected bouts against a value the live gauge and - // manual_session no longer use, which is the whole reason both sides - // read the same constant. - mergeGapCapS: Calories.defaultMergeGapCapS, - ); - kcal = cal.kcal; - kj = cal.kj; - calUsedDefaultAnchors = cal.usedDefaultAnchors; - } - - final avg = winBpm.reduce((a, b) => a + b) / winBpm.length; - final peak = winBpm.reduce(math.max).round(); - // Strain via the existing StrainScorer (reused, NOT re-derived). - // - // NO HIDDEN ANCHOR: StrainScorer.strain silently substitutes - // `defaultMaxHR() = 220 − 30 = 190` when maxHR is null, so a session used - // to report `hrmax: null, hrmax_source: 'unknown'` next to a concrete - // strain scored against an invented 190. We ABSTAIN instead — no anchor, - // no strain. (The zone gate above already drops such bouts; this keeps the - // guarantee local so it survives any future change to that gate. The - // fallback itself lives in clinical/load_trimp.dart and is not ours to - // change.) - final strain = effMaxHR == null - ? null - : StrainScorer.strain( - winBpm, - [for (final t in winTs) t.toDouble()], - maxHR: effMaxHR, - restingHR: restHR, - ); - - // HYBRID SEAM: type the bout. - final bout = WorkoutBout( - startSec: start, - endSec: end, - avgBpm: avg, - peakBpm: peak.toDouble(), - durationS: (end - start).toDouble(), - ); - // Mean motion intensity over the bout window → a 1 Hz amplitude feature. - var msum = 0.0; - var mcnt = 0; - for (final p in motion) { - if (p.ts >= start && p.ts <= end) { - msum += p.intensity; - mcnt++; - } - } - final feats = - mcnt == 0 ? null : MotionFeatures(meanIntensity: msum / mcnt); - final sport = classify(bout, feats); - - sessions.add(ExerciseSession( - start: start, - end: end, - avgHR: avg, - peakHR: peak, - strain: strain, - durationS: (end - start).toDouble(), - zoneTimePct: zonePct, - avgHRRPct: avgHRR, - hrmax: effMaxHR, - hrmaxSource: hrmaxSource, - caloriesKcal: kcal, - caloriesKJ: kj, - caloriesUsedDefaultAnchors: calUsedDefaultAnchors, - sport: sport, - )); - } - return sessions; - } -} - -/// Wrap detected workouts in the honesty envelope. Always present — an empty list -/// is a valid honest "no detected workouts" answer. -Metric> detectWorkouts({ - required List hrTs, - required List hrBpm, - required List gravTs, - required List gx, - required List gy, - required List gz, - double? restingHR, - double? maxHR, - double? age, - WorkoutUserProfile? profile, - List savedSpans = const [], - SportClassifier classify = defaultSportClassifier, -}) { - final list = WorkoutDetector.detect( - hrTs: hrTs, - hrBpm: hrBpm, - gravTs: gravTs, - gx: gx, - gy: gy, - gz: gz, - restingHR: restingHR, - maxHR: maxHR, - age: age, - profile: profile, - savedSpans: savedSpans, - classify: classify, - ); - // Distinguish "no workouts today" from "the zone-2 qualification gate could - // not be evaluated because there is no HRmax anchor" — with no anchor the - // detector correctly emits nothing, and the caller deserves to know why. - final noAnchor = - maxHR == null && StrainScorer.estimateHRmax(hrBpm, age).$1 == 0.0; - return Metric>( - value: list, - confidence: list.isEmpty ? 0.0 : 0.6, - tier: Tier.estimate, - inputs_used: [ - 'hr_1hz', - 'gravity_1hz', - if (profile != null) 'profile', - if (maxHR != null) 'max_hr', - if (age != null) 'age', - if (restingHR != null) 'resting_hr', - ], - note: noAnchor - ? 'no HRmax anchor (no caller HRmax, no age, too few HR samples for an ' - 'observed estimate) — the ≥50% time-in-zone-2+ qualification gate ' - 'cannot be evaluated, so NO workout is emitted (never guessed)' - : 'detected workouts (HR + motion gated, ≥5 min, ≥50% time in zone 2+); ' - 'wrist-HR ESTIMATE, not medical advice', - ); -} diff --git a/test/onehz/an_training_human_test.dart b/test/onehz/an_training_human_test.dart new file mode 100644 index 0000000..77e2603 --- /dev/null +++ b/test/onehz/an_training_human_test.dart @@ -0,0 +1,365 @@ +// TS-11 / TS-12 / MIND-11 / WH-08 — the refusals, mostly. Each of these +// features is one sentence of output wrapped in the conditions under which it +// must say nothing, and the abstention is the part that gets quietly deleted +// later, so it is the part that is pinned here. + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +List _dates(int n) { + final start = DateTime.utc(2026, 1, 1); + return [ + for (var i = 0; i < n; i++) + () { + final d = start.add(Duration(days: i)); + return '${d.year}-${d.month.toString().padLeft(2, '0')}' + '-${d.day.toString().padLeft(2, '0')}'; + }(), + ]; +} + +void main() { + // ------------------------------------------------------------------------- + group('MIND-11 alertness forecast', () { + test('ABSTAINS on a missing night — it never assumes eight hours', () { + for (final m in [ + alertnessForecast(wakeLocalHour: 7, sleepDurationHours: null), + alertnessForecast(wakeLocalHour: null, sleepDurationHours: 7.5), + alertnessForecast(wakeLocalHour: 7, sleepDurationHours: 0), + ]) { + expect(m.present, isFalse); + expect(m.confidence, 0); + expect(m.note, contains('no_judged_night')); + } + }); + + test('the safety refusal is on every output, present or absent', () { + final present = + alertnessForecast(wakeLocalHour: 7, sleepDurationHours: 7.5); + final absent = + alertnessForecast(wakeLocalHour: 7, sleepDurationHours: null); + for (final m in [present, absent]) { + expect(m.note, contains('fitness-to-drive')); + expect(m.note, contains('impaired')); + } + }); + + test('emits a shape and a named window, and NO score', () { + final m = alertnessForecast( + wakeLocalHour: 7, + sleepDurationHours: 7.5, + circadianAcrophaseHours: 16, + ); + expect(m.present, isTrue); + final j = m.value!.toJson(); + // Whatever else changes, there must never be a scalar alertness value. + for (final k in ['score', 'alertness', 'kss', 'value', 'level']) { + expect(j.containsKey(k), isFalse, reason: k); + } + expect(j['shape'], isA>()); + expect(m.value!.troughLabel, isNotEmpty); + expect(m.value!.shape.every((v) => v >= 0 && v <= 1), isTrue); + }); + + test('the trough is not just sleep inertia, and a nap moves the shape', () { + final m = alertnessForecast(wakeLocalHour: 7, sleepDurationHours: 7.5); + // The first hour after waking is the lowest raw point in the model. If + // the trough window reported that, the "forecast" would be the past. + expect(m.value!.troughStartHour, greaterThan(8.0)); + + final napped = alertnessForecast( + wakeLocalHour: 7, + sleepDurationHours: 7.5, + naps: const [DaytimeSleepWindow(13, 13.5)], + ); + expect(napped.value!.shape, isNot(equals(m.value!.shape))); + }); + + test('RD-01: the day peaks in the afternoon, not an hour after waking', () { + // τ_wake was 2.6 h (the PUBLISHED τ_sleep, written into the wake slot), + // so the homeostat fell from 12.53 to 4.57 in the first four hours and sat + // on its 2.4 floor from mid-morning: the whole curve was process C plus a + // decaying inertia term, and its MAXIMUM landed at 08:00 for a 07:00 + // riser. With the published rates (1/0.0353 and 1/0.3813, PMC4203690 + // §1.1–1.8) S runs 13.74 / 12.24 / 10.95 / 9.82 / 8.84 over 0/4/8/12/16 h + // awake and the peak sits just before the 16.8 h acrophase. + final m = alertnessForecast(wakeLocalHour: 7, sleepDurationHours: 8); + final shape = m.value!.shape; + var peak = 0; + for (var i = 1; i < shape.length; i++) { + if (shape[i] > shape[peak]) peak = i; + } + final peakClock = 7 + peak * 0.25; + expect(peakClock, closeTo(15.75, 0.5)); + }); + + test('RD-02: the low window moves with sleep length, not with the horizon', + () { + // The search used to run over a fixed 18 h curve, so the lowest window was + // whichever one fitted last: wake + 16.25 h in 24 of 24 replayed + // wake × sleep combinations, with sleep duration moving it 0.00 h. It is + // bounded by the waking day now (24 h − last night's sleep). + double offset(double sleep) { + final v = alertnessForecast(wakeLocalHour: 7, sleepDurationHours: sleep) + .value!; + return (v.troughStartHour - 7 + 24) % 24; + } + + expect(offset(5), closeTo(16.25, 1e-9)); + expect(offset(7), closeTo(15.25, 1e-9)); + expect(offset(8), closeTo(14.25, 1e-9)); + expect(offset(9), closeTo(13.25, 1e-9)); + // And the note says what the window is, so no one reads it as measured. + expect(alertnessForecast(wakeLocalHour: 7, sleepDurationHours: 8).note, + contains('not a dip')); + }); + + test('an assumed circadian phase is disclosed as assumed', () { + expect(alertnessForecast(wakeLocalHour: 7, sleepDurationHours: 7.5).note, + contains('ASSUMED')); + final fitted = alertnessForecast( + wakeLocalHour: 7, + sleepDurationHours: 7.5, + circadianAcrophaseHours: 15.2) + .note; + expect(fitted, isNot(contains('ASSUMED'))); + // RD-12: dropping the assumption clause must not leave the card implying + // the phase was MEASURED on the user. What the caller passes is a cosinor + // fitted on hourly HR — a proxy — and the note has to say so. + expect(fitted, contains('PROXY')); + }); + }); + + // ------------------------------------------------------------------------- + group('TS-11 next-morning cost by session type', () { + // 60 days of a 50 bpm resting HR with real dispersion, and a football + // session every 4th day whose next morning runs +10 bpm. + List series({required bool withEffect}) { + const noise = [0.0, 1, -1, 2, -2, 1, -1, 0, 2, -2]; + final v = [for (var i = 0; i < 60; i++) 50.0 + noise[i % 10]]; + if (withEffect) { + for (var i = 4; i < 60; i += 4) { + v[i] = v[i]! + 10; + } + } + return v; + } + + Map> football(List dates, {int every = 4}) => { + for (var i = 3; i < dates.length; i += every) dates[i]: ['football'], + }; + + test('reports the median move and the n it rests on', () { + final dates = _dates(60); + final m = sessionMorningEffects( + dates: dates, + values: series(withEffect: true), + metric: 'rhr', + sessionTypesByDate: football(dates), + ); + expect(m.present, isTrue); + final e = m.value!.single; + expect(e.sessionType, 'football'); + // Smaller than the injected 10: the trailing baseline is his USUAL, and + // his usual already contains a football morning every fourth day, which + // pulls the baseline up. The card says "above your baseline", and this + // is that baseline. + expect(e.medianDelta, greaterThan(6.0)); + expect(e.medianDelta, lessThan(10.0)); + expect(e.n, greaterThanOrEqualTo(10)); + expect(e.exceedsMdc, isTrue); + expect(m.note, contains('ASSOCIATION ONLY')); + }); + + test('a move inside the MDC is not reported as a finding', () { + final dates = _dates(60); + final v = [ + for (var i = 0; i < 60; i++) + 50.0 + [0.0, 1, -1, 2, -2, 1, -1, 0, 2, -2][i % 10] + ]; + for (var i = 4; i < 60; i += 4) { + v[i] = v[i]! + 0.5; // real, tiny, unresolvable + } + final e = sessionMorningEffects( + dates: dates, + values: v, + metric: 'rhr', + sessionTypesByDate: football(dates), + ).value!.single; + expect(e.exceedsMdc, isFalse); + }); + + test('a day with two sessions belongs to neither type', () { + final dates = _dates(60); + final both = { + for (var i = 3; i < 60; i += 4) dates[i]: ['football', 'run'], + }; + final m = sessionMorningEffects( + dates: dates, + values: series(withEffect: true), + metric: 'rhr', + sessionTypesByDate: both, + ); + expect(m.present, isFalse, reason: 'ambiguous days are dropped entirely'); + }); + + test('a low-coverage night is not a morning', () { + final dates = _dates(60); + final m = sessionMorningEffects( + dates: dates, + values: series(withEffect: true), + metric: 'rhr', + sessionTypesByDate: football(dates), + coverage: [for (var i = 0; i < 60; i++) 0.1], + ); + expect(m.present, isFalse); + }); + + test('refuses under the minimum n rather than showing a small one', () { + final dates = _dates(60); + final m = sessionMorningEffects( + dates: dates, + values: series(withEffect: true), + metric: 'rhr', + sessionTypesByDate: football(dates, every: 20), + ); + expect(m.present, isFalse); + expect(m.note, contains('need_sessions')); + }); + }); + + // ------------------------------------------------------------------------- + group('TS-12 overreaching conjunction', () { + final quiet = [for (var i = 0; i < 28; i++) 50.0 + (i.isEven ? 1 : -1)]; + + Metric loadWithRamp() => ctlAtlTsb([ + for (var i = 0; i < 35; i++) 40.0, + for (var i = 0; i < 7; i++) 200.0, + ]); + + test('fires only when BOTH facts hold', () { + final m = overreachingConjunction( + load: loadWithRamp(), + rhrRecent: const [58.0, 59, 57, 58, 50], + rhrBaselineWindow: quiet, + ); + expect(m.present, isTrue); + expect(m.value!.loadRatio, greaterThan(1.5)); + expect(m.value!.nightsElevated, 4); + expect(m.value!.bothPointSameWay, isTrue); + + // Same nights, no load ramp. + final flat = overreachingConjunction( + load: ctlAtlTsb([for (var i = 0; i < 42; i++) 40.0]), + rhrRecent: const [58.0, 59, 57, 58, 50], + rhrBaselineWindow: quiet, + ); + expect(flat.value!.bothPointSameWay, isFalse); + }); + + test('a rise inside the usual spread is not an elevated night', () { + final m = overreachingConjunction( + load: loadWithRamp(), + rhrRecent: const [50.5, 50.6, 50.4, 50.5, 50.5], + rhrBaselineWindow: quiet, + ); + expect(m.value!.nightsElevated, 0); + expect(m.value!.bothPointSameWay, isFalse); + }); + + test('abstains without load history or a usable baseline', () { + expect( + overreachingConjunction( + load: null, + rhrRecent: const [58.0, 59, 57, 58, 50], + rhrBaselineWindow: quiet, + ).present, + isFalse); + expect( + overreachingConjunction( + load: loadWithRamp(), + rhrRecent: const [58.0, 59, 57, 58, 50], + rhrBaselineWindow: List.filled(28, 50), // no dispersion + ).present, + isFalse); + }); + + test('the copy names the confounds and refuses the push channel', () { + final m = overreachingConjunction( + load: loadWithRamp(), + rhrRecent: const [58.0, 59, 57, 58, 50], + rhrBaselineWindow: quiet, + ); + for (final w in ['Illness', 'travel', 'altitude', 'no notification']) { + expect(m.note, contains(w)); + } + }); + }); + + // ------------------------------------------------------------------------- + group('WH-08 logged cycle lengths', () { + List onsets(List lengths) { + var d = DateTime.utc(2025, 1, 1); + final out = [_fmt(d)]; + for (final l in lengths) { + d = d.add(Duration(days: l)); + out.add(_fmt(d)); + } + return out; + } + + test('differences consecutive onsets and carries the criterion numbers', + () { + final m = cycleLengthSeries(onsets(List.filled(13, 28))); + expect(m.present, isTrue); + expect(m.value!.lengthsDays, everyElement(28)); + expect(m.value!.maxConsecutiveDifferenceDays, 0); + final j = m.value!.toJson(); + expect(j['difference_criterion_days'], 7); + expect(j['long_interval_criterion_days'], 60); + }); + + test('emits NO verdict text and no stage vocabulary', () { + final m = cycleLengthSeries( + onsets([28, 35, 26, 41, 27, 30, 45, 25, 33, 29, 38, 26, 31])); + final blob = '${m.note} ${m.value!.toJson()}'.toLowerCase(); + for (final word in [ + 'menopause', + 'perimenopause', + 'transition', + 'stage', + 'early', + 'late', + 'premature', + ]) { + expect(blob.contains(word), isFalse, reason: word); + } + expect(m.note, contains('clinician')); + }); + + test('REFUSES on a hole — a missed log and a long interval are identical', + () { + final m = cycleLengthSeries( + onsets([28, 28, 28, 120, 28, 28, 28, 28, 28, 28, 28, 28, 28])); + expect(m.present, isFalse); + expect(m.note, contains('log_has_holes')); + }); + + test('refuses under a long history', () { + final m = cycleLengthSeries(onsets(List.filled(4, 28))); + expect(m.present, isFalse); + expect(m.note, contains('need_baseline')); + }); + + test('a 60-day interval is charted, not judged', () { + final m = cycleLengthSeries( + onsets([28, 28, 60, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28])); + expect(m.present, isTrue); + expect(m.value!.longestIntervalDays, 60); + expect(m.value!.maxConsecutiveDifferenceDays, 32); + }); + }); +} + +String _fmt(DateTime d) => '${d.year}-${d.month.toString().padLeft(2, '0')}' + '-${d.day.toString().padLeft(2, '0')}'; diff --git a/test/onehz/clinical_test.dart b/test/onehz/clinical_test.dart index 291e17d..d401c06 100644 --- a/test/onehz/clinical_test.dart +++ b/test/onehz/clinical_test.dart @@ -22,6 +22,76 @@ void main() { test('absent on too-few beats', () { expect(hrvTime([800]).present, isFalse); }); + + test('HRV-02: RMSSD/pNN50 refused when the differences are white noise', + () { + // Differencing a smooth tachogram leaves ACF1 near 0; differencing white + // noise leaves exactly -0.5. gen5/MG nights measure -0.426..-0.517 — + // essentially pure jitter — while every gen4 night sits at -0.057..-0.324. + final rnd = math.Random(7); + final jitter = [ + for (var i = 0; i < 600; i++) 1000 + (rnd.nextDouble() - 0.5) * 120 + ]; + final noisy = hrvTime(jitter); + expect(noisy.value!.diffAcf1!, lessThan(kNnDiffAcf1Floor)); + expect(noisy.value!.diffAcf1!, closeTo(-0.5, 0.1), + reason: 'differenced white noise'); + expect(noisy.value!.rmssd, isNull, reason: 'jitter, not vagal tone'); + expect(noisy.value!.pnn50, isNull); + // SDNN is a dispersion of LEVELS and survives — the header's advice. + expect(noisy.value!.sdnn, isNotNull); + expect(noisy.note, contains('rmssd_refused:acf1=')); + + // A smooth respiratory-sinus-shaped series keeps its RMSSD. + final smooth = [ + for (var i = 0; i < 600; i++) 1000 + 40 * math.sin(2 * math.pi * i / 16) + ]; + final clean = hrvTime(smooth); + expect(clean.value!.diffAcf1!, greaterThan(kNnDiffAcf1Floor)); + expect(clean.value!.rmssd, isNotNull); + expect(clean.value!.pnn50, isNotNull); + }); + + test('HRV-02: confidence carries jitter and artifact, not beat count alone', + () { + // It used to be clamp(n/250, .3, .95), which published 0.95 on all 13 + // nights of the audit corpus. n/250 saturates after ~4 min, so on a whole + // night confidence was a constant. + final smooth = [ + for (var i = 0; i < 600; i++) 1000 + 40 * math.sin(2 * math.pi * i / 16) + ]; + final base = hrvTime(smooth).confidence; + expect(base, closeTo(0.95, 1e-9), + reason: 'a clean series still tops out'); + expect(hrvTime(smooth, artifactFraction: 0.15).confidence, + closeTo(0.85, 1e-9), + reason: '15% artifact must cost confidence, as hrvFreq already does'); + final rnd = math.Random(7); + final jitter = [ + for (var i = 0; i < 600; i++) 1000 + (rnd.nextDouble() - 0.5) * 120 + ]; + expect(hrvTime(jitter).confidence, 0.3, + reason: 'confidence bottoms out where RMSSD is refused'); + }); + + test('successive differences do not span a DROPPED run', () { + // an-clinical-6. correctRr drops multi-beat artifact runs while advancing + // its clock across them, so nn[i-1] and nn[i] can sit either side of a + // seconds-long hole. Differencing straight down the compacted list + // manufactured one large difference per dropped run — a fraction of a ms + // on a night's RMSSD, but pNN50 can be entirely seam-driven. + // 800-ms beats, a 3-beat hole, then 900-ms beats: the seam Δ is 100 ms + // (a pNN50 hit) and the real Δs are all 0. + final nn = [800, 800, 800, 900, 900, 900]; + final times = [800, 1600, 2400, 5700, 6600, 7500]; // 3.3 s hole + final seamed = hrvTime(nn).value!; // no beat times -> seam included + expect(seamed.pnn50, closeTo(100.0 / 5, 1e-9)); + final clean = hrvTime(nn, nnTimesMs: times).value!; + expect(clean.rmssd, 0.0, reason: '4 contiguous pairs, all Δ = 0'); + expect(clean.pnn50, 0.0); + // SDNN is a dispersion of levels, not of differences — unaffected. + expect(clean.sdnn, closeTo(seamed.sdnn!, 1e-12)); + }); }); group('frequency-domain HRV (Lomb-Scargle on beat times)', () { @@ -45,7 +115,13 @@ void main() { }); test('GATES HF when artifact fraction exceeds the threshold', () { - final rr = [for (var i = 0; i < 64; i++) 1000 + (i.isEven ? 30 : -30)]; + // RE-PINNED 2026-08: 400 beats, not 64. Band powers are now Welch- + // averaged over segments long enough to RESOLVE the band (10 cycles of + // its lowest frequency: 250 s for LF), so a 64 s record honestly reports + // no LF and no HF at all rather than a grid-aliased number. + final rr = [ + for (var i = 0; i < 400; i++) 1000 + (i.isEven ? 30 : -30) + ]; final times = []; var t = 0.0; for (final v in rr) { @@ -58,6 +134,58 @@ void main() { expect(m.value!.lf, isNotNull); // LF still reported }); + test('MAGNITUDE: total power ≈ SDNN², and lf_hf is grid-stable', () { + // T-01. Nothing used to assert a spectral MAGNITUDE, which is how three + // separate defects survived: lombScargle returned Horne–Baliunas + // variance-NORMALISED power (dimensionless) that was labelled ms², and a + // fixed 600-point grid undersampled a night's periodogram ~19x so band + // power was a lucky sample rather than an integral. + // + // Known-answer synthetic: 30 ms LF tone at 0.10 Hz (variance 450 ms²) + + // 20 ms HF tone at 0.25 Hz (variance 200 ms²) on 9000 beats. + final nn = []; + final ts = []; + var t = 0.0; + for (var i = 0; i < 9000; i++) { + final tsec = t / 1000.0; + final v = 1000.0 + + 30.0 * math.sin(2 * math.pi * 0.10 * tsec) + + 20.0 * math.sin(2 * math.pi * 0.25 * tsec); + nn.add(v); + t += v; + ts.add(t); + } + final sd = stddev(nn)!; + final m = hrvFreq(nn, ts, artifactFraction: 0.0); + final v = m.value!; + // Bands land on their injected variances, in ms². + expect(v.lf!, closeTo(450.0, 45.0)); + expect(v.hf!, closeTo(200.0, 20.0)); + // Parseval: the total is the series variance = SDNN². Pre-fix this + // printed 0.1 for a 651 ms² night. + expect(v.total!, closeTo(sd * sd, 0.05 * sd * sd)); + // And the ratio no longer depends on how finely we happened to sample. + final coarse = hrvFreq(nn, ts, artifactFraction: 0.0, oversample: 1.0); + final fine = hrvFreq(nn, ts, artifactFraction: 0.0, oversample: 16.0); + expect(coarse.value!.lfhf!, closeTo(fine.value!.lfhf!, 0.1)); + }); + + test('a band the record cannot RESOLVE is absent, not 0.0', () { + // ULF needs a 24-h record (its own header says so); the gate used to be + // 333 s, and `bandPower` skipped the grid's lowest point, so any session + // of 333–429 s emitted `ulf` as exactly 0.0. + final nn = [for (var i = 0; i < 400; i++) 1000.0 + (i % 7)]; + final ts = []; + var t = 0.0; + for (final v in nn) { + t += v; + ts.add(t); + } + final m = hrvFreq(nn, ts, artifactFraction: 0.0); + expect(m.value!.ulf, isNull); + expect(m.value!.toJson().containsKey('ulf'), isFalse); + }); + test('REGRESSION: a gated HF is not republished through `total`', () { // total used to sum hfRaw back in, so the gated and ungated totals were // bit-identical (0.49944 in both) — the suppression was cosmetic. @@ -80,6 +208,36 @@ void main() { expect(gated.value!.toJson().containsKey('total'), isFalse); expect(gated.note, contains('total')); }); + + test('total NAMES what it summed instead of folding absent bands in as 0', + () { + // an-clinical-3. `total` was `(ulf ?? 0) + (vlf ?? 0) + lf + hfRaw`, and + // ULF can never resolve on a night — _welchBandPower needs 10 cycles at + // 0.0003 Hz = 33 333 s of span and a night is ~28 700 s — so the ULF term + // entered the published total as a literal 0 on every single night, while + // the comment above it claimed total was withheld when an unresolvable + // band was missing. The number is unchanged (0 adds nothing); what + // changes is that the sum now says which bands it is a sum OVER. + final nn = [for (var i = 0; i < 400; i++) 1000.0 + (i % 7)]; + final ts = []; + var t = 0.0; + for (final v in nn) { + t += v; + ts.add(t); + } + final v = hrvFreq(nn, ts, artifactFraction: 0.0).value!; + expect(v.ulf, isNull, reason: 'a 400-beat record cannot resolve ULF'); + expect(v.totalBands, isNotNull); + expect(v.totalBands, isNot(contains('ulf'))); + // The sum is EXACTLY the named bands, with nothing implicit in it. + final named = {'ulf': v.ulf, 'vlf': v.vlf, 'lf': v.lf, 'hf': v.hf}; + var sum = 0.0; + for (final b in v.totalBands!) { + sum += named[b]!; + } + expect(v.total!, closeTo(sum, 1e-9)); + expect(v.toJson()['total_bands'], v.totalBands); + }); }); group('PRSA DC/AC (Bauer 2006)', () { @@ -96,7 +254,11 @@ void main() { expect(dc.value!.capacity, greaterThan(0)); expect(ac.value!.capacity, lessThan(0)); expect(dc.value!.kind, 'DC'); - expect(dc.value!.riskTier, isNotNull); + // HRV-03. Bauer's post-MI mortality tier is GONE from the payload — its + // cut-offs are 24-h Holter ECG in post-MI patients, and on one subject in + // one 9-day window it read 'low' on gen4 (DC 7.6-9.9), 'intermediate' on + // MG and 'high' on WHOOP 5 (DC 1.70): the tier was decided by the strap. + expect(dc.toJson((v) => v.toJson()).toString(), isNot(contains('risk'))); }); test('absent without enough beats', () { expect(decelerationCapacity([1000, 1010, 990]).present, isFalse); @@ -217,7 +379,8 @@ void main() { expect(out.every((d) => d.state == IllnessState.green), isTrue); expect(out.every((d) => d.cusum == null), isTrue); }); - test('REGRESSION: a DEGENERATE (zero-dispersion) baseline abstains instead ' + test( + 'REGRESSION: a DEGENERATE (zero-dispersion) baseline abstains instead ' 'of standardizing against a fabricated 1 bpm scale', () { // 9 identical quantized nights, then a 5 bpm bump. MAD = 0 AND SD = 0, so // there is no dispersion at all. The old `max(1.0, SD)` fallback made @@ -264,7 +427,9 @@ void main() { test('absent under min nights', () { expect(readinessLnRmssd([4.0, 4.1]).present, isFalse); }); - test("rolling mean is prior nights only, doesn't include tonight's own value", () { + test( + "rolling mean is prior nights only, doesn't include tonight's own value", + () { // 3 identical prior nights + a low tonight. baseline mean should be // 4.0 (the prior nights), not 3.5 (which is what you get if tonight's // own drop gets averaged into its own comparison baseline). @@ -273,7 +438,8 @@ void main() { expect(m.present, isTrue); expect(m.value!.rolling7Mean, 4.0); }); - test('REGRESSION: an UNDEFINED baseline SD abstains instead of emitting ' + test( + 'REGRESSION: an UNDEFINED baseline SD abstains instead of emitting ' 'cvPct 0.0 / band "normal"', () { // One prior night => stddev() is null => CV, SWC and the band are // undefined. The metric used to publish cvPct 0.0, swc null and @@ -352,14 +518,16 @@ void main() { group('TRIMP + CTL/ATL/TSB', () { test('Banister TRIMP needs anchors; produces positive load', () { - final none = banisterTrimp([120, 130], restingHr: null, maxHr: 190, sex: Sex.male); + final none = + banisterTrimp([120, 130], restingHr: null, maxHr: 190, sex: Sex.male); expect(none.present, isFalse); final m = banisterTrimp([120, 140, 160], restingHr: 50, maxHr: 190, sex: Sex.male); expect(m.value!, greaterThan(0)); }); - test('REGRESSION: ONE Banister implementation, matching the published ' + test( + 'REGRESSION: ONE Banister implementation, matching the published ' 'sex-specific y = c·e^(b·x)', () { // The two implementations in this file disagreed by 1.5625× (the // top-level one dropped the 0.64/0.86 coefficient entirely) and the @@ -390,12 +558,8 @@ void main() { expect(StrainScorer.banisterY(x, female: true), closeTo(0.86 * math.exp(1.67 * x), 1e-12)); }); - test('Edwards zone-sum is the weighted dot product', () { - // zones [10,5,0,0,0] -> 10*1 + 5*2 = 20 - final m = edwardsTrimp([10, 5, 0, 0, 0]); - expect(m.value!, 20); - }); - test('CTL>ATL after a long steady block, then ATL spikes on a hard day', () { + test('CTL>ATL after a long steady block, then ATL spikes on a hard day', + () { final steady = [for (var i = 0; i < 60; i++) 50.0]; final base = ctlAtlTsb(steady); // both converge to ~50, TSB ~ 0. @@ -408,7 +572,8 @@ void main() { expect(s.value!.tsb, lessThan(0)); }); - test('REGRESSION: one training day does NOT fabricate 42 days of chronic ' + test( + 'REGRESSION: one training day does NOT fabricate 42 days of chronic ' 'load', () { // ctlAtlTsb([500]) used to seed BOTH accumulators at dailyTrimp.first → // ctl 500, atl 500, tsb 0.0: a fully-adapted, perfectly-fresh athlete @@ -449,7 +614,8 @@ void main() { expect(zones.zoneNumber(180.0), 5); }); - test('accumulates duration until next sample and rounds to zone minutes', () { + test('accumulates duration until next sample and rounds to zone minutes', + () { final zoneSet = HeartRateZones.zonesFromMaxHr(200); final time = HeartRateZones.timeInZone([ const HrSample(0, 110), // z1 for 60 s @@ -499,9 +665,13 @@ void main() { final win = i ~/ 300; final inBurst = win == 2 || win == 5; final base = 1000.0; + // Both parts are OSCILLATIONS, not alternations: a perfectly + // alternating series has diff-ACF1 = −1, which the jitter floor now + // (correctly) refuses as pure beat-timing noise, and no real arousal + // burst looks like that. final v = inBurst - ? base + (i.isEven ? 250.0 : -250.0) // ±250 ms whipsaw - : base + (i.isEven ? 8.0 : -8.0); // small ±8 ms wobble + ? base + 250.0 * math.sin(2 * math.pi * i / 4) // fast ±250 ms swing + : base + 16.0 * math.sin(2 * math.pi * i / 12); nn.add(v); t += v; times.add(t); @@ -511,7 +681,7 @@ void main() { // Whole-night RMSSD is dragged way up by the bursts. expect(whole, greaterThan(100), reason: 'whole-night RMSSD inflated by bursts'); - // The stable wobble RMSSD ≈ sqrt((16^2)) ≈ 16 ms; robust median stays low. + // The stable wobble RMSSD ≈ 8 ms; robust median stays low. expect(robust, lessThan(40), reason: 'median-of-windows is robust to a few burst windows'); expect(robust, lessThan(whole / 3), @@ -523,7 +693,8 @@ void main() { final times = []; var t = 0.0; for (var i = 0; i < 1200; i++) { - nn.add(1000.0 + (i.isEven ? 10.0 : -10.0)); + // Slow oscillation, not an alternation — see the burst test above. + nn.add(1000.0 + 10.0 * math.sin(2 * math.pi * i / 12)); t += nn.last; times.add(t); } @@ -578,6 +749,36 @@ void main() { expect(m.present, isTrue); expect(m.value, closeTo(0.0, 1e-9)); }); + + test('HRV-02: no difference is manufactured across a REJECTED beat', () { + // The window cleaner compacts, so differencing straight down its output + // spanned every rejected beat with one invented difference. Here the two + // sides of the ectopic sit 100 ms apart, so the seam Δ is 100 ms while + // every real Δ is 0. THIS is most of the "gen5 reads 2x gen4" gap: on the + // real corpus it inflated the nightly headline by 51-102 % on MG + // (87.7 -> 58.2, 82.9 -> 53.0, 76.9 -> 48.4 ms) and 2-13 % on gen4. + final rr = [900, 900, 900, 200, 1000, 1000, 1000]; + final ts = [for (var i = 0; i < 7; i++) 1000.0 + i * 1000.0]; + final m = sleepSessionWindowedRmssd(rr, ts, startSec: 1, endSec: 301); + expect(m.present, isTrue); + expect(m.value, closeTo(0.0, 1e-9), + reason: '2 runs of flat beats, no seam difference'); + }); + + test('HRV-02: the nightly headline is ABSENT on a jitter-dominated night', + () { + // WHOOP 5 measures ACF1 -0.50/-0.51 on this series even after the + // window cleaner, so the headline 111.8 / 118.7 ms was noise. Absent + // beats plausible: readiness treats a null HRV driver as absent. + final rnd = math.Random(11); + final rr = [ + for (var i = 0; i < 900; i++) 1000 + (rnd.nextDouble() - 0.5) * 120 + ]; + final ts = [for (var i = 0; i < 900; i++) 1000.0 + i * 1000.0]; + final m = sleepSessionWindowedRmssd(rr, ts, startSec: 1, endSec: 1801); + expect(m.present, isFalse); + expect(m.note, contains('rmssd_refused:acf1=')); + }); }); // The CALIBRATION of this scale (what a rest / active / hard / maximal day @@ -585,7 +786,8 @@ void main() { // than on the formula restated. This group keeps only the mechanical // properties of the map itself. group('strain score (0-21 map of TRIMP above the waking baseline)', () { - test('subtracts the quiet-waking baseline for the observed wake window', () { + test('subtracts the quiet-waking baseline for the observed wake window', + () { // 960 waking minutes accrue ~180 TRIMP just by being awake. Charging that // as effort is what put an inactive day at 12.8/21. expect(baselineTrimp(960), closeTo(180.4, 0.5)); @@ -612,7 +814,7 @@ void main() { }); }); - group('StrainScorer (Edwards/Banister TRIMP → 0–100)', () { + group('StrainScorer (Banister TRIMP → 0–100)', () { test('trimpToStrain pins: 0→0, 7200→~100, monotone, 2dp', () { expect(StrainScorer.trimpToStrain(0), 0.0); // ln(7201)/ln(7201)=1 → 100. @@ -634,13 +836,18 @@ void main() { expect(StrainScorer.trimpToStrain(335), lessThan(100.0)); }); - test('REGRESSION: strain integrates PER-SAMPLE durations, not the first ' + test( + 'REGRESSION: strain integrates PER-SAMPLE durations, not the first ' 'inter-sample gap applied to everything', () { const bpmv = 150.0; // (a) 21 samples over 20 min whose FIRST two are 1 s apart — exactly the // sparse stream minSparseReadings admits. sampleDuration was 1 s for all // 21 samples → strain 8.08 instead of ~47. - final tsIrregular = [0, 1, for (var i = 1; i < 20; i++) 1 + i * 63.1]; + final tsIrregular = [ + 0, + 1, + for (var i = 1; i < 20; i++) 1 + i * 63.1 + ]; final bpm21 = List.filled(21, bpmv); final irregular = StrainScorer.strain(bpm21, tsIrregular, maxHR: 190, restingHR: 50)!; @@ -654,8 +861,7 @@ void main() { // (b) The inverse: 700 samples at 1 Hz behind a 300 s leading gap. The // 5-min first gap became every sample's duration → strain 104.25. final tsGap = [0, for (var i = 0; i < 699; i++) 300.0 + i]; - final gapped = StrainScorer.strain( - List.filled(700, bpmv), tsGap, + final gapped = StrainScorer.strain(List.filled(700, bpmv), tsGap, maxHR: 190, restingHR: 50)!; final dense = StrainScorer.strain(List.filled(700, bpmv), [for (var i = 0; i < 700; i++) i.toDouble()], @@ -672,15 +878,14 @@ void main() { expect(durs.reduce(math.max), closeTo(1 / 60.0, 1e-12)); }); - test('Edwards zone weight at %HRR boundaries (RHR=0,reserve=100 → bpm=%HRR)', () { - int w(double pct) => StrainScorer.zoneWeight(pct, 0, 100); - expect(w(49), 0); - expect(w(50), 1); - expect(w(60), 2); - expect(w(70), 3); - expect(w(80), 4); - expect(w(90), 5); - expect(w(100), 5); + test('MOT-11: no HRmax → no number, not a 220−age stand-in', () { + // `strain` used to fall back to defaultMaxHR() = 190 — a 30 y/o's + // ceiling applied to whoever's wrist arrived. The perimeter caught it; + // the source now does. + final bpm = List.filled(700, 150.0); + final ts = [for (var i = 0; i < 700; i++) i.toDouble()]; + expect(StrainScorer.strain(bpm, ts, maxHR: null), isNull); + expect(StrainScorer.strain(bpm, ts, maxHR: 190), isNotNull); }); test('Banister monotonic increasing in intensity', () { @@ -713,7 +918,8 @@ void main() { expect(h2, greaterThan(StrainScorer.tanakaHRmax(30))); }); - test('gating: too few samples → null; spanning ≥600s with ≥20 → computes', () { + test('gating: too few samples → null; spanning ≥600s with ≥20 → computes', + () { // 30 samples but spanning only 30s → fails sparse-span gate → null. final ts30 = [for (var i = 0; i < 30; i++) i.toDouble()]; expect( @@ -735,7 +941,9 @@ void main() { isNull); }); - test('trimpStrain envelope: present/ESTIMATE on enough data, absent otherwise', () { + test( + 'trimpStrain envelope: present/ESTIMATE on enough data, absent otherwise', + () { final ts = [for (var i = 0; i < 700; i++) i.toDouble()]; final m = trimpStrain(List.filled(700, 140), ts, maxHr: 190, restingHr: 50); @@ -747,7 +955,9 @@ void main() { expect(absent.confidence, 0); }); - test('trimpStrain is absent (not fabricated) when maxHr or restingHr is missing', () { + test( + 'trimpStrain is absent (not fabricated) when maxHr or restingHr is missing', + () { final ts = [for (var i = 0; i < 700; i++) i.toDouble()]; final bpm = List.filled(700, 140); final noMax = trimpStrain(bpm, ts, restingHr: 50); @@ -765,8 +975,8 @@ void main() { expect(one.present, isFalse); expect(one.confidence, 0); expect(one.note, 'need_baseline:have=1,need=$readinessLnRmssdMinNights'); - final enough = readinessLnRmssd( - List.generate(readinessLnRmssdMinNights, (i) => 3.5 + i * 0.01)); + final enough = readinessLnRmssd(List.generate( + readinessLnRmssdMinNights, (i) => 3.5 + i * 0.01)); expect(enough.present, isTrue); }); @@ -777,7 +987,8 @@ void main() { final rhr = [for (var i = 0; i < n; i++) 55.0 + (i.isEven ? 0.0 : 1.0)]; final days = illnessCusum(dates, rhr); // First night: have=0 baseline, need=7. - expect(days[0].need, 'need_baseline:have=0,need=$illnessCusumMinBaseline'); + expect( + days[0].need, 'need_baseline:have=0,need=$illnessCusumMinBaseline'); expect(days[0].cusum, isNull); // A night past the minimum baseline is evaluated (no need note). expect(days[illnessCusumMinBaseline].need, isNull); @@ -788,14 +999,16 @@ void main() { // -------------------------------------------- Baevsky Stress Index (#6) group('baevskyStressIndex — direct coverage', () { test('too few clean beats → honest absent', () { - final m = baevskyStressIndex([for (var i = 0; i < 10; i++) 900.0]); + final m = + baevskyStressIndex([for (var i = 0; i < 10; i++) 900.0]); expect(m.present, isFalse); expect(m.value, isNull); expect(m.tier, Tier.estimate); expect(m.note, contains('30 clean beats')); }); - test('a narrow, near-regular RR distribution yields a finite SI + band', () { + test('a narrow, near-regular RR distribution yields a finite SI + band', + () { // ~300 beats around 900 ms with small bounded variation → a well-defined // mode and a non-zero MxDMn, so SI is finite (not the degenerate ÷0 case). final nn = [ @@ -813,14 +1026,17 @@ void main() { expect(v.toJson()['si'], isNotNull); }); - test('a constant RR series has zero range → no valid SI window (absent)', () { + test('a constant RR series has zero range → no valid SI window (absent)', + () { // MxDMn = 0 makes SI degenerate; the impl must return absent, never ∞/0. - final m = baevskyStressIndex([for (var i = 0; i < 300; i++) 900.0]); + final m = + baevskyStressIndex([for (var i = 0; i < 300; i++) 900.0]); expect(m.present, isFalse); expect(m.value, isNull); }); - test('REGRESSION: a NEAR-degenerate RR range abstains instead of reporting ' + test( + 'REGRESSION: a NEAR-degenerate RR range abstains instead of reporting ' 'SI 48780 / "high"', () { // 300 beats alternating 1000/1001 ms — plausible 1 Hz beat-timing // quantization at a steady sleeping HR. The guard was only mxdmnS <= 0, @@ -848,8 +1064,8 @@ void main() { final rr = []; var t = 0.0; for (var i = 0; i < nBeats; i++) { - final v = 900 + - amplitudeMs * math.sin(2 * math.pi * 0.0917 * (t / 1000)); + final v = + 900 + amplitudeMs * math.sin(2 * math.pi * 0.0917 * (t / 1000)); rr.add(v); t += v; } @@ -866,7 +1082,9 @@ void main() { return times; } - test('finds the peak at the guided pace and reports a high ratio/score on a clean paced signal', () { + test( + 'finds the peak at the guided pace and reports a high ratio/score on a clean paced signal', + () { final rr = pacedRr(); final times = beatTimes(rr); final m = cardiacCoherence(rr, times, pacedHz: 0.0917); @@ -881,7 +1099,9 @@ void main() { expect(m.note, contains('matches guided pace')); }); - test('a noisy, unpaced tachogram yields a lower ratio/score than the clean paced one', () { + test( + 'a noisy, unpaced tachogram yields a lower ratio/score than the clean paced one', + () { final rng = math.Random(7); final noisyRr = [ for (var i = 0; i < 180; i++) 900 + (rng.nextDouble() - 0.5) * 200, @@ -922,4 +1142,73 @@ void main() { expect(m.confidence, 0); }); }); + + group('trailing windows are CALENDAR days, not rows', () { + test('a wear gap breaks the illness CUSUM persistence run', () { + // T-14 / B-01. `persistDays: 2` used to mean two RECORDED nights, and the + // caller only passes days that produced a derived row — so an elevated + // Monday and an elevated night three weeks later escalated to red + // "sustained elevation". The 28-day baseline stretched over months the + // same way. + final dates = []; + final rhr = []; + for (var i = 1; i <= 20; i++) { + dates.add('2026-06-${i.toString().padLeft(2, '0')}'); + rhr.add(55.0 + (i % 3)); + } + dates.add('2026-06-21'); + rhr.add(75.0); // elevated + dates.add('2026-07-14'); + rhr.add(75.0); // elevated, but 23 days later + + final out = illnessCusum(dates, rhr); + expect(out[20].state, IllnessState.yellow); + expect(out[21].state, IllnessState.green, + reason: 'pre-fix: red, "sustained elevation" across a 3-week gap'); + // And with no recent baseline left, it says so rather than scoring. + expect(out[21].cusum, isNull); + expect(out[21].need, contains('need_baseline')); + }); + + test('a wear gap clears the ACCUMULATOR, not just the run counters', () { + // an-clinical-2. The gap guard zeroed yellowRun/normalRun but cusum is a + // loop-external accumulator, so an episode's charge survived an + // arbitrarily long gap: 5 illness nights, 70 days off-wrist, then the + // first scorable night back fired yellow at cusum 28.55 on a night + // measured z = -0.12 BELOW baseline — health_screen rendered that as + // "tracking above your own baseline, 0.1 standardised deviations below + // it". A CUSUM is evidence accumulated over CONSECUTIVE observations; + // across a gap there are none, so there is nothing to carry. + String d(int dayOffset) => DateTime.utc(2026, 1, 1) + .add(Duration(days: dayOffset)) + .toIso8601String() + .substring(0, 10); + final dates = []; + final rhr = []; + var off = 0; + for (var i = 0; i < 30; i++) { + dates.add(d(off++)); + rhr.add(55 + 2.0 * math.sin(i.toDouble())); + } + for (var i = 0; i < 5; i++) { + dates.add(d(off++)); + rhr.add(64); // illness episode: ends red with a large accumulator + } + off += 70; // 70-day wear gap + for (var i = 0; i < 14; i++) { + dates.add(d(off++)); + rhr.add(55 + 2.0 * math.sin(i.toDouble())); // fully normal nights + } + final out = illnessCusum(dates, rhr); + expect( + out.sublist(30, 35).map((e) => e.state), contains(IllnessState.red), + reason: 'the episode itself must still fire'); + final back = out.sublist(35); + // Every night back is green, and the first one that CAN be scored starts + // from a cleared accumulator rather than 28.55. + expect(back.every((e) => e.state == IllnessState.green), isTrue); + final firstScored = back.firstWhere((e) => e.cusum != null); + expect(firstScored.cusum, lessThan(1.0)); + }); + }); } diff --git a/test/onehz/coaching_test.dart b/test/onehz/coaching_test.dart index 03dda2c..04fa328 100644 --- a/test/onehz/coaching_test.dart +++ b/test/onehz/coaching_test.dart @@ -1,7 +1,10 @@ -// Coaching surface — synthetic known-answer tests, incl. a regression for the -// physiological-age oversleep bug. Covers PR #11's untested coaching API. -import 'dart:convert'; - +// Coaching surface — synthetic known-answer tests. Covers PR #11's untested +// coaching API. +// +// vo2maxEstimate and physiologicalAge and their tests are GONE (CV-02): the +// first was 15.3·maxHr/rhr with maxHr a constant per user, i.e. k/RHR — the +// RHR chart with the wrong unit on the axis — and the second then counted the +// same RHR twice, once through that VO2max and once directly. import 'package:test/test.dart'; import 'package:openstrap_analytics/src/onehz/types.dart'; import 'package:openstrap_analytics/src/onehz/human/coaching.dart'; @@ -134,26 +137,89 @@ void main() { }); group('recommendedWake', () { - test('90-minute cycle-aligned wake from bedtime', () { - // bed 23:00 = 1380, need 7.5h=27000s=450min → round(450/90)=5 cycles. - // wake = (1380 + 5*90) mod 1440 = 1830 mod 1440 = 390 = 06:30. - final m = recommendedWake(bedtimeMinOfDay: 1380, needSec: 27000); - expect(m.present, isTrue); - expect(m.tier, Tier.estimate); - expect(m.confidence, closeTo(0.55, 1e-9)); - expect(m.value!.wakeMinOfDay, closeTo(390.0, 1e-9)); - }); - - test('cycles floor at 1 for tiny need', () { - // need 30 min → round(30/90)=0 → max(1,0)=1 cycle = 90 min. - final m = recommendedWake(bedtimeMinOfDay: 100, needSec: 1800); - expect(m.value!.wakeMinOfDay, closeTo(190.0, 1e-9)); + test('wake = bedtime + the SAME time in bed the bedtime was backed off', + () { + // an-wellness-2. This used to add round(need/90)*90 SLEEP minutes onto a + // bedtime built from an IN-BED duration, so the pair always described a + // short night: at need 8 h / eff 88 % it gave bed 21:55 and wake 05:25 — + // a 7.50 h span for an 8.00 h need, and 95 min before the 07:00 typical + // wake the bedtime was anchored to. Now the two ends agree exactly, so + // target wake lands back on the user's own typical wake. + // need 8h, eff 88% -> inBed = 28800/0.88 = 32727.27s = 545.4545 min. + final bed = recommendedBedtime( + needSec: 28800, + typicalWakeMinOfDay: 420, // 07:00 + typicalEfficiencyPct: 88, + ); + final wake = recommendedWake( + bedtimeMinOfDay: bed.value!.bedtimeMinOfDay, + needSec: 28800, + typicalEfficiencyPct: 88, + ); + expect(wake.present, isTrue); + expect(wake.tier, Tier.estimate); + expect(wake.confidence, closeTo(0.55, 1e-9)); + expect(wake.value!.wakeMinOfDay, closeTo(420.0, 1e-9)); + }); + + test('INVARIANT: the span is never short of need/efficiency', () { + // The old shortfall was systematic, not unlucky: the efficiency gap is + // need*0.136 (65 min at an 8 h need) while the biggest possible cycle + // round-UP was 45 min. Grid it. + for (final needH in [6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 10.0, 11.0]) { + for (final effPct in [75.0, 80.0, 85.0, 88.0, 92.0, 99.0]) { + final needSec = needH * 3600.0; + final bed = recommendedBedtime( + needSec: needSec, + typicalWakeMinOfDay: 420, + typicalEfficiencyPct: effPct, + ).value!.bedtimeMinOfDay; + final wake = recommendedWake( + bedtimeMinOfDay: bed, + needSec: needSec, + typicalEfficiencyPct: effPct, + ).value!.wakeMinOfDay; + final span = (wake - bed) % 1440.0; + final required = needSec / (effPct / 100.0) / 60.0; + expect(span, greaterThanOrEqualTo(required - 1e-6), + reason: 'need ${needH}h at $effPct% eff: ' + 'span $span min < in-bed $required min'); + } + } + }); + + test('more need never moves target wake EARLIER', () { + // The cycle rounding used to make a 0.5 h need increase shift target wake + // 55 min LATER (05:25 -> 06:20 at a fixed 07:00 typical wake) by crossing + // a 90-minute boundary. Monotone now, and pinned at typical wake. + double wakeFor(double needH) { + final bed = recommendedBedtime( + needSec: needH * 3600.0, + typicalWakeMinOfDay: 420, + typicalEfficiencyPct: 88, + ).value!.bedtimeMinOfDay; + return recommendedWake( + bedtimeMinOfDay: bed, + needSec: needH * 3600.0, + typicalEfficiencyPct: 88, + ).value!.wakeMinOfDay; + } + + for (final h in [7.0, 7.5, 8.0, 8.5, 9.0]) { + expect(wakeFor(h), closeTo(420.0, 1e-9)); + } }); test('wraps around midnight into [0,1440)', () { - // bed 23:30=1410, need ~7.5h → 5 cycles=450 → 1860 mod 1440 = 420. - final m = recommendedWake(bedtimeMinOfDay: 1410, needSec: 27000); - expect(m.value!.wakeMinOfDay, closeTo(420.0, 1e-9)); + // bed 23:30 = 1410, need 7.5h at 88% -> inBed 511.36 min -> 1921.36 mod + // 1440 = 481.36 (08:01). + final m = recommendedWake( + bedtimeMinOfDay: 1410, + needSec: 27000, + typicalEfficiencyPct: 88, + ); + expect(m.value!.wakeMinOfDay, closeTo(481.3636, 1e-3)); + expect(m.value!.wakeMinOfDay, greaterThanOrEqualTo(0.0)); expect(m.value!.wakeMinOfDay, lessThan(1440.0)); }); }); @@ -203,7 +269,8 @@ void main() { // low-recovery day asked for a number the user had already passed before // getting out of bed. A recover ceiling must sit above a rest day (2–4) // and below a typical active day (8–11). - final m = strainTarget(recovery0to100: 20, ctl: null, atl: null, tsb: null); + final m = + strainTarget(recovery0to100: 20, ctl: null, atl: null, tsb: null); expect(m.value!.band, 'recover'); expect(m.value!.targetMin, closeTo(0, 1e-9)); expect(m.value!.targetMax, greaterThan(4.0)); @@ -214,30 +281,36 @@ void main() { // 21 is a maximal day. A push ceiling above ~19 is not a target, it is a // dare — the old band topped out at 18 on a scale whose real ceiling was // ~16 for a marathon. - final m = strainTarget(recovery0to100: 90, ctl: null, atl: null, tsb: null); + final m = + strainTarget(recovery0to100: 90, ctl: null, atl: null, tsb: null); expect(m.value!.band, 'push'); expect(m.value!.targetMin, closeTo(13, 1e-9)); expect(m.value!.targetMax, lessThanOrEqualTo(19.0)); }); - test('fatigue is judged on the ATL:CTL RATIO, not a raw TRIMP difference', () { + test('fatigue is judged on the ATL:CTL RATIO, not a raw TRIMP difference', + () { // ctl/atl arrive as raw daily TRIMP (hundreds), but the thresholds were // sized as if they were 0–21 strain points: `atl − ctl > 10` fired on // ordinary week-to-week noise. 320 vs 300 is a 6.7 % lift — not fatigue — // yet the old absolute test (diff 20 > 10) shrank the window for it. - final noise = strainTarget(recovery0to100: 70, ctl: 300, atl: 320, tsb: null); + final noise = + strainTarget(recovery0to100: 70, ctl: 300, atl: 320, tsb: null); expect(noise.value!.targetMin, closeTo(9, 1e-9)); expect(noise.value!.targetMax, closeTo(14, 1e-9)); // A genuine 30 % acute lift over chronic still lowers the window. - final real = strainTarget(recovery0to100: 70, ctl: 100, atl: 130, tsb: null); + final real = + strainTarget(recovery0to100: 70, ctl: 100, atl: 130, tsb: null); expect(real.value!.targetMin, closeTo(8, 1e-9)); expect(real.value!.targetMax, closeTo(12, 1e-9)); }); - test('freshness is judged on TSB relative to CTL, not a raw TRIMP value', () { + test('freshness is judged on TSB relative to CTL, not a raw TRIMP value', + () { // tsb 6 against a chronic load of 300 is 2 % — noise, not freshness. - final noise = strainTarget(recovery0to100: 70, ctl: 300, atl: 294, tsb: 6); + final noise = + strainTarget(recovery0to100: 70, ctl: 300, atl: 294, tsb: 6); expect(noise.value!.targetMax, closeTo(14, 1e-9)); // tsb 20 against a chronic load of 100 is a real 20 % taper. @@ -246,7 +319,8 @@ void main() { }); test('no load history leaves the recovery window untouched', () { - final m = strainTarget(recovery0to100: 70, ctl: null, atl: null, tsb: null); + final m = + strainTarget(recovery0to100: 70, ctl: null, atl: null, tsb: null); expect(m.value!.targetMin, closeTo(9, 1e-9)); expect(m.value!.targetMax, closeTo(14, 1e-9)); // A zero chronic load must not divide by zero into an adjustment. @@ -263,105 +337,6 @@ void main() { }); }); - group('vo2maxEstimate', () { - test('Uth ratio 15.3×maxHr/restingHr on a known value', () { - final m = vo2maxEstimate(restingHr: 50, maxHr: 190, sex: Sex.male, age: 30); - expect(m.present, isTrue); - expect(m.tier, Tier.estimate); - expect(m.confidence, closeTo(0.45, 1e-9)); - expect(m.value!, closeTo(15.3 * 190 / 50, 1e-6)); // 58.14 - }); - - test('absent when maxHr <= restingHr (no divide-by-invalid)', () { - expect( - vo2maxEstimate(restingHr: 190, maxHr: 180, sex: Sex.male, age: 30) - .present, - isFalse); - }); - - test('absent on null restingHr / null maxHr (no divide-by-zero)', () { - expect( - vo2maxEstimate(restingHr: null, maxHr: 190, sex: Sex.male, age: 30) - .present, - isFalse); - expect( - vo2maxEstimate(restingHr: 50, maxHr: null, sex: Sex.male, age: 30) - .present, - isFalse); - }); - }); - - group('physiologicalAge — sleep deviation (regression)', () { - PhysioAge run(double h) => physiologicalAge( - chronologicalAge: 30, - sex: Sex.male, - vo2max: null, - restingHr: null, - rmssd: null, - sleepDurationH: h, - sleepEfficiency: null, - dailySteps: null, - ).value!; - - test('oversleep does NOT make you younger', () { - expect(run(10.0).physioAge, greaterThan(30.0)); - }); - test('undersleep ages you', () { - expect(run(5.0).physioAge, greaterThan(30.0)); - }); - test('optimal ~7.5h is neutral', () { - expect(run(7.5).physioAge, closeTo(30.0, 0.01)); - }); - test('symmetry: 5h and 10h age you by the same amount', () { - // Both are 2.5h from the 7.5h optimum → identical penalty. - expect(run(5.0).physioAge, closeTo(run(10.0).physioAge, 1e-9)); - }); - - test('baseline case: better-than-average biomarkers lower physio age', () { - final m = physiologicalAge( - chronologicalAge: 40, - sex: Sex.male, - vo2max: 50, // above 35 → subtracts - restingHr: 48, // below 60 → subtracts - rmssd: 60, // above 35 → subtracts - sleepDurationH: 7.5, // optimal → neutral - sleepEfficiency: 94, // above 88 → subtracts - dailySteps: 12000, // above 7000 → subtracts - ); - expect(m.present, isTrue); - expect(m.tier, Tier.estimate); - expect(m.value!.physioAge, lessThan(40.0)); - expect(m.value!.deltaYears, lessThan(0.0)); - expect(m.value!.deltaYears, - closeTo(m.value!.physioAge - 40.0, 1e-9)); - }); - - test('physio age is clamped to [18,95]', () { - final young = physiologicalAge( - chronologicalAge: 18, - sex: Sex.female, - vo2max: 80, - restingHr: 40, - rmssd: 120, - sleepDurationH: 7.5, - sleepEfficiency: 99, - dailySteps: 20000, - ); - expect(young.value!.physioAge, greaterThanOrEqualTo(18.0)); - final old = physiologicalAge( - chronologicalAge: 95, - sex: Sex.male, - vo2max: 10, - restingHr: 100, - rmssd: 5, - sleepDurationH: 3, - sleepEfficiency: 60, - dailySteps: 0, - ); - expect(old.value!.physioAge, lessThanOrEqualTo(95.0)); - }); - }); - group('journalCorrelations', () { test('insufficient sample (<2 per side) is gated as insufficient', () { // Only one tagged day for "coffee" → cannot compare. @@ -385,29 +360,31 @@ void main() { }); test('clear positive correlation is detected and marked meaningful', () { - // "alcohol" days have clearly lower recovery than untagged days. + // "alcohol" days have clearly lower recovery than untagged days. FIVE per + // side: at two per side there are only six label assignments, so no + // permutation p can reach the FDR bar (see the RD-10 group below). final journal = [ - const JournalDay('d0', {'alcohol'}), - const JournalDay('d1', {'alcohol'}), - const JournalDay('d2', {}), - const JournalDay('d3', {}), + for (var i = 0; i < 5; i++) JournalDay('d$i', const {'alcohol'}), + for (var i = 5; i < 10; i++) JournalDay('d$i', const {}), ]; - final dates = ['d0', 'd1', 'd2', 'd3']; + final dates = [for (var i = 0; i < 10; i++) 'd$i']; final outcomes = >{ - 'recovery': [40, 42, 80, 82], // tagged mean 41, untagged mean 81 + 'recovery': [40, 41, 42, 43, 44, 80, 81, 82, 83, 84], }; final out = journalCorrelations( journal: journal, dates: dates, outcomes: outcomes); - final eff = - out.firstWhere((c) => c.tag == 'alcohol').effects.single; + final eff = out.firstWhere((c) => c.tag == 'alcohol').effects.single; expect(eff.insufficient, isFalse); expect(eff.meaningful, isTrue); - expect(eff.delta, closeTo(41 - 81, 1e-9)); // −40 - expect(eff.higherSide, 'untagged'); // untagged (non-alcohol) recovers more - expect(eff.nTagged, 2); - expect(eff.nUntagged, 2); + expect(eff.delta, closeTo(42 - 82, 1e-9)); // −40 + expect( + eff.higherSide, 'untagged'); // untagged (non-alcohol) recovers more + expect(eff.nTagged, 5); + expect(eff.nUntagged, 5); expect(eff.pctChange, isNotNull); expect(eff.pctChange!.abs(), greaterThanOrEqualTo(3.0)); + expect(eff.q, isNotNull); + expect(eff.q!, lessThanOrEqualTo(0.10)); }); test('nulls are dropped from both sides before comparing', () { @@ -440,100 +417,6 @@ void main() { }); }); - // ------------------------------------------------------------------------- - // REGRESSION: physiologicalAge must ABSTAIN with no physiology, and must - // report the inputs it ACTUALLY used. - // ------------------------------------------------------------------------- - group('physiologicalAge — honesty envelope (regression)', () { - test('every physiological input null => ABSENT, not "your age"', () { - // PRE-FIX: score started at chronologicalAge, nothing moved it, and the - // function returned a PRESENT metric (physioAge 30, delta 0, conf 0.35) - // claiming six inputs it had never seen. - final m = physiologicalAge( - chronologicalAge: 30, - sex: Sex.male, - vo2max: null, - restingHr: null, - rmssd: null, - sleepDurationH: null, - sleepEfficiency: null, - dailySteps: null, - ); - expect(m.present, isFalse); - expect(m.value, isNull); - expect(m.confidence, 0); - expect(m.toJson()['value'], '—'); - expect(m.inputs_used, ['profile']); - }); - - test('inputs_used lists only the inputs actually supplied', () { - // PRE-FIX this was a hardcoded six-entry list in EVERY partial case. - final m = physiologicalAge( - chronologicalAge: 30, - sex: Sex.male, - vo2max: null, - restingHr: 55, - rmssd: null, - sleepDurationH: 7.5, - sleepEfficiency: null, - dailySteps: null, - ); - expect(m.present, isTrue); - expect(m.inputs_used, ['profile', 'resting_hr', 'sleep_duration']); - expect(m.inputs_used, isNot(contains('vo2max'))); - expect(m.inputs_used, isNot(contains('rmssd'))); - expect(m.inputs_used, isNot(contains('steps'))); - }); - - test('confidence scales with how much physiology went in', () { - Metric build(int n) => physiologicalAge( - chronologicalAge: 40, - sex: Sex.male, - vo2max: n >= 1 ? 50 : null, - restingHr: n >= 2 ? 48 : null, - rmssd: n >= 3 ? 60 : null, - sleepDurationH: n >= 4 ? 7.5 : null, - sleepEfficiency: n >= 5 ? 94 : null, - dailySteps: n >= 6 ? 12000 : null, - ); - expect(build(6).confidence, greaterThan(build(1).confidence)); - expect(build(6).inputs_used, hasLength(7)); // profile + 6 - }); - }); - - // ------------------------------------------------------------------------- - // REGRESSION: vo2maxEstimate must not divide by a zero resting HR. - // ------------------------------------------------------------------------- - group('vo2maxEstimate — zero resting HR (regression)', () { - test('restingHr == 0 (the off-skin sentinel) ABSTAINS, never Infinity', () { - // PRE-FIX `maxHr <= restingHr` did not catch it: 15.3 * (190/0) produced - // value: Infinity, which Metric.toJson emits raw and jsonEncode throws on. - final m = vo2maxEstimate(restingHr: 0, maxHr: 190, sex: Sex.male, age: 30); - expect(m.present, isFalse); - expect(m.value, isNull); - expect(() => jsonEncode(m.toJson()), returnsNormally); - }); - - test('a negative or non-finite resting HR also abstains', () { - expect( - vo2maxEstimate(restingHr: -5, maxHr: 190, sex: Sex.male, age: 30) - .present, - isFalse); - expect( - vo2maxEstimate( - restingHr: double.nan, maxHr: 190, sex: Sex.male, age: 30) - .present, - isFalse); - }); - - test('a valid pair still computes', () { - final m = vo2maxEstimate(restingHr: 50, maxHr: 190, sex: Sex.male, age: 30); - expect(m.present, isTrue); - expect(m.value!.isFinite, isTrue); - expect(() => jsonEncode(m.toJson()), returnsNormally); - }); - }); - // ------------------------------------------------------------------------- // REGRESSION: journalCorrelations needs a dispersion test, and must not // index an outcome list by dates.length without checking. @@ -567,6 +450,27 @@ void main() { }); test('a large, well-separated effect is still meaningful', () { + final out = journalCorrelations( + journal: [ + for (var i = 0; i < 5; i++) JournalDay('d$i', const {'alcohol'}), + for (var i = 5; i < 10; i++) JournalDay('d$i', const {}), + ], + dates: [for (var i = 0; i < 10; i++) 'd$i'], + outcomes: const { + 'recovery': [40, 41, 42, 43, 44, 80, 81, 82, 83, 84] + }, + ); + final eff = out.firstWhere((c) => c.tag == 'alcohol').effects.single; + expect(eff.meaningful, isTrue); + expect(eff.cohensD!.abs(), greaterThan(0.5)); + }); + + // ----------------------------------------------------------------------- + // RD-10. The verdict used to be `|Δ%| ≥ 3 AND |d| ≥ 0.5` with no test and no + // multiplicity correction, which under the null called a cell meaningful + // 65.5 % of the time at 2 vs 2 and 42.9 % at 3 vs 20. + // ----------------------------------------------------------------------- + test('2 vs 2 cannot be meaningful however cleanly it separates', () { final out = journalCorrelations( journal: const [ JournalDay('d0', {'alcohol'}), @@ -580,8 +484,67 @@ void main() { }, ); final eff = out.firstWhere((c) => c.tag == 'alcohol').effects.single; - expect(eff.meaningful, isTrue); + expect(eff.cohensD!.abs(), greaterThan(0.5), + reason: 'the old d bar IS cleared'); + expect(eff.pctChange!.abs(), greaterThanOrEqualTo(3.0), + reason: 'the old percentage bar IS cleared'); + // Six label assignments, two of them at least as extreme => p ~ 1/3. No + // amount of separation can beat that, which is the whole point. + expect(eff.p!, greaterThan(0.3)); + expect(eff.meaningful, isFalse); + }); + + test('3 tagged vs 20 untagged: a d ≥ 0.5 gap is not enough on its own', () { + final untagged = [ + for (var i = 0; i < 2; i++) ...[52, 54, 56, 58, 60, 62, 64, 66, 68, 70] + ]; + final out = journalCorrelations( + journal: [ + for (var i = 0; i < 3; i++) JournalDay('d$i', const {'late_meal'}), + for (var i = 3; i < 23; i++) JournalDay('d$i', const {}), + ], + dates: [for (var i = 0; i < 23; i++) 'd$i'], + outcomes: { + 'recovery': [64, 66, 68, ...untagged], + }, + ); + final eff = out.firstWhere((c) => c.tag == 'late_meal').effects.single; expect(eff.cohensD!.abs(), greaterThan(0.5)); + expect(eff.pctChange!.abs(), greaterThanOrEqualTo(3.0)); + expect(eff.meaningful, isFalse, + reason: 'three days is not evidence (p=${eff.p}, q=${eff.q})'); + }); + + test('the FDR correction runs over the whole tag × outcome grid', () { + // One real effect, seven pure-noise tags on the same days. The noise tags + // are tested too, so the real one has to survive the correction. + final dates = [for (var i = 0; i < 12; i++) 'd$i']; + final out = journalCorrelations( + journal: [ + for (var i = 0; i < 12; i++) + JournalDay('d$i', { + if (i < 6) 'alcohol', + if (i.isEven) 'noise_a', + if (i % 3 == 0) 'noise_b', + }), + ], + dates: dates, + outcomes: const { + 'recovery': [40, 41, 42, 43, 44, 45, 80, 81, 82, 83, 84, 85], + 'hrv': [55, 56, 57, 58, 59, 60, 55, 56, 57, 58, 59, 60], + }, + ); + final alcohol = out.firstWhere((c) => c.tag == 'alcohol'); + final rec = alcohol.effects.firstWhere((e) => e.outcome == 'recovery'); + expect(rec.meaningful, isTrue); + expect(rec.q, isNotNull); + // Every cell carries its q, and no noise tag survives. + for (final tc in out) { + for (final e in tc.effects) { + if (tc.tag == 'alcohol' && e.outcome == 'recovery') continue; + expect(e.meaningful, isFalse, reason: '${tc.tag}/${e.outcome}'); + } + } }); test('two constant sides with only 2 days each are NOT meaningful', () { diff --git a/test/onehz/device_test.dart b/test/onehz/device_test.dart new file mode 100644 index 0000000..1dfd7f9 --- /dev/null +++ b/test/onehz/device_test.dart @@ -0,0 +1,36 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + group('device family dispatch seam', () { + test('known ids parse, everything else is unknown', () { + expect(deviceFamilyOf('gen4'), DeviceFamily.gen4); + expect(deviceFamilyOf('gen5'), DeviceFamily.gen5); + for (final id in [null, '', 'GEN4', 'gen6', 'whoop4', 'imported']) { + expect(deviceFamilyOf(id), isNull, reason: 'id=$id must be unknown'); + } + }); + + test('id round-trips', () { + for (final f in DeviceFamily.values) { + expect(deviceFamilyOf(deviceFamilyId(f)), f); + } + }); + + test('unknown family gets NO constants — never gen4 as a fallback', () { + const k = {DeviceFamily.gen4: 230, DeviceFamily.gen5: 210}; + expect(calibrationFor(k, 'gen4'), 230); + expect(calibrationFor(k, 'gen5'), 210); + expect(calibrationFor(k, null), isNull); + expect(calibrationFor(k, 'gen6'), isNull); + // a family the metric itself has not calibrated is also a refusal + expect(calibrationFor(const {DeviceFamily.gen5: 1}, 'gen4'), isNull); + }); + + test('refusal note is machine-readable', () { + expect(unknownFamilyNote(null), 'unknown_device_family:id=none'); + expect(unknownFamilyNote(''), 'unknown_device_family:id=none'); + expect(unknownFamilyNote('gen6'), 'unknown_device_family:id=gen6'); + }); + }); +} diff --git a/test/onehz/foundations_test.dart b/test/onehz/foundations_test.dart index 1199437..935d2cf 100644 --- a/test/onehz/foundations_test.dart +++ b/test/onehz/foundations_test.dart @@ -8,34 +8,88 @@ void main() { final cfg = Baselines.hrvCfg; // min5 max250 floor5 hlB14 hlS21 test('first valid night seeds at the value, floor spread, calibrating', () { - final s = Baselines.update(null, 60.0, cfg); + final s = Baselines.update(null, 60.0, cfg)!; expect(s.baseline, 60.0); expect(s.spread, cfg.floorSpread); expect(s.nValid, 1); expect(s.status, BaselineStatus.calibrating); }); - test('out-of-range first night seeds at midpoint, nValid 0', () { - final s = Baselines.update(null, 999.0, cfg); // > maxVal - expect(s.baseline, (cfg.minVal + cfg.maxVal) / 2.0); - expect(s.nValid, 0); - expect(s.nightsSinceUpdate, 1); + test('out-of-range first night yields NO baseline (was: the midpoint)', () { + // an-clinical-1. This used to return a state whose baseline was + // (minVal + maxVal) / 2 = 127.5 ms — a number nobody measured — against + // which deviation() reported z = -13 for an ordinary 45 ms night. + expect(Baselines.update(null, 999.0, cfg), isNull); // > maxVal + expect(Baselines.update(null, null, cfg), isNull); + }); + + test('foldHistory over no usable night is absent, not an invented centre', + () { + expect(Baselines.foldHistory(const [], cfg), isNull); + expect(Baselines.foldHistory(const [null, null], cfg), isNull); + // A single usable night later in the series still builds a real state. + final s = Baselines.foldHistory([null, 52.0], cfg)!; + expect(s.baseline, 52.0); + expect(s.nValid, 1); + }); + + test('deviation abstains on a zero-night state', () { + const empty = BaselineState( + baseline: 127.5, + spread: 5.0, + nValid: 0, + nightsSinceUpdate: 0, + status: BaselineStatus.calibrating); + expect(Baselines.deviation(45.0, empty), isNull); }); test('hard-outlier night past early life is SEEN but NOT folded', () { // Build a settled (non-young, nValid>=8) flat baseline at 60. final vals = [for (var i = 0; i < 10; i++) 60.0]; - final settled = Baselines.foldHistory(vals, cfg); + final settled = Baselines.foldHistory(vals, cfg)!; expect(settled.nValid, 10); expect(settled.baseline, closeTo(60.0, 1e-9)); // spread is at the floor (flat history), so a value > 5*floor away is hard-rejected. final before = settled.baseline; - final after = Baselines.update(settled, 60.0 + 6 * cfg.floorSpread, cfg); + final after = Baselines.update(settled, 60.0 + 6 * cfg.floorSpread, cfg)!; expect(after.baseline, before, reason: 'hard outlier not folded'); - expect(after.nValid, settled.nValid, reason: 'nValid unchanged on reject'); + expect(after.nValid, settled.nValid, + reason: 'nValid unchanged on reject'); + // STAT-10b. `nightsSinceUpdate` counts nights the baseline DID NOT MOVE — + // the only thing computeStatus reads it for. A rejected night moved + // nothing, so it increments like every other hold. It used to reset to 0 + // here while an out-of-range night incremented, so the same counter meant + // two different things depending on which gate fired. + expect(after.nightsSinceUpdate, settled.nightsSinceUpdate + 1); + final outOfRange = Baselines.update(settled, 999.0, cfg)!; + expect(outOfRange.nightsSinceUpdate, after.nightsSinceUpdate, + reason: 'both hold branches mean the same thing'); }); - test('early-life fast adapt: a high seed is pulled toward reality in days', () { + // STAT-10. The spread EWMA is an EWMA of ABSOLUTE DEVIATION, so the + // deviation has to be measured against the baseline the night ARRIVED at. + // Measuring against the already-updated baseline shrinks it by exactly + // (1 − λ_B) for anything inside the Winsor band — 4.83 % at halfLifeB = 14, + // 20.63 % while young — which tightens the spread and inflates every z. + test('spread deviation is measured against the OLD baseline (STAT-10)', () { + final settled = Baselines.foldHistory( + [for (var i = 0; i < 20; i++) 60.0], cfg)!; + // Inside the Winsor band (spread is at the floor 5, band is ±15). + final after = Baselines.update(settled, 60.0 + 10.0, cfg)!; + final ls = Baselines.lambda(cfg.halfLifeS); + final want = + math.max(cfg.floorSpread, ls * 10.0 + (1.0 - ls) * settled.spread); + expect(after.spread, closeTo(want, 1e-9)); + // The shipped version used |value − newBaseline| = (1 − λ_B)·10. + final lb = Baselines.lambda(cfg.halfLifeB); + final shrunk = math.max(cfg.floorSpread, + ls * ((1 - lb) * 10.0) + (1.0 - ls) * settled.spread); + expect(after.spread, greaterThan(shrunk)); + expect((1 - lb), closeTo(0.9517, 1e-4), reason: 'the 4.83 % shrink'); + }); + + test('early-life fast adapt: a high seed is pulled toward reality in days', + () { // Seed high at 90, then feed true lower nights at 55. Young (nValid<8) uses // halfLife 3 and a suspended hard-outlier gate, so it tracks down fast. var s = Baselines.update(null, 90.0, cfg); @@ -44,23 +98,23 @@ void main() { } // With earlyHalfLifeB=3 (λ≈0.206) over 4 nights from 90 toward 55, // it should drop well below the midpoint, proving the anti-anchoring fix. - expect(s.baseline, lessThan(80.0)); + expect(s!.baseline, lessThan(80.0)); expect(s.baseline, greaterThan(55.0)); }); test('deviation z = (v-baseline)/(1.253*spread)', () { - final s = BaselineState( + const s = BaselineState( baseline: 60.0, spread: 5.0, nValid: 14, nightsSinceUpdate: 0, status: BaselineStatus.trusted); - final d = Baselines.deviation(60.0 + 1.253 * 5.0, s); + final d = Baselines.deviation(60.0 + 1.253 * 5.0, s)!; expect(d.z, closeTo(1.0, 1e-9)); expect(d.delta, closeTo(1.253 * 5.0, 1e-9)); // A value comfortably inside ±σ is in-normal-range. - expect(Baselines.deviation(62.0, s).inNormalRange, isTrue); - final d2 = Baselines.deviation(60.0 + 2 * 1.253 * 5.0, s); + expect(Baselines.deviation(62.0, s)!.inNormalRange, isTrue); + final d2 = Baselines.deviation(60.0 + 2 * 1.253 * 5.0, s)!; expect(d2.inNormalRange, isFalse); }); @@ -72,27 +126,20 @@ void main() { }); test('skip-and-hold on null night increments nightsSinceUpdate', () { - final seeded = Baselines.update(null, 60.0, cfg); - final held = Baselines.update(seeded, null, cfg); + final seeded = Baselines.update(null, 60.0, cfg)!; + final held = Baselines.update(seeded, null, cfg)!; expect(held.baseline, seeded.baseline); expect(held.nValid, seeded.nValid); expect(held.nightsSinceUpdate, 1); }); - - test('trailing-30 fallback: mean + sample SD, σ floor, internal spread', () { - final vals = [50.0, 60.0, 70.0]; // mean 60, SD=10 - final s = Baselines.rollingMeanSD(vals, cfg); - expect(s.baseline, closeTo(60.0, 1e-9)); - // SD=10 > floor σ (5); stored as SD/1.253. - expect(s.spread, closeTo(10.0 / 1.253, 1e-9)); - expect(s.nValid, 3); - }); }); group('RR artifact correction (Lipponen-Tarvainen)', () { test('clean physiological series classifies all normal', () { // 60 beats around 1000 ms with small +/-15 ms wobble (HRV). - final rr = [for (var i = 0; i < 60; i++) 1000 + (i.isEven ? 15 : -15)]; + final rr = [ + for (var i = 0; i < 60; i++) 1000 + (i.isEven ? 15 : -15) + ]; final r = correctRr(rr); expect(r.cleanFraction, closeTo(1.0, 1e-9)); expect(r.droppedCount, 0); @@ -118,7 +165,8 @@ void main() { ]; } - test('REGRESSION: a clean physiological RSA record is NOT flagged — the ' + test( + 'REGRESSION: a clean physiological RSA record is NOT flagged — the ' 'quartile deviation is taken on the SIGNED dRR series ' '(Lipponen-Tarvainen 2019)', () { // Taking the QD of |dRR| folds the symmetric ±dRR distribution onto one @@ -156,7 +204,8 @@ void main() { expect(r.cleanFraction, lessThan(1.0)); }); - test('flags EXACTLY one injected isolated ectopic and spline-corrects it', () { + test('flags EXACTLY one injected isolated ectopic and spline-corrects it', + () { final rr = [for (var i = 0; i < 60; i++) 1000.0]; rr[30] = 500; // single short extra beat (isolated) final r = correctRr(rr); @@ -183,30 +232,6 @@ void main() { }); }); - group('PPG SQI gate (Elgendi + Orphanidou)', () { - test('rejects off-skin / out-of-range HR', () { - expect(ppgTrust([1, 2, 3, 2, 1], 0).trusted, isFalse); // off-skin - expect(ppgTrust([1, 2, 3, 2, 1], 220).trusted, isFalse); // hr too high - }); - test('rejects flatline PPG even with plausible HR', () { - final r = ppgTrust([5, 5, 5, 5, 5], 60); - expect(r.trusted, isFalse); - expect(r.reasons, contains('flatline')); - }); - test('accepts a skewed, in-range window', () { - // right-skewed window (systolic-upstroke-like). - final w = [0, 0, 0, 1, 4, 9, 2, 0, 0, 0]; - final r = ppgTrust(w, 60); - expect(r.skewness, greaterThan(0)); - expect(r.trusted, isTrue); - }); - test('RR physiological-plausibility rule', () { - expect(rrPhysiologicallyPlausible([900, 950, 1000]), isTrue); - expect(rrPhysiologicallyPlausible([900, 250]), isFalse); // <300 - expect(rrPhysiologicallyPlausible([300, 1000]), isFalse); // ratio>3 - }); - }); - group('robust baseline', () { test('median+MAD with Iglewicz-Hoaglin outlier flag', () { final b = robustBaseline([10, 11, 9, 10, 12, 8, 10]); @@ -226,44 +251,12 @@ void main() { expect(robustBaseline([1, 2], minValid: 3).sufficient, isFalse); expect(robustBaseline([1, 2, 3], minValid: 3).sufficient, isTrue); }); - test('lambda from half-life: half-life => 0.5 effective at one HL', () { - final lam = lambdaFromHalfLife(1); // 1 - 2^-1 = 0.5 - expect(lam, closeTo(0.5, 1e-12)); - }); - test('gap-aware EWMA converges and flags gaps', () { - final t = [0, 1000, 2000, 3000, 100000]; - final v = [10, 10, 10, 10, 20]; - final e = gapAwareEwma(t, v, halfLifeMs: 1000, maxGapMs: 10000); - expect(e.first.value, 10); - expect(e.last.gap, isTrue); // the big jump is a flagged gap - // clamped: one late sample can't fully take over (<= 0.5 weight). - expect(e.last.value, lessThan(20)); - expect(e.last.value, greaterThan(10)); - }); - test('REGRESSION: gap-aware EWMA never extrapolates outside the data on a ' - 'duplicate or non-monotonic timestamp', () { - // dt <= 0 made lambda = 1 - 2^(-dt/H) NEGATIVE, so the update ran - // BACKWARDS: [0,1000,500] with values [10,10,20] produced 5.857 — below - // every input (all >= 10). - final e = gapAwareEwma([0, 1000, 500], [10, 10, 20], halfLifeMs: 1000); - expect(e.length, 3); - for (final p in e) { - expect(p.value, greaterThanOrEqualTo(10.0)); - expect(p.value, lessThanOrEqualTo(20.0)); - } - // No elapsed time => no new weight => the estimate does not move. - expect(e.last.value, closeTo(10.0, 1e-12)); - // Duplicate timestamps behave the same way. - final dup = gapAwareEwma([0, 0, 0], [10, 50, 50], halfLifeMs: 1000); - expect(dup.map((p) => p.value), everyElement(closeTo(10.0, 1e-12))); - }); - test('MDC gate: small change suppressed, large surfaced', () { + test('MDC: a dispersion-free baseline has no detectable change', () { final b = robustBaseline([10, 11, 9, 10, 12, 8, 10, 11, 9, 10]); - expect(changeExceedsMdc(0.1, b), isFalse); - expect(changeExceedsMdc(50, b), isTrue); - // no dispersion known => never claim a change. - final flat = robustBaseline([5, 5, 5]); - expect(changeExceedsMdc(99, flat), isFalse); + expect(mdc(b)!, greaterThan(0.1)); + expect(mdc(b)!, lessThan(50)); + // no dispersion known => no MDC => never claim a change. + expect(mdc(robustBaseline([5, 5, 5])), isNull); }); }); @@ -290,12 +283,106 @@ void main() { [const FusionInput(1, 1, trusted: false, label: 'x')]); expect(r.value, isNull); }); - test('quality->variance and GUM weighted-sum uncertainty', () { - expect(varianceFromQuality(1.0, 4), 4); - expect(varianceFromQuality(0.5, 4), 8); // worse quality => more variance - // y = 0.5*x1 + 0.5*x2, each u=2 => u_c = sqrt(1+1)=1.414 - expect(gumWeightedSumUncertainty([0.5, 0.5], [2, 2])!, - closeTo(1.41421, 1e-4)); + }); + + group('RR correction — the beat clock is WALL CLOCK', () { + test('a dropped run still advances nnTimesMs', () { + // T-12. Dropped runs used only to bump a counter, never `t`, so the two + // sides of every dropped run were spliced together: 299.0 s of real + // record came out as a 294.0 s span. cvhr_per_hour divides by that span, + // so the apnea screen read high on exactly the noisy nights that needed + // the correction. + final rr = [for (var i = 0; i < 300; i++) 1000.0]; + for (var k = 100; k < 105; k++) { + rr[k] = 250.0; // 5-beat artifact run -> dropped, never interpolated + } + var real = 0.0; + for (final v in rr) { + real += v; + } + final c = correctRr(rr); + expect(c.droppedCount, greaterThan(1)); + // times[0] is the END of the first interval, so the span is the total + // elapsed time minus that first interval — exactly, no compaction. + final span = c.nnTimesMs.last - c.nnTimesMs.first; + expect(span, closeTo(real - rr.first, 1e-6)); + }); + + test('a spline-corrected isolated beat advances by the REAL interval', () { + // an-clinical-4, the sibling the dropped-run fix above missed. Every + // ~60th beat is a MISSED detection: one ~2000 ms interval where two + // ~1000 ms ones belong. The spline emits ~1000 ms as the NN value (right) + // but the clock used to advance by that 1000 too (wrong) — deleting ~1 s + // of record per corrected beat, 1.6 % of a night at this rate, which + // inflated cvhr_per_hour and shortened hrvFreq's spanSec. + final rr = [ + // The anomaly sits at i % 60 == 30 so the first and last beats are + // normal — the span below is then exactly total-minus-first-interval. + for (var i = 0; i < 3600; i++) i % 60 == 30 ? 2000.0 : 1000.0, + ]; + var real = 0.0; + for (final v in rr) { + real += v; + } + final c = correctRr(rr); + expect(c.correctedCount, greaterThan(50), + reason: 'spline path exercised'); + final span = c.nnTimesMs.last - c.nnTimesMs.first; + // Exact wall clock: total elapsed minus the first interval. Before the + // fix this came out ~59 s (1.6 %) short of it. + expect(span, closeTo(real - rr.first, 1e-6)); + // The emitted NN value is still the interpolated one, not the raw 2000. + expect(c.nn.reduce((a, b) => a > b ? a : b), lessThan(1500.0)); + }); + + test('a SENSOR DROPOUT is not spliced out when rrTsMs is passed', () { + // HRV-01. The two tests above fix runs *we* drop. This is the one we + // cannot see from `rrMs` alone: beats the band never reported at all, so + // the interval is absent from the array and Σrr walks straight over the + // hole. Measured on 13 real nights, the cumsum span was 0.13–0.87 of the + // true rec_ts span, and the Lomb-Scargle spectrum built on it moved + // LF/HF across the sympatho-vagal line (whoop-mg 2026-08-12: 0.65 with + // the cumsum clock, 1.53 with this one; whoop-4 2026-08-13: 1.32 -> 2.03). + const rr = 860.0; // realistic, and NOT a whole second + const nBeats = 4000; + const holeSec = 600; + const holeAt = 2000; + final rrMs = []; + final rrTsMs = []; + var wallMs = 1700000000000.0; + for (var i = 0; i < nBeats; i++) { + wallMs += rr; + if (i == holeAt) wallMs += holeSec * 1000; // sensor reported nothing + rrMs.add(rr); + // rr_ts_ms = rec_ts*1000 on 100 % of real rows — WHOLE SECONDS. + rrTsMs.add((wallMs / 1000).floorToDouble() * 1000); + } + + final spliced = correctRr(rrMs); + final anchored = correctRr(rrMs, rrTsMs: rrTsMs); + expect(spliced.nn.length, anchored.nn.length, + reason: 'the clock must not change which beats are kept'); + + final splicedSpan = spliced.nnTimesMs.last - spliced.nnTimesMs.first; + final anchoredSpan = anchored.nnTimesMs.last - anchored.nnTimesMs.first; + // Without the timestamps the hole simply does not exist. + expect(splicedSpan, closeTo((nBeats - 1) * rr, 1e-6)); + // With them it is exactly one hole longer — to within the 1 s + // quantisation of rr_ts_ms, which is the honest floor here and is not + // smoothed away. + expect(anchoredSpan - splicedSpan, closeTo(holeSec * 1000, 1000)); + + // And the re-anchor fires ONCE. Whole-second timestamps make every + // in-run wall step 0 or 1000 ms against an 860 ms interval; if the guard + // were on accumulated drift instead of the local step, quantisation + // alone would re-anchor on most beats and the axis would just become + // rec_ts, throwing away the only sub-second information we have. + var jumps = 0; + for (var i = 1; i < anchored.nnTimesMs.length; i++) { + final step = anchored.nnTimesMs[i] - anchored.nnTimesMs[i - 1]; + if ((step - rr).abs() > 1e-6) jumps++; + } + expect(jumps, 1, reason: 'one dropout -> one re-anchor'); }); }); } diff --git a/test/onehz/hr_recovery_test.dart b/test/onehz/hr_recovery_test.dart index bcab072..05d87c9 100644 --- a/test/onehz/hr_recovery_test.dart +++ b/test/onehz/hr_recovery_test.dart @@ -1,4 +1,6 @@ // HRR (heart-rate recovery) — synthetic known-answer tests. +import 'dart:math' as math; + import 'package:test/test.dart'; import 'package:openstrap_analytics/src/onehz/types.dart'; import 'package:openstrap_analytics/src/onehz/workout/hr_recovery.dart'; @@ -32,10 +34,143 @@ void main() { expect(m.present, isFalse); }); + test('with tsSec the PEAK window is clock time, not array positions', () { + // an-motion-4. Supplying tsSec used to convert only the +60 s recovery + // point; the peak window and the ±3 s median window stayed positional. + // On this 20 s-spaced tail a 30-POSITION peak window spanned 580 s of + // clock, so "peak at exercise end" came from nine minutes earlier: + // pre-fix peak 180, drop 69, confidence 0.90 (bonus included). The bout + // actually ended at 151 bpm, and a real 30 s window holds 151-152. + final hr = []; + final ts = []; + var t = 0; + for (var i = 0; i < 30; i++) { + hr.add(180 - i); // 180 -> 151 over 580 s + ts.add(t); + t += 20; + } + for (var i = 0; i < 200; i++) { + hr.add(151 - i < 100 ? 100 : 151 - i); + ts.add(t); + t += 1; + } + final m = hrRecovery(hr, endIndex: 29, tsSec: ts, peakWindowSec: 30); + expect(m.present, isTrue, reason: m.note); + expect(m.value!.peakHr, 152, reason: 'the two samples inside 30 s'); + expect(m.value!.dropBpm, 41); + // The +0.2 timestamped bonus is withheld: 20 s of tail is not the 30 s + // asked for, so the timestamps proved the window short, not sound. + expect(m.confidence, closeTo(0.7, 1e-9)); + }); + + test('a dense timestamped tail still earns the +0.2 bonus', () { + final hr = []; + final ts = []; + for (var i = 0; i < 30; i++) { + hr.add(170); + ts.add(i); + } + for (var s = 1; s <= 70; s++) { + hr.add((170 - (40 * s / 60)).round()); + ts.add(29 + s); + } + final m = hrRecovery(hr, endIndex: 29, tsSec: ts, peakWindowSec: 30); + expect(m.present, isTrue, reason: m.note); + expect(m.confidence, closeTo(0.9, 1e-9)); + }); + test('tail too short to reach +60s → absent', () { final hr = [for (var i = 0; i < 40; i++) 160 - i]; final m = hrRecovery(hr, endIndex: 10, recoverySec: 60); expect(m.present, isFalse); }); }); + + // ------------------------------------------------------------------------- + // CV-08 — the exponential recovery time constant, alongside HRR-60. + // ------------------------------------------------------------------------- + group('hrRecovery — tau', () { + /// A 1 Hz tail: 30 s of steady exercise at [start] bpm, then + /// `asymptote + amp·exp(-t/tau)` for [postSec] seconds. + ({List hr, List ts, int end}) tail({ + required double asymptote, + required double amp, + required double tau, + int postSec = 200, + double noise = 0, + }) { + final hr = []; + final ts = []; + var t = 0; + for (var i = 0; i < 30; i++) { + hr.add((asymptote + amp).round()); + ts.add(t++); + } + final end = hr.length - 1; + for (var s = 1; s <= postSec; s++) { + final v = asymptote + + amp * math.exp(-s / tau) + + (noise == 0 ? 0.0 : noise * math.sin(s / 3.0)); + hr.add(v.round()); + ts.add(t++); + } + return (hr: hr, ts: ts, end: end); + } + + test('recovers a known time constant from a clean tail', () { + final d = tail(asymptote: 70, amp: 100, tau: 55); + final m = hrRecovery(d.hr, endIndex: d.end, tsSec: d.ts); + expect(m.present, isTrue); + final v = m.value!; + expect(v.tauSec, isNotNull); + expect(v.tauSec!, closeTo(55, 3)); + expect(v.tauAmpBpm!, closeTo(100, 4)); + expect(v.tauAsymptoteBpm!, closeTo(70, 3)); + // The intensity it started from ships WITH it — tau is not comparable + // across intensities and the number that conditions it is not optional. + expect(v.tauStartHr!, closeTo(170, 4)); + expect(v.tauResidualRatio!, lessThan(0.05)); + expect(v.tauAbsenceReason, isNull); + // And it did NOT replace HRR-60. + expect(v.dropBpm, greaterThan(0)); + }); + + test('a short tail abstains with a reason, it does not extrapolate', () { + // The +75 s tail the caller slices today. Half a window is not a curve. + final d = tail(asymptote: 70, amp: 100, tau: 55, postSec: 75); + final v = hrRecovery(d.hr, endIndex: d.end, tsSec: d.ts).value!; + expect(v.tauSec, isNull); + expect(v.tauAbsenceReason, startsWith('tau_tail_short')); + expect(v.dropBpm, greaterThan(0), reason: 'HRR-60 still publishes'); + }); + + test('without timestamps there is no clock, so there is no tau', () { + final d = tail(asymptote: 70, amp: 100, tau: 55); + final v = hrRecovery(d.hr, endIndex: d.end).value!; + expect(v.tauSec, isNull); + expect(v.tauAbsenceReason, 'tau_needs_timestamps'); + }); + + test('a tail that is not one clean exponential is rejected', () { + // Noise the size of the swing itself: the residual gate is the feature. + final d = tail(asymptote: 70, amp: 20, tau: 55, noise: 12); + final v = hrRecovery(d.hr, endIndex: d.end, tsSec: d.ts).value!; + expect(v.tauSec, isNull); + expect(v.tauAbsenceReason, 'tau_fit_rejected'); + }); + + test('the residual gate is a FRACTION of the swing, not bpm', () { + // Same shape, same relative noise, ten times the amplitude. A bpm + // threshold would pass one and fail the other; the same recovery on a + // strap with a different noise floor is still the same recovery. + final small = tail(asymptote: 70, amp: 20, tau: 50, noise: 0.4); + final big = tail(asymptote: 70, amp: 200, tau: 50, noise: 4.0); + final a = + hrRecovery(small.hr, endIndex: small.end, tsSec: small.ts).value!; + final b = hrRecovery(big.hr, endIndex: big.end, tsSec: big.ts).value!; + expect(a.tauSec, isNotNull); + expect(b.tauSec, isNotNull); + expect(a.tauResidualRatio!, closeTo(b.tauResidualRatio!, 0.03)); + }); + }); } diff --git a/test/onehz/human_test.dart b/test/onehz/human_test.dart index a8399df..617929f 100644 --- a/test/onehz/human_test.dart +++ b/test/onehz/human_test.dart @@ -31,7 +31,8 @@ void main() { final v = m.value!; expect(v.sjlHours, closeTo(2.0, 0.15)); expect(v.absHours, closeTo(2.0, 0.15)); - expect(v.sjlHours, greaterThan(0), reason: 'weekend runs later => positive'); + expect(v.sjlHours, greaterThan(0), + reason: 'weekend runs later => positive'); }); test('insufficient free nights => absent', () { @@ -68,7 +69,10 @@ void main() { for (var d = 0; d < 7; d++) { regular.addAll(day()); } - final mReg = sleepRegularityIndex(regular, epochsPerDay: epd); + // sleepRegularityIndex was a SECOND, gap-blind Phillips SRI (A-21) and + // has been deleted; phillipsSri is the live one and takes a validity mask + // so holes cannot fabricate concordance. + final mReg = phillipsSri(regular, epd); expect(mReg.present, isTrue); expect(mReg.value!.sri, closeTo(100.0, 0.001), reason: 'identical days => perfect 24h concordance'); @@ -79,28 +83,11 @@ void main() { final shifted = d.isEven ? day() : day().map((b) => !b).toList(); irregular.addAll(shifted); } - final mIrr = sleepRegularityIndex(irregular, epochsPerDay: epd); + final mIrr = phillipsSri(irregular, epd); expect(mIrr.value!.sri, lessThan(mReg.value!.sri)); expect(mIrr.value!.sri, lessThan(40)); }); - test('forgiving streak survives one grace miss but not two', () { - // met,met,MISS,met,met with grace=1 => one run of length 4, alive. - final s1 = forgivingStreak([true, true, false, true, true], grace: 1); - expect(s1.current, 4); - expect(s1.graceUsed, 1); - expect(s1.alive, isTrue); - - // met,met,MISS,MISS,met grace=1 => second miss breaks; trailing run = 1. - final s2 = forgivingStreak([true, true, false, false, true], grace: 1); - expect(s2.current, 1); - expect(s2.best, 2, reason: 'first run had 2 met days before the break'); - - // No grace: a single miss breaks immediately. - final s0 = forgivingStreak([true, true, false, true], grace: 0); - expect(s0.best, 2); - expect(s0.current, 1); - }); }); group('sleep debt — honest when no free night', () { @@ -217,17 +204,39 @@ void main() { test('top-of-window value => high percentile', () { // 30 days centred ~50 with spread; tonight = 70 is near the top. final hist = List.generate(30, (i) => 40.0 + i); // 40..69 - final m = percentileOfYou(70, hist); + final m = percentileOfYou(70, hist, better: Better.higher); expect(m.present, isTrue); expect(m.value!.percentile, greaterThan(95)); expect(m.value!.label, 'among your best'); }); + test( + 'the label follows `better`, so RHR is not congratulated for a bad night', + () { + // an-wellness-1. The band ladder used to be a bare pct->string map, so + // the RHR detail screen printed the user's WORST-ever resting HR as + // "among your best" and their best-ever as "among your lowest". + final nights = List.generate(29, (i) => 54.0 + (i % 5)); // 54..58 + final worst = percentileOfYou(72, nights, better: Better.lower); + expect(worst.value!.percentile, 100.0); + expect(worst.value!.label, 'among your worst'); + final best = percentileOfYou(50, nights, better: Better.lower); + expect(best.value!.percentile, 0.0); + expect(best.value!.label, 'among your best'); + // Same numbers, higher-is-better metric: the verdicts swap. + expect(percentileOfYou(72, nights, better: Better.higher).value!.label, + 'among your best'); + // No good end of the scale: position, not verdict. + expect(percentileOfYou(72, nights, better: Better.neither).value!.label, + 'among your highest'); + }); + test('record gated by MDC: tiny beat is NOT a record, big beat is', () { final hist = List.generate(30, (i) => 40.0 + (i % 10)); // spread final priorMax = hist.reduce((a, b) => a > b ? a : b); // Barely beats the max => within MDC noise => NOT a record. - final small = personalRecord(priorMax + 0.01, hist, better: Better.higher); + final small = + personalRecord(priorMax + 0.01, hist, better: Better.higher); expect(small.value!.isRecord, isFalse); // Clearly beats it. final big = personalRecord(priorMax + 50, hist, better: Better.higher); @@ -236,13 +245,14 @@ void main() { }); test('short history => absent', () { - final m = percentileOfYou(10, [1, 2, 3]); + final m = percentileOfYou(10, [1, 2, 3], better: Better.higher); expect(m.present, isFalse); }); }); group('glass-box readiness + deterministic narrative', () { - test('one driver off => narrative names that driver, breakdown complete', () { + test('one driver off => narrative names that driver, breakdown complete', + () { // HRV tanks tonight; everything else is dead-on the personal median. final hrvHist = List.generate(20, (i) => 4.0 + (i % 5) * 0.02); final rhrHist = List.generate(20, (i) => 55.0 + (i % 5) * 0.2); @@ -251,7 +261,10 @@ void main() { final inputs = [ GlassBoxInput( - label: 'hrv', value: 3.0, history: hrvHist, weight: wHrv), // way low + label: 'hrv', + value: 3.0, + history: hrvHist, + weight: wHrv), // way low GlassBoxInput( label: 'rhr', value: 55.4, @@ -259,10 +272,16 @@ void main() { weight: wRhr, lowerIsBetter: true), GlassBoxInput( - label: 'resp', value: 14.2, history: respHist, weight: wResp, + label: 'resp', + value: 14.2, + history: respHist, + weight: wResp, lowerIsBetter: true), GlassBoxInput( - label: 'temp', value: 0.04, history: tempHist, weight: wTemp, + label: 'temp', + value: 0.04, + history: tempHist, + weight: wTemp, lowerIsBetter: true), ]; // ignore: deprecated_member_use_from_same_package @@ -280,13 +299,44 @@ void main() { expect(v.score, lessThan(50)); }); - test('a sub-MDC mover is never named as a driver', () { - // All inputs essentially at their median => no driver clears MDC. + test('RD-09: a real night off baseline names its driver', () { + // The user's own nightly resting HR from whoop-4.db (lowest 30-min mean, + // 00:00–06:00 IST). Baseline of seven, tonight the eighth. + const rhr = [57.73, 57.07, 60.87, 56.90, 60.40, 55.97, 67.67]; + const tonight = 60.93; // +3.20 bpm on a median of 57.73 + // Robust scale of that window is 2.61 bpm, so the old gate — mdc(), which + // with no measured typical error is 2.77 × that same scale — stood at + // 7.23 bpm. NOTHING in this user's real spread reaches it, so + // `drivers` was empty and the narrative said "nothing moved beyond your + // normal day-to-day noise" every single night. The smallest worthwhile + // change is 1.30 bpm, and this night clears it. + final inputs = [ + const GlassBoxInput( + label: 'rhr', + value: tonight, + history: rhr, + weight: wRhr, + lowerIsBetter: true), + ]; + // ignore: deprecated_member_use_from_same_package + final m = glassBoxReadiness(inputs); + final v = m.value!; + expect(v.breakdown.single.beyondUsualSpread, isTrue); + expect(v.drivers, isNotEmpty); + expect(v.drivers.first.label, 'rhr'); + expect(v.narrative.toLowerCase(), isNot(contains('noise'))); + }); + + test('a mover inside the usual spread is never named as a driver', () { + // All inputs essentially at their median => nothing clears the SWC. final hist = List.generate(20, (i) => 50.0 + (i % 5) * 0.1); final inputs = [ GlassBoxInput(label: 'hrv', value: 50.2, history: hist, weight: wHrv), GlassBoxInput( - label: 'rhr', value: 50.2, history: hist, weight: wRhr, + label: 'rhr', + value: 50.2, + history: hist, + weight: wRhr, lowerIsBetter: true), ]; // ignore: deprecated_member_use_from_same_package @@ -300,7 +350,8 @@ void main() { final inputs = [ GlassBoxInput(label: 'hrv', value: 64, history: hist, weight: wHrv), // temp has no history => dropped + reweighted, not zeroed. - GlassBoxInput(label: 'temp', value: 0.0, history: const [], weight: wTemp), + GlassBoxInput( + label: 'temp', value: 0.0, history: const [], weight: wTemp), ]; // ignore: deprecated_member_use_from_same_package final m = glassBoxReadiness(inputs); @@ -313,17 +364,16 @@ void main() { }); // --------------------------------------------------------------------------- - group('PLAUSIBILITY on real ../whoop_hist.jsonl (single-night pieces only)', () { + group('PLAUSIBILITY on real ../whoop_hist.jsonl (single-night pieces only)', + () { final histFile = File('../whoop_hist.jsonl'); test('real HR/RR feed the human layer without crashing', () { if (!histFile.existsSync()) { markTestSkipped('whoop_hist.jsonl not found beside the repo'); return; } - final lines = histFile - .readAsLinesSync() - .where((l) => l.trim().isNotEmpty) - .toList(); + final lines = + histFile.readAsLinesSync().where((l) => l.trim().isNotEmpty).toList(); final hr = []; final rrMs = []; for (final line in lines) { @@ -361,14 +411,15 @@ void main() { expect(m, isNotNull); expect(Tier.all.contains(m.tier), isTrue); // ignore: avoid_print - print('REAL human-layer plausibility: realRHR=${realRhr.toStringAsFixed(1)} ' + print( + 'REAL human-layer plausibility: realRHR=${realRhr.toStringAsFixed(1)} ' 'realRMSSD=${realRmssd.toStringAsFixed(1)} ' 'state=${m.present ? m.value!.state : "absent"} ' '(baseline SYNTHETIC — capture too short for multi-night)'); // percentile-of-you against a synthetic 30-day window using the real RHR. final win = List.generate(30, (i) => realRhr + (i - 15) * 0.3); - final p = percentileOfYou(realRhr, win); + final p = percentileOfYou(realRhr, win, better: Better.lower); expect(p.present, isTrue); expect(p.value!.percentile, inInclusiveRange(0, 100)); }); @@ -432,4 +483,48 @@ void main() { expect(m.value!.typeLabel, contains('evening')); }); }); + + group('serialised envelopes are always jsonEncode-able', () { + test('an under-baselined glass-box input is ABSENT, never NaN', () { + // T-11 / A-00 — the worst finding in the R1 sweep. An input with fewer + // than `minHistory` days was still pushed into `breakdown` carrying + // `percentileOfYou: double.nan`; `round6` passed NaN straight through and + // edge's `jsonEncode(buildCrossDayBundle(...))` THREW, so a catch + // discarded the ENTIRE cross-day bundle — illness, anomaly, CTL/ATL/TSB, + // chronotype, sleep coach, VO2max, every percentile — for every user from + // day 3 to day 8, and permanently for anyone with a sparse input. + final inputs = [ + GlassBoxInput( + label: 'hrv', + value: 60, + history: List.generate(20, (i) => 50.0 + i), + weight: 0.40), + GlassBoxInput( + label: 'temp', + value: 0.5, + history: const [0.1, 0.2], // only 2 days + weight: 0.12, + lowerIsBetter: true), + ]; + // ignore: deprecated_member_use_from_same_package + final m = glassBoxReadiness(inputs); + final json = m.toJson((v) => v.toJson()); + expect(() => jsonEncode(json), returnsNormally); + + final breakdown = (json['value'] as Map)['breakdown'] as List; + final temp = breakdown.firstWhere((b) => (b as Map)['label'] == 'temp') + as Map; + expect(temp['percentile_of_you'], isNull); + expect(temp['used'], isFalse); + // Machine-readable reason, the package convention. + expect(temp['note'], 'need_baseline:have=2,need=7'); + }); + + test('round6 encodes a non-finite as null, not NaN', () { + expect(round6(double.nan), isNull); + expect(round6(double.infinity), isNull); + expect(round6(1.23456789), 1.234568); + expect(() => jsonEncode({'x': round6(double.nan)}), returnsNormally); + }); + }); } diff --git a/test/onehz/journal_numeric_correlations_test.dart b/test/onehz/journal_numeric_correlations_test.dart index 6378efa..56388cf 100644 --- a/test/onehz/journal_numeric_correlations_test.dart +++ b/test/onehz/journal_numeric_correlations_test.dart @@ -6,6 +6,8 @@ // than a difference-of-means does: a correlation over a handful of days is // close to meaningless, and that is exactly when it looks most convincing. +import 'dart:math' as math; + import 'package:openstrap_analytics/onehz.dart'; import 'package:test/test.dart'; @@ -41,9 +43,10 @@ JournalNumericEffect _effect( List out, String field, String outcome, -) => out.firstWhere((e) => e.field == field).effects.firstWhere( - (e) => e.outcome == outcome, -); +) => + out.firstWhere((e) => e.field == field).effects.firstWhere( + (e) => e.outcome == outcome, + ); void main() { group('spearmanRho', () { @@ -84,6 +87,8 @@ void main() { final caffeine = [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6]; final rmssd = [for (final c in caffeine) 80.0 - 6 * c]; final out = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: [ for (var i = 0; i < dates.length; i++) JournalNumericDay(dates[i], {'caffeine': caffeine[i]}), @@ -109,12 +114,16 @@ void main() { // not surface as a finding just because rho happens to be 1. final dates = _dates(5); final out = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: [ for (var i = 0; i < 5; i++) JournalNumericDay(dates[i], {'water': (i + 1).toDouble()}), ], dates: dates, - outcomes: {'readiness': [for (var i = 0; i < 5; i++) 50.0 + i]}, + outcomes: { + 'readiness': [for (var i = 0; i < 5; i++) 50.0 + i] + }, ); final e = _effect(out, 'water', 'readiness'); expect(e.insufficient, isTrue); @@ -138,7 +147,9 @@ void main() { ), ], dates: dates, - outcomes: {'rmssd': [for (var i = 0; i < 10; i++) 60.0 + i]}, + outcomes: { + 'rmssd': [for (var i = 0; i < 10; i++) 60.0 + i] + }, ); final e = _effect(out, 'caffeine', 'rmssd'); expect(e.n, 5, reason: 'only the days the field was actually recorded'); @@ -170,13 +181,53 @@ void main() { // A field that wanders independently of the outcome must come back not // meaningful — the confidence interval straddles zero. final dates = _dates(20); - const field = [4, 1, 3, 2, 5, 3, 1, 4, 2, 5, - 3, 2, 4, 1, 5, 2, 3, 4, 1, 5]; + const field = [ + 4, + 1, + 3, + 2, + 5, + 3, + 1, + 4, + 2, + 5, + 3, + 2, + 4, + 1, + 5, + 2, + 3, + 4, + 1, + 5 + ]; // The same twenty outcome values, permuted to a rank correlation of // exactly 0 against the field above — so this test fails if the gate // ever starts calling an unrelated field a finding. - const outcome = [53, 47, 50, 49, 49, 51, 49, 48, 51, 50, - 52, 52, 48, 52, 47, 53, 51, 50, 48, 53]; + const outcome = [ + 53, + 47, + 50, + 49, + 49, + 51, + 49, + 48, + 51, + 50, + 52, + 52, + 48, + 52, + 47, + 53, + 51, + 50, + 48, + 53 + ]; final out = journalNumericCorrelations( journal: [ for (var i = 0; i < dates.length; i++) @@ -198,12 +249,15 @@ void main() { expect(e.rhoHigh!, greaterThan(0)); }); - test('a misaligned outcome series is reported, not truncated or thrown', () { + test('a misaligned outcome series is reported, not truncated or thrown', + () { final dates = _dates(10); final out = journalNumericCorrelations( journal: _days('water', [for (var i = 0; i < 10; i++) i.toDouble()]), dates: dates, - outcomes: {'rhr': const [50.0, 51.0]}, + outcomes: { + 'rhr': const [50.0, 51.0] + }, ); final e = _effect(out, 'water', 'rhr'); expect(e.insufficient, isTrue); @@ -215,7 +269,9 @@ void main() { final out = journalNumericCorrelations( journal: _days('water', [for (var i = 0; i < 14; i++) 2.0]), dates: dates, - outcomes: {'readiness': [for (var i = 0; i < 14; i++) 50.0 + i]}, + outcomes: { + 'readiness': [for (var i = 0; i < 14; i++) 50.0 + i] + }, ); final e = _effect(out, 'water', 'readiness'); expect(e.insufficient, isTrue); @@ -253,12 +309,16 @@ void main() { List run(int n) { final dates = _dates(n); return journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: [ for (var i = 0; i < n; i++) JournalNumericDay(dates[i], {'caffeine': i.toDouble()}), ], dates: dates, - outcomes: {'rmssd': [for (var i = 0; i < n; i++) 90.0 - 5 * i]}, + outcomes: { + 'rmssd': [for (var i = 0; i < n; i++) 90.0 - 5 * i] + }, ).single.effects; } @@ -293,6 +353,8 @@ void main() { }; final permissive = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: journal, dates: dates, outcomes: outcomes, @@ -301,6 +363,8 @@ void main() { expect(permissive.meaningful, isTrue); final strict = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: journal, dates: dates, outcomes: outcomes, @@ -324,6 +388,8 @@ void main() { // Default floor of 8 refuses five days outright. expect( journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: journal, dates: dates, outcomes: outcomes, @@ -334,6 +400,8 @@ void main() { // Lowered below the pair count, rho is computed and — because 5 > 3 — // still carries an interval. final e = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: journal, dates: dates, outcomes: outcomes, @@ -347,18 +415,31 @@ void main() { // Four pairs is where the interval gets absurdly wide but still exists // — and being unable to exclude zero is exactly the right answer there. expect(e.rhoLow!, lessThan(0), reason: 'five days cannot clear zero'); - expect(e.meaningful, isFalse); + // STAT-08. The DISPLAY interval (Bonett & Wright) and the GATE + // (permutation p) are now two different tests, and this is where they + // part company: five days in perfect rank order has an exact two-sided + // permutation p of 2/5! = 0.0167, which clears 0.05, while the B-W + // interval — a continuous-data simulation result applied to ordinal + // journal fields, evaluated at the observed r — cannot. The exact test is + // the honest one; `minN` (8 in production, overridden to 5 here) is what + // keeps five-day findings off the screen, not a conservative SE. + expect(e.p!, closeTo(2 / 120, 0.01)); + expect(e.meaningful, isTrue); // At three the standard error is undefined outright, so there is no // interval at all and therefore no verdict. final three = _dates(3); final e3 = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, journal: [ for (var i = 0; i < 3; i++) JournalNumericDay(three[i], {'water': i.toDouble()}), ], dates: three, - outcomes: {'readiness': [for (var i = 0; i < 3; i++) 50.0 + i]}, + outcomes: { + 'readiness': [for (var i = 0; i < 3; i++) 50.0 + i] + }, minN: 3, ).single.effects.single; expect(e3.rho, closeTo(1.0, 1e-9)); @@ -370,6 +451,191 @@ void main() { ); }); + // ----------------------------------------------------------------------- + // MIND-01 — Benjamini-Hochberg over the returned grid. + // ----------------------------------------------------------------------- + + test('a grid of pure noise produces per-test hits and NO findings', () { + // The whole reason the correction is not optional. Twenty unrelated + // fields against one outcome, all noise: at a per-test 0.05 gate about + // one of them is a "finding" every time, for every user, forever. Under + // BH the same grid publishes nothing. + const n = 40; + final dates = _dates(n); + final rng = math.Random(1); + final journal = [ + for (var i = 0; i < n; i++) + JournalNumericDay(dates[i], { + for (var f = 0; f < 20; f++) 'f$f': rng.nextDouble() * 5, + }), + ]; + final out = journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: { + 'readiness': [ + for (var i = 0; i < n; i++) 50.0 + rng.nextDouble() * 10 + ], + }, + ); + final effects = [for (final f in out) ...f.effects]; + expect(effects, hasLength(20)); + expect( + effects.any((e) => e.p != null && e.p! < 0.05), + isTrue, + reason: 'if no per-test hit turns up here the test has stopped ' + 'exercising the thing it guards', + ); + expect( + effects.every((e) => !e.meaningful), + isTrue, + reason: 'pure noise must publish nothing', + ); + for (final e in effects) { + expect(e.q!, greaterThanOrEqualTo(e.p!), reason: 'q never below p'); + } + }); + + test('a family of one is the identity — q equals p', () { + final dates = _dates(12); + final caffeine = [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6]; + final out = journalNumericCorrelations( + // MIND-02: this test is about the statistic, not the alignment. + fieldLagDays: const {}, + journal: [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay(dates[i], {'caffeine': caffeine[i]}), + ], + dates: dates, + outcomes: { + 'rmssd': [for (final c in caffeine) 80.0 - 6 * c] + }, + ); + final e = _effect(out, 'caffeine', 'rmssd'); + expect(e.q, closeTo(e.p!, 1e-12)); + expect(e.meaningful, isTrue, reason: 'a real relationship still lands'); + }); + + // ----------------------------------------------------------------------- + // MIND-04 — a 0/1 field is a tick box, not a dose. + // ----------------------------------------------------------------------- + + test('a 0/1 field routes to a difference of means, not to rho', () { + // Habits are custom journal fields with max == 1, so this is the habit + // path. "On the 9 days you did this, HRV was 4 ms higher." + const n = 18; + final dates = _dates(n); + final flag = [for (var i = 0; i < n; i++) i.isEven ? 1.0 : 0.0]; + // +4 ms on the days it was ticked, with real within-group spread. + final hrv = [ + for (var i = 0; i < n; i++) 60.0 + (flag[i] == 1 ? 4.0 : 0.0) + (i % 3), + ]; + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < n; i++) + JournalNumericDay(dates[i], {'meditate': flag[i]}), + ], + dates: dates, + outcomes: {'rmssd': hrv}, + ); + final e = _effect(out, 'meditate', 'rmssd'); + expect(e.binary, isTrue); + expect(e.rho, isNull, reason: 'a two-valued field has no dose to rank'); + expect(e.slopePerUnit, isNull); + expect(e.rhoLow, isNull); + expect(e.nWith, 9); + expect(e.nWithout, 9); + expect(e.delta!, closeTo(4.0, 1e-9)); + expect(e.cohensD!, greaterThan(0.5)); + expect(e.insufficient, isFalse); + expect(e.meaningful, isTrue); + }); + + test('a ticked habit with no real difference is shippable as "not yet"', + () { + // Not insufficient — we ran the comparison and it came back nothing. + // "No difference detectable yet" needs that distinction to be printable. + const n = 20; + final dates = _dates(n); + final flag = [for (var i = 0; i < n; i++) i.isEven ? 1.0 : 0.0]; + final rng = math.Random(11); + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < n; i++) + JournalNumericDay(dates[i], {'stretch': flag[i]}), + ], + dates: dates, + outcomes: { + 'readiness': [ + for (var i = 0; i < n; i++) 50.0 + rng.nextDouble() * 8 + ], + }, + ); + final e = _effect(out, 'stretch', 'readiness'); + expect(e.binary, isTrue); + expect(e.insufficient, isFalse); + expect(e.meaningful, isFalse); + expect(e.delta, isNotNull); + }); + + test('a 0/1 field with too few days on one side gives no verdict', () { + const n = 12; + final dates = _dates(n); + final out = journalNumericCorrelations( + journal: [ + for (var i = 0; i < n; i++) + JournalNumericDay(dates[i], {'rare': i < 2 ? 1.0 : 0.0}), + ], + dates: dates, + outcomes: { + 'rhr': [for (var i = 0; i < n; i++) 50.0 + i] + }, + ); + final e = _effect(out, 'rare', 'rhr'); + expect(e.insufficient, isTrue); + expect(e.meaningful, isFalse); + }); + + // STAT-03. A permutation test's null distribution is DISCRETE. At exactly + // n = 8 with a 3/5 split the label vector has C(8,3) = 56 assignments, so + // the smallest two-sided p is 1/56 = 0.0179 — and after the BH correction + // over a family of 4 that is q = 0.071, above 0.05. The test CANNOT be + // published however perfect the separation. At n = 9 it can + // (1/126 × 4 = 0.032). What shipped was silence with no reason attached. + test('an unreachable test abstains and says how many days it needs', () { + List run(int n) { + final dates = _dates(n); + // Perfectly separated: every ticked day is 20 ms above every other. + final flag = [for (var i = 0; i < n; i++) i < 3 ? 1.0 : 0.0]; + return journalNumericCorrelations( + fieldLagDays: const {}, + journal: [ + for (var i = 0; i < n; i++) + JournalNumericDay(dates[i], {'sauna': flag[i]}), + ], + dates: dates, + // FOUR outcomes: the family size the only production caller passes. + outcomes: { + for (final k in ['rmssd', 'rhr', 'readiness', 'efficiency']) + k: [ + for (var i = 0; i < n; i++) + 60.0 + (flag[i] == 1 ? 20.0 : 0.0) + (i % 3) + ], + }, + ); + } + + final at8 = _effect(run(8), 'sauna', 'rmssd'); + expect(at8.insufficient, isTrue, reason: '1/C(8,3) x 4 = 0.071 > 0.05'); + expect(at8.meaningful, isFalse); + expect(at8.note, 'need_history:have=8,need=9'); + + final at9 = _effect(run(9), 'sauna', 'rmssd'); + expect(at9.insufficient, isFalse, reason: '1/C(9,3) x 4 = 0.032'); + expect(at9.note, isNull); + expect(at9.meaningful, isTrue); + }); + test('empty input is empty output, not a crash', () { expect( journalNumericCorrelations( @@ -381,4 +647,83 @@ void main() { ); }); }); + + // MIND-02 — the journal row for day D describes the DAYTIME of D, but the + // outcome labelled D comes from the night that ENDED on the morning of D. So + // a behaviour field has to be matched against D+1, and a retrospective field + // must NOT be. + group('MIND-02 per-field lag', () { + test('a behaviour field is matched against the FOLLOWING night', () { + final dates = _dates(14); + // Coffee on day i wrecks the night that follows, i.e. the outcome + // labelled i+1. Same-day pairing sees a scrambled series and nothing + // else — that is the bug, reproduced here as the control. + final caffeine = [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2]; + final rmssd = [ + null, // day 0's outcome belongs to the night before the log starts + for (var i = 0; i + 1 < caffeine.length; i++) 80.0 - 6 * caffeine[i], + ]; + final journal = [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay(dates[i], {'caffeine': caffeine[i]}), + ]; + + final lagged = journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: {'rmssd': rmssd}, + ); + final e = _effect(lagged, 'caffeine', 'rmssd'); + expect(e.rho, closeTo(-1.0, 1e-9)); + expect(e.meaningful, isTrue); + expect(lagged.first.lagDays, 1, reason: 'disclosed, not silent'); + + // The old alignment cannot see it at all. + final sameDay = journalNumericCorrelations( + journal: journal, + dates: dates, + outcomes: {'rmssd': rmssd}, + fieldLagDays: const {}, + ); + expect(_effect(sameDay, 'caffeine', 'rmssd').rho!.abs(), lessThan(0.9)); + }); + + test('a retrospective field is NOT shifted', () { + // mood on day D describes the daytime of D, which the night ending that + // morning produced. A blanket +1 would break the fields that are already + // right, which is why the constant is per-field. + expect(journalFieldLagDays['mood'], 0); + expect(journalFieldLagDays['sleep_quality'], 0); + expect(journalFieldLagDays['caffeine'], 1); + expect(journalFieldLagDays['alcohol'], 1); + // An unlisted (custom) field is not silently re-aligned either. + expect(journalFieldLagDays['a_field_we_invented'], isNull); + }); + + test('a lagged day with no outcome the next day is DROPPED, not backfilled', + () { + final dates = _dates(12); + final out = [for (var i = 0; i < 12; i++) 50.0 + i]; + out[6] = null; // the night after day 5 was never judged + final res = journalNumericCorrelations( + journal: [ + for (var i = 0; i < dates.length; i++) + JournalNumericDay(dates[i], {'water': i.toDouble()}), + ], + dates: dates, + outcomes: {'readiness': out}, + ); + // 12 journal days; day 11 has no day-12 outcome in the series, and day 5 + // loses its (null) one. 10 pairs, never 12. + expect(_effect(res, 'water', 'readiness').n, 10); + }); + + test('shiftDayLabel crosses a month and a DST boundary correctly', () { + expect(shiftDayLabel('2026-01-31', 1), '2026-02-01'); + expect(shiftDayLabel('2026-03-29', 1), '2026-03-30'); // EU DST Sunday + expect(shiftDayLabel('2026-12-31', 1), '2027-01-01'); + expect(shiftDayLabel('d0', 1), isNull); + expect(shiftDayLabel('d0', 0), 'd0'); + }); + }); } diff --git a/test/onehz/motion_test.dart b/test/onehz/motion_test.dart index 564936b..fb6783f 100644 --- a/test/onehz/motion_test.dart +++ b/test/onehz/motion_test.dart @@ -72,8 +72,8 @@ void main() { final m = relativeIntensityBands(enmo); expect(m.present, isTrue); final b = m.value!; - expect(b.lightCut, lessThan(b.moderateCut)); - expect(b.moderateCut, lessThan(b.vigorousCut)); + expect(b.lightCut!, lessThan(b.moderateCut!)); + expect(b.moderateCut!, lessThan(b.vigorousCut!)); expect(m.tier, Tier.relative); // a zero-ENMO minute is sedentary; the largest is vigorous. expect(b.labels.first, 'sedentary'); @@ -81,6 +81,34 @@ void main() { expect(b.minutesInBand.values.reduce((a, c) => a + c), 100); }); + test('frozen cut-points override the input\'s own percentiles', () { + // MT-11: the whole point. A rest day whose busiest minute is 0.03 g + // labels its own top decile "vigorous" when the cuts come from itself, + // and has no vigorous minute at all against cuts frozen over a real + // history. Same minutes, same function, two different readings — which + // is why the cuts have to be frozen before this is ever called. + final restDay = [for (var i = 0; i < 100; i++) 0.005 + i * 0.00025]; + final self = relativeIntensityBands(restDay); + expect(self.value!.minutesInBand['vigorous'], greaterThan(0)); + + final frozen = relativeIntensityBands( + restDay, + frozenCuts: (light: 0.05, moderate: 0.15, vigorous: 0.30), + ); + expect(frozen.value!.minutesInBand['vigorous'], 0); + expect(frozen.value!.minutesInBand['moderate'], 0); + expect(frozen.value!.vigorousCut, 0.30); + expect(frozen.note, contains('FROZEN')); + }); + + test('non-monotone frozen cut-points → absent, never reordered', () { + final m = relativeIntensityBands( + [for (var i = 0; i < 100; i++) i * 0.01], + frozenCuts: (light: 0.30, moderate: 0.15, vigorous: 0.05), + ); + expect(m.present, isFalse); + }); + test('empty input → absent, confidence 0', () { final m = relativeIntensityBands(const []); expect(m.present, isFalse); @@ -115,14 +143,15 @@ void main() { return out; } - test('a still wrist reads dynAmp ≈ 0 at ANY gravity reading, while ENMO ' + test( + 'a still wrist reads dynAmp ≈ 0 at ANY gravity reading, while ENMO ' 'moves with the reference', () { // Same physical stillness, two orientations the sensor reads differently // (this ~0.05 g spread between postures is the real, measured fault). - final high = enmoSeries(_series(2, active: (_) => false, gz: 1.03), - gRef: 1.0); - final low = enmoSeries(_series(2, active: (_) => false, gz: 0.94), - gRef: 1.0); + final high = + enmoSeries(_series(2, active: (_) => false, gz: 1.03), gRef: 1.0); + final low = + enmoSeries(_series(2, active: (_) => false, gz: 0.94), gRef: 1.0); for (final m in high.minutes) { expect(m.dynAmp, closeTo(0.0, 1e-12)); } @@ -235,7 +264,8 @@ void main() { test('high-motion epoch → absent (no posture during motion)', () { final s = []; for (var i = 0; i < 30; i++) { - // wild swings → jitter ≫ maxJitterG + // wild swings along ONE axis: direction constant, ‖a‖ swinging — the + // case only the magnitude gate can see. s.add(AccelSample(i * 1000.0, 0, 0, i.isEven ? 0.2 : 2.0)); } final m = staticTilt(s); @@ -243,6 +273,35 @@ void main() { expect(m.confidence, 0); }); + test('a ROTATING wrist yields no posture (magnitude gate was blind to it)', + () { + // an-motion-3. mean |Δ‖a‖| cannot see rotation by construction, and the + // 1 Hz record ships a firmware-fused ~1 g vector, so the old gate could + // not fire on real data at all. Measured pre-fix: a 180° sweep scored + // jitter 4.6e-17 g, stillness 1.000, confidence 0.90, position + // 'lateral_right'. + final s = [ + for (var i = 0; i < 30; i++) + AccelSample(i * 1000.0, 0, 1.03 * math.sin(math.pi * i / 29.0), + 1.03 * math.cos(math.pi * i / 29.0)), + ]; + final m = staticTilt(s); + expect(m.present, isFalse, reason: 'got ${m.value?.position}'); + expect(m.confidence, 0); + expect(m.note, contains('rotating')); + }); + + test('10 min of continuous rotation publishes ZERO postures', () { + // Pre-fix positionSeries kept all 20 epochs and split them 10 + // lateral_right / 10 lateral_left — a posture history of a turning wrist. + final s = [ + for (var i = 0; i < 600; i++) + AccelSample(i * 1000.0, 0, 1.03 * math.sin(2 * math.pi * i / 60.0), + 1.03 * math.cos(2 * math.pi * i / 60.0)), + ]; + expect(positionSeries(s), isEmpty); + }); + test('positionSeries segments and emits stable supine posture', () { final s = _still(120, 0, 0, 1.0); final series = positionSeries(s, epochSec: 30); @@ -360,14 +419,15 @@ void main() { } final medEnmo = res.minutes.isEmpty ? 0.0 - : (res.minutes.map((m) => m.enmo).toList()..sort())[ - res.minutes.length ~/ 2]; + : (res.minutes.map((m) => m.enmo).toList() + ..sort())[res.minutes.length ~/ 2]; // ignore: avoid_print print('REAL ENMO median/min = ${medEnmo.toStringAsFixed(4)} g'); expect(medEnmo, lessThan(0.2), reason: 'median minute is restful'); // Posture: low-motion epochs → stable, dominant posture, no crash. - final postures = positionSeries(accel, epochSec: 30, maxJitterG: 0.1); + final postures = positionSeries(accel, + epochSec: 30, maxRotationDeg: 6.0, maxJitterG: 0.1); // ignore: avoid_print print('REAL posture epochs=${postures.length} ' 'positions=${{for (final p in postures) p.position}}'); @@ -378,20 +438,24 @@ void main() { for (final p in postures) { counts[p.position] = (counts[p.position] ?? 0) + 1; } - final dominant = - counts.values.fold(0, (a, c) => math.max(a, c)); + final dominant = counts.values.fold(0, (a, c) => math.max(a, c)); expect(dominant / postures.length, greaterThan(0.5), reason: 'one body position dominates a rest snippet'); // Energy fusion runs end-to-end without crashing; load is finite & small. - final fuse = branchedEnergyFusion(ts, [ - for (final a in accel) - math.max(0.0, math.sqrt(a.x * a.x + a.y * a.y + a.z * a.z) - res.gRef) - ], hr); + final fuse = branchedEnergyFusion( + ts, + [ + for (final a in accel) + math.max( + 0.0, math.sqrt(a.x * a.x + a.y * a.y + a.z * a.z) - res.gRef) + ], + hr); expect(fuse.present, isTrue); expect(fuse.value!.relativeLoad.isFinite, isTrue); // ignore: avoid_print - print('REAL energy relativeLoad=${fuse.value!.relativeLoad.toStringAsFixed(2)} ' + print( + 'REAL energy relativeLoad=${fuse.value!.relativeLoad.toStringAsFixed(2)} ' 'over ${fuse.value!.points.length} samples'); }); }); diff --git a/test/onehz/nap_test.dart b/test/onehz/nap_test.dart index 2220e24..68488a7 100644 --- a/test/onehz/nap_test.dart +++ b/test/onehz/nap_test.dart @@ -41,6 +41,16 @@ class _Day { } } + /// A block the band never decoded gravity for: the edge hands these over as + /// exact (0,0,0) with `valid: false` (HR decoded, gravity did not). HR is + /// still real. This is the shape that used to read as perfect immobility. + void unmeasuredAccel(int minutes, {double bpm = 60}) { + for (var s = 0; s < minutes * 60; s++, _i++) { + accel.add(AccelSample(_i * 1000.0, 0, 0, 0, valid: false)); + hr.add(bpm); + } + } + /// A motionless block at one fixed orientation. /// `bpm <= 0` writes an HR gap — the substrate's own "no reading" encoding, /// which is what off-wrist and dropout actually look like. @@ -507,4 +517,54 @@ void main() { } }); }); + + group('detectNaps — absent accel is not stillness (an-sleep-1)', () { + test('a 50-min all-invalid block emits NO nap', () { + // Reproduced pre-fix as `NAP {tib_sec: 3006, tst_sec: 3006, + // efficiency: 1.0, confidence: 0.85}` — the confidence CAP, for a nap + // built entirely out of missing data. The z-angle of an exact (0,0,0) + // sample is a perfectly well-formed constant 0.0 deg, which is why the + // angle rule could not see the difference. + final d = _Day() + ..active(120) + ..unmeasuredAccel(50) + ..active(120); + + final m = detectNaps(d.accel, d.hr); + expect(m.present, isTrue, reason: 'the DAY was judged; it just held no nap'); + expect(m.value, isEmpty, reason: 'pre-fix: 1 nap, 3006 s, confidence 0.85'); + }); + + test('a nap is reported at its OBSERVED length, not its wall-clock one', + () { + // 20 min of genuine stillness followed by 25 min the band never measured. + // The unmeasured seconds fail `stillAt` outright, so they neither extend + // the bout nor bridge into a second one: the user is told about the 20 + // minutes we watched, and nothing about the 25 we did not. + final d = _Day() + ..active(120) + ..still(20) + ..unmeasuredAccel(25) + ..active(120); + + final m = detectNaps(d.accel, d.hr); + final nap = m.value!.single; + expect(nap.tibSec, closeTo(20 * 60, 60), + reason: 'pre-fix: ~45 min — the unmeasured half read as more sleep'); + expect(nap.tstSec, nap.tibSec); + }); + + test('a fully-measured nap of the same shape still lands', () { + // The control: identical geometry, valid samples throughout. Without this + // the two tests above would also pass if the detector had simply stopped + // working. + final d = _Day() + ..active(120) + ..still(45) + ..active(120); + + final m = detectNaps(d.accel, d.hr); + expect(m.value!.single.tibSec, greaterThan(40 * 60)); + }); + }); } diff --git a/test/onehz/observed_max_hr_test.dart b/test/onehz/observed_max_hr_test.dart new file mode 100644 index 0000000..dc576fc --- /dev/null +++ b/test/onehz/observed_max_hr_test.dart @@ -0,0 +1,110 @@ +// TS-03 — the observed HR ceiling and its two guards (hold + motion), and the +// TS-04 %HRR zones that anchor on it. + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +/// n seconds of HR at [bpm] with the given accel deviation from 1 g, starting +/// at t0 ms. `dev` is applied on z so ‖a‖ = 1 + dev. +({List hr, List accel}) _run( + double t0, + int seconds, + double bpm, + double dev, +) { + final hr = []; + final accel = []; + for (var i = 0; i < seconds; i++) { + final t = t0 + i * 1000.0; + hr.add(HrSample(t, bpm)); + accel.add(AccelSample(t, 0, 0, 1.0 + dev)); + } + return (hr: hr, accel: accel); +} + +void main() { + group('sessionHrCeiling', () { + test('a held, moving high HR sets the ceiling', () { + final s = _run(0, 40, 180, 0.20); + final m = sessionHrCeiling(s.hr, s.accel, deviceFamily: 'gen4'); + expect(m.present, isTrue); + expect(m.value!.bpm, 180); + expect(m.value!.heldSeconds, greaterThanOrEqualTo(15)); + }); + + test('a one-sample spike cannot inflate the ceiling', () { + final s = _run(0, 40, 150, 0.20); + s.hr[20] = HrSample(s.hr[20].tsMs, 205); // PPG artifact + final m = sessionHrCeiling(s.hr, s.accel, deviceFamily: 'gen4'); + expect(m.value!.bpm, 150, reason: '205 was never held for 15 s'); + }); + + test('high HR while still is not a ceiling', () { + // Same 180 bpm, but the wrist is not moving: stress, fever or artifact. + final s = _run(0, 40, 180, 0.001); + final m = sessionHrCeiling(s.hr, s.accel, deviceFamily: 'gen4'); + expect(m.present, isFalse); + expect(m.note, contains('no_held_ceiling')); + }); + + test('the motion gate is per-family, not shared', () { + // 0.06 g of wrist motion is inside gen4's own resting noise floor and + // well outside gen5's. One shared gate would have to be wrong for one. + final s = _run(0, 40, 175, 0.06); + expect(sessionHrCeiling(s.hr, s.accel, deviceFamily: 'gen4').present, + isFalse); + expect(sessionHrCeiling(s.hr, s.accel, deviceFamily: 'gen5').present, + isTrue); + }); + + test('unknown family refuses instead of borrowing gen4 numbers', () { + final s = _run(0, 40, 180, 0.20); + for (final id in [null, '', 'imported', 'gen6']) { + final m = sessionHrCeiling(s.hr, s.accel, deviceFamily: id); + expect(m.present, isFalse, reason: 'id=$id'); + expect(m.note, unknownFamilyNote(id)); + } + }); + + test('a gap in the stream breaks the hold', () { + final a = _run(0, 10, 180, 0.20); + final b = _run(60000, 10, 180, 0.20); // 50 s later + final m = sessionHrCeiling( + [...a.hr, ...b.hr], + [...a.accel, ...b.accel], + deviceFamily: 'gen4', + ); + expect(m.present, isFalse); + }); + }); + + group('reserveZones (TS-04)', () { + final rhr28 = List.filled(28, 50); + + test('anchors on the median of the history, not one night', () { + final z = + HeartRateZones.reserveZones(restingHrHistory: rhr28, maxHr: 190)!; + expect(z.zones.first.lower, closeTo(50 + 0.50 * 140, 1e-9)); + expect(z.zones.last.upper, closeTo(190, 1e-9)); + expect(z.source, 'karvonen'); + }); + + test('one bad night cannot move the boundaries much', () { + final polluted = [...rhr28]..[0] = 78; + final a = + HeartRateZones.reserveZones(restingHrHistory: rhr28, maxHr: 190)!; + final b = + HeartRateZones.reserveZones(restingHrHistory: polluted, maxHr: 190)!; + expect(b.zones.first.lower, closeTo(a.zones.first.lower, 1.0)); + }); + + test('refuses on thin history and on an inverted reserve', () { + expect( + HeartRateZones.reserveZones( + restingHrHistory: List.filled(5, 50), maxHr: 190), + isNull); + expect(HeartRateZones.reserveZones(restingHrHistory: rhr28, maxHr: 45), + isNull); + }); + }); +} diff --git a/test/onehz/real_night_cardio_stager_test.dart b/test/onehz/real_night_cardio_stager_test.dart index 52c0f04..c1d470c 100644 --- a/test/onehz/real_night_cardio_stager_test.dart +++ b/test/onehz/real_night_cardio_stager_test.dart @@ -115,7 +115,15 @@ void main() { // Pre-fix (broken) output: wake=294 light=173 deep=26 rem=41. // Bands are wide — this is a wrist ESTIMATE, not a PSG-equivalence check — // but tight enough that the pre-fix numbers fail every one of them. - expect(wakeMin, lessThan(20), + // RE-PINNED 2026-08 (was `lessThan(20)`) when the Webster continuity + // rescore was restored to its PUBLISHED table (15 min context bridges ≤4 + // min of wake, not ≤10). Measured on this fixture, that single change moves + // wake 17.5 → 26.0 min, light 295.0 → 289.5, REM 153.0 → 150.0, i.e. TST + // 516.5 → 508.0. The old table was closer to this night's Apple Watch label + // (wake=3) only because it was bridging up to 10 minutes of genuine + // awakening into light sleep; one wrist-reference night is not a licence to + // deviate from Webster 1982 / Cole 1992. + expect(wakeMin, lessThan(35), reason: 'WAKE over-call regression guard (pre-fix: 294 min)'); expect(remMin, greaterThan(100), reason: 'REM under-call regression guard (pre-fix: 41 min)'); diff --git a/test/onehz/respiration_test.dart b/test/onehz/respiration_test.dart index 2e71824..4327a2b 100644 --- a/test/onehz/respiration_test.dart +++ b/test/onehz/respiration_test.dart @@ -63,6 +63,7 @@ import 'package:openstrap_protocol/openstrap_protocol.dart'; } void main() { + _resp01Tests(); // ------------------------------------------------------------------------- // 1. SYNTHETIC KNOWN-ANSWER // ------------------------------------------------------------------------- @@ -89,9 +90,137 @@ void main() { expect(m.value!.brpm!, closeTo(12, 1.5)); }); - test('RSA: absent on too-few beats -> null + confidence 0', () { - final m = rsaRespRate([800, 810, 790], [800, 1610, 2400], + test('RSA: 26.4 br/min is REPORTED, not folded to a plausible 22', () { + // an-motion-1. The search band stopped at rsaHiHz (0.40 Hz = 24 br/min) + // while the anti-alias guard tested respHiHz (0.5), so the guard was dead + // code and any real rate above 24 br/min came back as the largest + // SPURIOUS in-band peak at confidence 0.90. Measured pre-fix: + // 26.4 -> 22.23, 28.8 -> 17.43, both present. The search now runs to the + // window's own beat-rate Nyquist. + for (final trueBrpm in [26.4, 28.8]) { + final s = syntheticRsaRr( + modHz: trueBrpm / 60.0, hrBpm: 75, ampMs: 40, beats: 600); + final corr = correctRr(s.rr); + final m = rsaRespRate(corr.nn, corr.nnTimesMs, + artifactFraction: 1 - corr.cleanFraction); + expect(m.present, isTrue, reason: m.note); + expect(m.value!.brpm!, closeTo(trueBrpm, 1.5), + reason: 'got ${m.value!.brpm} for a true $trueBrpm br/min'); + } + }); + + test('RSA: a rate the BEAT RATE cannot resolve is ABSENT, not a number', + () { + // The tachogram is sampled once per beat, so at HR 43 (NN ~1400 ms) the + // Nyquist is ~21 br/min — below the HF band itself. A 26 br/min breather + // folds to a full-height peak at 16.9 br/min that no spectral test can + // tell from a genuine 16.9. The honest output is nothing. + final s = + syntheticRsaRr(modHz: 26 / 60.0, hrBpm: 43, ampMs: 40, beats: 600); + final corr = correctRr(s.rr); + final m = rsaRespRate(corr.nn, corr.nnTimesMs, + artifactFraction: 1 - corr.cleanFraction); + expect(m.present, isFalse, reason: 'got ${m.value?.brpm}'); + expect(m.confidence, 0); + expect(m.note, contains('alias')); + }); + + test('RSA: the note states the ceiling actually enforced', () { + // It used to say "1 Hz Nyquist caps rate at 30 br/min" while enforcing 24. + final s = syntheticRsaRr(modHz: 0.25, hrBpm: 60, ampMs: 40, beats: 500); + final corr = correctRr(s.rr); + final m = rsaRespRate(corr.nn, corr.nnTimesMs, + artifactFraction: 1 - corr.cleanFraction); + expect(m.note, contains('could resolve up to')); + }); + + // ----------------------------------------------------------------------- + // WHOLE-NIGHT PERIODOGRAM regression. The old robustness surrogate took ONE + // periodogram over the whole input and re-sampled it on 300/450/700-point + // grids. Over a night the periodogram's bins are ~30x finer than that grid + // step, so the three grids read three near-independent NOISE samples of a + // band that (measured on 14 real nights) holds ~2600 local maxima. It + // withheld 17 of 30 of the owner's nights, and when it did publish, the + // number was often not even the band's maximum — one real night published + // 10.66 br/min where that night's own sub-windows agree on 16.97. + // + // Signal below: 8 h of RSA whose rate ramps 19 -> 16 br/min (the drift + // measured across his real nights), on a random-walk RR baseline. The + // time-median rate is 17.5 br/min. + // ----------------------------------------------------------------------- + ({List rr, List t}) rsaNight(int seed, {double hours = 8}) { + final rnd = math.Random(seed); + final rr = [], t = []; + final totalSec = hours * 3600; + const f0 = 19.0 / 60, f1 = 16.0 / 60; + var tMs = 0.0, walk = 0.0; + while (tMs < totalSec * 1000) { + final sec = tMs / 1000; + // Phase is the INTEGRAL of the instantaneous frequency. + final phase = + 2 * math.pi * (f0 * sec + (f1 - f0) * sec * sec / (2 * totalSec)); + walk = (walk + (rnd.nextDouble() - 0.5) * 12) * 0.98; + final v = 1000.0 + walk + 40 * math.sin(phase); + tMs += v; + rr.add(v); + t.add(tMs); + } + return (rr: rr, t: t); + } + + test('RSA: an 8-hour night resolves its time-median rate, not a noise spike', + () { + // Measured, same three noise seeds: OLD published 18.48 / 16.94 / 16.06 — + // three different answers to the same signal, each passing its own + // agreement gate. NEW: 17.52 / 17.52 / 17.49 against a truth of 17.5. + for (final seed in [11, 12, 13]) { + final s = rsaNight(seed); + final corr = correctRr(s.rr, rrTsMs: s.t); + final m = rsaRespRate(corr.nn, corr.nnTimesMs, + artifactFraction: 1 - corr.cleanFraction); + expect(m.present, isTrue, reason: m.note); + expect(m.value!.brpm!, closeTo(17.5, 0.3), + reason: 'seed $seed got ${m.value!.brpm}'); + expect(m.confidence, greaterThan(0.85), reason: m.note); + } + }); + + test('RSA: a shuffled (no-RSA) surrogate night is WITHHELD', () { + // The honesty half of the gate. Shuffling NN destroys the respiratory + // modulation and keeps the sampling geometry; measured sub-window + // consensus falls to ~20% (33-42 of ~190) against 100% for the real + // signal, and 15-28% on the owner's 14 real nights shuffled. + final s = rsaNight(11); + final corr = correctRr(s.rr, rrTsMs: s.t); + final shuffled = [...corr.nn]..shuffle(math.Random(3)); + final m = + rsaRespRate(shuffled, corr.nnTimesMs, artifactFraction: 0); + expect(m.present, isFalse, reason: 'got ${m.value?.brpm}'); + expect(m.confidence, 0); + expect(m.note, contains('sub-windows')); + }); + + test('RSA: trimming the night barely moves the rate', () { + // The failure that started this: the SAME real night replayed with a + // 24-minute-longer sleep window was withheld once and published 17.56 the + // other time. Here, dropping the last 10% of the night (whose true + // time-median rises 17.5 -> 17.65) moves the estimate 17.52 -> 17.65, + // while the old rule moved 18.48 -> 16.37. + final s = rsaNight(11); + final corr = correctRr(s.rr, rrTsMs: s.t); + final full = rsaRespRate(corr.nn, corr.nnTimesMs, artifactFraction: 0); + final k = (corr.nn.length * 0.9).floor(); + final trimmed = rsaRespRate( + corr.nn.sublist(0, k), corr.nnTimesMs.sublist(0, k), artifactFraction: 0); + expect(full.present && trimmed.present, isTrue); + expect((full.value!.brpm! - trimmed.value!.brpm!).abs(), lessThan(0.5), + reason: '${full.value!.brpm} vs ${trimmed.value!.brpm}'); + }); + + test('RSA: absent on too-few beats -> null + confidence 0', () { + final m = + rsaRespRate([800, 810, 790], [800, 1610, 2400], artifactFraction: 0); expect(m.present, isFalse); expect(m.confidence, 0); }); @@ -169,7 +298,8 @@ void main() { var s = 0; while (s < totalSec) { // Sinusoidal HR modulation: 60 ± 6 bpm, period = cycleSec. - final hr = 60.0 - 6.0 * math.sin(2 * math.pi * (s % cycleSec) / cycleSec); + final hr = + 60.0 - 6.0 * math.sin(2 * math.pi * (s % cycleSec) / cycleSec); final r = 60000.0 / hr; // ms nn.add(r); t += r; @@ -187,6 +317,59 @@ void main() { expect(m.note!.toLowerCase(), contains('screen')); }); + test('CVHR: an off-wrist HOLE does not dilute the per-hour index', () { + // an-motion-6. analyzedHours was the first-to-last beat SPAN and + // _resampleLinear drew a straight line across the hole, so the identical + // beats with a 2 h gap in the middle read 53.33/h instead of 80.00/h — a + // 33% "improvement" in an apnea screen bought by taking the band off. + const cycleSec = 60; + final nn = []; + final nnTimes = []; + var t = 0.0; + var s = 0; + while (s < cycleSec * 14) { + final hr = + 60.0 - 6.0 * math.sin(2 * math.pi * (s % cycleSec) / cycleSec); + final r = 60000.0 / hr; + nn.add(r); + t += r; + nnTimes.add(t); + s = (t / 1000.0).floor(); + } + final whole = cvhrApneaScreen(nn, nnTimes, artifactFraction: 0); + + // Same beats, second half displaced by a 2 h hole. + final half = nnTimes.length ~/ 2; + final gapped = [ + for (var i = 0; i < nnTimes.length; i++) + i < half ? nnTimes[i] : nnTimes[i] + 2 * 3600 * 1000 + ]; + final holed = cvhrApneaScreen(nn, gapped, artifactFraction: 0); + + expect(holed.present, isTrue, reason: holed.note); + expect( + holed.value!.analyzedHours, closeTo(whole.value!.analyzedHours, 0.02), + reason: 'the 2 h hole is not observed time'); + expect(holed.value!.cvhrPerHour, closeTo(whole.value!.cvhrPerHour, 3.0), + reason: 'pre-fix this fell by a third'); + }); + + test('CVHR: no cycles means mean depth/width are ABSENT, not 0', () { + // an-motion-7. They were non-nullable and filled with 0, serialising an + // absence as a measurement. + final rr = List.filled(400, 1000.0); + final jit = [ + for (var i = 0; i < rr.length; i++) rr[i] + (i.isEven ? 5 : -5) + ]; + final corr = correctRr(jit); + final m = cvhrApneaScreen(corr.nn, corr.nnTimesMs, + artifactFraction: 1 - corr.cleanFraction); + expect(m.value!.cycleCount, 0); + expect(m.value!.meanDepthMs, isNull); + expect(m.value!.meanWidthSec, isNull); + expect(m.value!.toJson()['mean_depth_ms'], isNull); + }); + test('CVHR: a steady (non-cyclic) RR yields ~0 cycles', () { final rr = List.filled(400, 1000.0); // flat 60 bpm // add tiny jitter so correction has dispersion @@ -284,10 +467,8 @@ void main() { markTestSkipped('whoop_hist.jsonl not found beside the repo'); return; } - final lines = histFile - .readAsLinesSync() - .where((l) => l.trim().isNotEmpty) - .toList(); + final lines = + histFile.readAsLinesSync().where((l) => l.trim().isNotEmpty).toList(); final rrMs = []; final green = []; @@ -410,8 +591,94 @@ void main() { final m = rsaRespRate(corr.nn, corr.nnTimesMs, artifactFraction: 1 - corr.cleanFraction); final j = m.value!.toJson(); - expect((j['peak_hz'] as double) * 60.0, - closeTo(j['brpm'] as double, 1e-4)); + expect( + (j['peak_hz'] as double) * 60.0, closeTo(j['brpm'] as double, 1e-4)); + }); + }); +} + +// --------------------------------------------------------------------------- +// RESP-01 / RESP-02 (gate half) — the 30-night personal CVHR distribution. +// --------------------------------------------------------------------------- +void _resp01Tests() { + CvhrNight night(int day, double perHour, + {double hours = 7, bool irregular = false}) => + CvhrNight( + dayKey: '2026-01-${day.toString().padLeft(2, '0')}', + cvhrPerHour: perHour, + analyzedHours: hours, + irregularRhythm: irregular, + ); + + group('RESP-01 personal distribution', () { + test('under 5 qualifying nights is ABSENT with a machine-readable note', + () { + final m = cvhrPersonalDistribution([ + for (var d = 1; d <= 4; d++) night(d, 10), + ]); + expect(m.present, isFalse); + expect(m.value, isNull); + expect(m.note, contains('need_baseline:nights=4/5')); + }); + + test('thin nights (<4 analyzed hours) never weigh on the screen', () { + final m = cvhrPersonalDistribution([ + for (var d = 1; d <= 5; d++) night(d, 10), + for (var d = 6; d <= 10; d++) night(d, 90, hours: 1), + ]); + expect(m.present, isTrue); + expect(m.value!.nightsUsed, 5); + expect(m.value!.nightsExcludedThin, 5); + expect(m.value!.weightedMean, closeTo(10, 1e-9)); + }); + + test( + 'RESP-02 cross-gate: irregular-rhythm nights are EXCLUDED, not weighted', + () { + final withAf = cvhrPersonalDistribution([ + for (var d = 1; d <= 5; d++) night(d, 10), + night(6, 200, irregular: true), + ]); + expect(withAf.value!.nightsUsed, 5); + expect(withAf.value!.nightsExcludedIrregular, 1); + // the AF night's 200/h moved nothing + expect(withAf.value!.weightedMean, closeTo(10, 1e-9)); + }); + + test('nights are weighted by their OWN analyzed hours', () { + final m = cvhrPersonalDistribution([ + night(1, 20, hours: 8), + night(2, 10, hours: 4), + night(3, 10, hours: 4), + night(4, 10, hours: 4), + night(5, 10, hours: 4), + ]); + // unweighted mean would be 12.0; hours-weighted is 20*8+10*16 over 24 + expect(m.value!.weightedMean, closeTo((20 * 8 + 10 * 16) / 24, 1e-9)); + }); + + test('the only comparison is against the user own retained spread', () { + final quiet = cvhrPersonalDistribution([ + for (var d = 1; d <= 10; d++) night(d, 10), + ]); + expect(quiet.value!.aboveOwnUsual, isFalse); + final rising = cvhrPersonalDistribution([ + for (var d = 1; d <= 8; d++) night(d, 5), + night(9, 40), + night(10, 40), + night(11, 40), + ]); + expect(rising.value!.aboveOwnUsual, isTrue); + // and it still refuses to be an AHI or a severity + expect(rising.toJson().toString(), isNot(contains('ahi'))); + }); + + test('a re-derived day counts once', () { + final m = cvhrPersonalDistribution([ + for (var d = 1; d <= 5; d++) night(d, 10), + night(5, 10), + ]); + expect(m.value!.nightsUsed, 5); }); }); } diff --git a/test/onehz/sleep_honesty_test.dart b/test/onehz/sleep_honesty_test.dart index fdb1abb..4884745 100644 --- a/test/onehz/sleep_honesty_test.dart +++ b/test/onehz/sleep_honesty_test.dart @@ -26,7 +26,8 @@ void main() { // (1) A window with NO accelerometer at all must not be scored as sleep. // ═══════════════════════════════════════════════════════════════════════════ group('honesty — no accelerometer in the window', () { - test('a forced window holding zero samples yields NO sleep ' + test( + 'a forced window holding zero samples yields NO sleep ' '(was: 8 h of light sleep, efficiency 100%)', () { // 4 h of perfectly good data, then a user-asserted 8 h window ~13 h later // that contains not one sample. @@ -40,17 +41,16 @@ void main() { final s = segmentSleep(accel, hr, forcedWindow: (onsetSec: onset, offsetSec: onset + 8 * 3600)); - // The window itself is the user's word, so it is honored... - expect(s.present, isTrue); - expect(s.inBedSec, 8 * 3600); - // ...but NOTHING inside it may be claimed as sleep. - expect(s.tstSec, 0, reason: 'no data ⇒ no sleep (pre-fix: 28800)'); - expect(s.lightSec, 0, reason: 'pre-fix: the whole window read "light"'); - expect(s.deepSec, 0); - expect(s.remSec, 0); - expect(s.wakeSec, 8 * 3600); - expect(s.efficiencyPct, 0.0, reason: 'pre-fix: 100.0'); - expect(s.stages.every((x) => x == SleepStage.wake), isTrue); + // RE-PINNED 2026-08: this used to be `present` with tstSec 0 and + // efficiencyPct 0.0 — which reads to the user as "0 h 0 m slept, 0 % + // efficiency", a measurement of a night we never observed. A window in + // which not one second stages as sleep is an ABSTENTION. + expect(s.present, isFalse, + reason: + 'was: present with tst 0 / efficiency 0 %; pre-that: 28800 s'); + expect(s.tstSec, isNull); + expect(s.efficiencyPct, isNull); + expect(s.stages, isEmpty); }); }); @@ -76,7 +76,8 @@ void main() { expect(cardioStager(withHr, accel).base.stages, isNotEmpty); }); - test('the strap-on-the-nightstand night is not reported as a perfect sleep ' + test( + 'the strap-on-the-nightstand night is not reported as a perfect sleep ' '(was: TST 7h54, efficiency 100%)', () { final accel = []; final hr = []; @@ -85,8 +86,8 @@ void main() { void seg(int secs, {required bool active}) { for (var k = 0; k < secs; k++, i++) { final x = active ? (k.isEven ? 0.0 : 0.3) : 0.005; - accel.add(AccelSample( - (start + i) * 1000.0, x, 0.0, active ? 0.95 : 1.0)); + accel.add( + AccelSample((start + i) * 1000.0, x, 0.0, active ? 0.95 : 1.0)); hr.add(0.0); // NO heart rate at all — the band is off the wrist. } } @@ -96,11 +97,11 @@ void main() { seg(2 * 3600, active: true); final s = segmentSleep(accel, hr, tzOffsetSec: 0); - expect(s.tstSec, 0, reason: 'pre-fix: 28438 s of sleep from zero HR'); - expect(s.efficiencyPct, 0.0, reason: 'pre-fix: 100.0'); - expect(s.remSec, 0); - expect(s.deepSec, 0); - expect(s.lightSec, 0); + // RE-PINNED 2026-08: absent, not "present with zero sleep" — see above. + expect(s.present, isFalse, + reason: 'pre-fix: 28438 s of sleep from zero HR'); + expect(s.tstSec, isNull); + expect(s.efficiencyPct, isNull); }); }); @@ -108,8 +109,9 @@ void main() { // (3) An accelerometer dropout must not be carried forward into "stillness". // ═══════════════════════════════════════════════════════════════════════════ group('honesty — bounded accel carry-forward', () { - test('a 6 h dropout inside an 8 h window stays unstaged ' - '(was: TST 8 h, WASO 0, efficiency 100%)', () { + test( + 'a 6 h dropout inside an 8 h window makes the night ABSENT ' + '(was: TST 8 h, WASO 0, efficiency 100%; then WASO 6 h)', () { final accel = []; final hr = []; void block(int fromSec, int secs) { @@ -123,17 +125,49 @@ void main() { // hours 2-7: NOTHING block(7 * 3600, 3600); // hour 8: real data + final s = segmentSleep(accel, hr, + forcedWindow: (onsetSec: _t0, offsetSec: _t0 + 8 * 3600)); + // RE-PINNED 2026-08 (an-sleep-2): only 2 of the 8 h were observed — 25%, + // below the 50% floor — so there is no honest figure to publish. The + // previous pin (present, wake ≈ 6 h, efficiency < 30%) was itself the + // defect: it reported six hours of MEASURED wakefulness for six hours + // nobody recorded. + expect(s.present, isFalse); + expect(s.tstSec, isNull); + expect(s.wakeSec, isNull); + expect(s.efficiencyPct, isNull); + expect(s.absenceReason, contains('observed'), + reason: 'the absence must carry its reason, not be a bare hole'); + }); + + test('a 2 h dropout inside an 8 h window is UNOBSERVED, not WASO', () { + // 75% observed — above the floor, so the night publishes, but the hole + // must not be counted as measured wake in any figure. + final accel = []; + final hr = []; + for (var k = 0; k < 8 * 3600; k++) { + if (k >= 3 * 3600 && k < 5 * 3600) continue; // 2 h hole + accel.add(AccelSample((_t0 + k) * 1000.0, 0.005, 0.0, 1.0)); + hr.add(52); + } final s = segmentSleep(accel, hr, forcedWindow: (onsetSec: _t0, offsetSec: _t0 + 8 * 3600)); expect(s.present, isTrue); expect(s.inBedSec, 8 * 3600); - // Only the ~2 h that actually has data can be staged; the 6 h hole is - // wake. Allow the bounded 60 s carry-forward tail on each real block. - expect(s.tstSec!, lessThan(2 * 3600 + 2 * 61), - reason: 'pre-fix: 28800 — the whole window read as sleep'); - expect(s.wakeSec!, greaterThan(5 * 3600 + 45 * 60), - reason: 'pre-fix: 0'); - expect(s.efficiencyPct!, lessThan(30.0), reason: 'pre-fix: 100.0'); + // Exactly the 7200 s hole: the seconds on either side of it have both a + // sample and a heart rate, so the ±15 s HR-evidence margin adds nothing + // here (it only bites where samples exist but HR does not). + expect(s.unobservedSec, 7200, reason: 'pre-fix: 0 (no such state)'); + // NEW VALUES, and why they moved: the denominator is now the OBSERVED + // 28800-7200 = 21600 s, not the wall-clock 28800. WASO no longer absorbs + // the hole (pre-fix it was ~7200). + expect(s.wasoSec!, lessThan(120), reason: 'pre-fix: ~7200'); + expect(s.tstSec! + s.wakeSec!, 8 * 3600 - 7200); + expect(s.efficiencyPct!, greaterThan(95.0), + reason: + 'pre-fix: ~74.9 (7200 unrecorded seconds in the denominator)'); + // The hypnogram must break at the hole rather than draw across it. + expect(s.stages4[4 * 3600], 'unobserved'); }); test('a SHORT dropout is still carried forward (the bound is 60 s, not 0)', @@ -197,16 +231,21 @@ void main() { return c; } - test('cardio_stager: only the bout with REAL flanking sleep is bridged', () { - // 15 min sleep, then 5 × (10 min wake + 1 min sleep). The cardio rule - // table bridges ≤10 min of wake given ≥15 min of flanking sleep, so bout - // #1 legitimately bridges. Bouts #2-#5 are flanked by ONE minute of sleep - // and must survive — pre-fix each bridged bout was counted as context for - // the next, so all five collapsed and WASO went to 0. - final sm = fragmented(leadMin: 15, wakeMin: 10, reps: 5); + test('cardio_stager: only the bout with REAL flanking sleep is bridged', + () { + // 15 min sleep, then 5 × (4 min wake + 1 min sleep). The rule table is + // now Webster's published one — ≥15 min of flanking sleep bridges ≤4 min + // of wake — so bout #1 legitimately bridges. Bouts #2-#5 are flanked by + // ONE minute of sleep and must survive: pre-fix each bridged bout was + // counted as context for the next, so all five collapsed and WASO hit 0. + // + // RE-PINNED with the rule table: this used to use 10-min bouts, which the + // old non-published [15 → 10] row bridged. Under Webster a 10-min bout is + // never bridgeable at all, so the case stopped exercising the cascade. + final sm = fragmented(leadMin: 15, wakeMin: 4, reps: 5); websterRescoreCardio(sm, 30); - expect(wakeInBody(sm), 4 * 20, - reason: 'four 10-min bouts survive; pre-fix: 0 (full cascade)'); + expect(wakeInBody(sm), 4 * 8, + reason: 'four 4-min bouts survive; pre-fix: 0 (full cascade)'); }); test('stager: only the bout with REAL flanking sleep is bridged', () { @@ -321,7 +360,8 @@ void main() { () { const stdOffset = -5 * 3600; // e.g. EST const dstOffset = -4 * 3600; // e.g. EDT - const localMidsleep = 3 * 3600; // the sleeper is dead-on 03:00 every night + const localMidsleep = + 3 * 3600; // the sleeper is dead-on 03:00 every night const switchDay = 8; // The transition instant: between day 7's and day 8's midsleep. final switchTs = _midnight + switchDay * 86400; @@ -348,8 +388,8 @@ void main() { // The old behavior — ONE frozen offset for the whole history — splits the // days across two local clock times and lands the anchor half an hour out. - final frozen = habitualMidsleepSecFromHistory(history, - tzOffsetSeconds: dstOffset); + final frozen = + habitualMidsleepSecFromHistory(history, tzOffsetSeconds: dstOffset); expect(frozen!, closeTo(localMidsleep + 1800, 60)); expect((frozen - dstCorrect).abs(), greaterThan(1500)); }); @@ -378,7 +418,8 @@ void main() { for (var m = 30; m < 170; m++) { for (var b = 0; b < 40; b++) { rrTs.add((onset + m * 60) * 1000.0 + b * 1400.0); - rrMs.add(1000.0 + 30.0 * math.sin(m / 14.0) + (b.isEven ? 6.0 : -6.0)); + rrMs.add( + 1000.0 + 30.0 * math.sin(m / 14.0) + (b.isEven ? 6.0 : -6.0)); } } @@ -416,7 +457,8 @@ void main() { ]; } - test('a record still to its last sample does not certify the final 5 min ' + test( + 'a record still to its last sample does not certify the final 5 min ' '(pre-fix: spt_sec 3600 and immobile.last == true)', () { // Movement at 3280 sits just BEFORE the tail, so the pre-fix global // trailing window [n-win, n) never saw it and declared all 299 tail @@ -437,17 +479,18 @@ void main() { expect(w.offsetIdx, lessThanOrEqualTo(n - win + 1)); // ...and the tail is reported as UNDECIDABLE, not as movement. - expect(w.unresolvedTailSec, win - 1, reason: 'pre-fix: 0 (no such state)'); + expect(w.undecidableSec, win - 1, reason: 'pre-fix: 0 (no such state)'); expect(w.immobileUnknown.length, n); expect(w.immobileUnknown.sublist(n - win + 1).every((u) => u), isTrue); for (var i = 0; i < n - win + 1; i++) { expect(w.immobileUnknown[i], isFalse, reason: 'second $i has a full forward window — it is decided'); } - expect(w.toJson()['unresolved_tail_sec'], win - 1); + expect(w.toJson()['undecidable_sec'], win - 1); }); - test('a move inside the tail is resolved PER SECOND ' + test( + 'a move inside the tail is resolved PER SECOND ' '(pre-fix: one shared verdict for all 299 tail seconds)', () { final m = vanHeesSleepWindow(record(moveAt: 3400)); expect(m.present, isTrue); @@ -470,7 +513,7 @@ void main() { // The two seconds must not share one answer — that is the whole bug. expect(w.immobileUnknown[3350] == w.immobileUnknown[3500], isFalse); - expect(w.unresolvedTailSec, 193, reason: 'pre-fix: 0 (no such state)'); + expect(w.undecidableSec, 193, reason: 'pre-fix: 0 (no such state)'); }); }); @@ -478,30 +521,15 @@ void main() { // (9) CPC: the Thomas 2005 HFC/LFC ratio is a MEASUREMENT or it is nothing — // never a sentinel, never a NaN dressed up as a number. // ═══════════════════════════════════════════════════════════════════════════ - group('honesty — cardiopulmonary coupling ratio', () { - test('a spectrum with zero coupling power abstains ' - '(pre-fix: present, hfc=0 lfc=0 cpc_ratio=0.0)', () { - // Every beat carries the SAME timestamp (a stuck clock / corrupt offload) - // and the NN deviations cancel exactly, so the Lomb-Scargle power is 0.0 - // at every frequency and both bands integrate to exactly zero. HFC/LFC is - // then 0/0 — undefined. Pre-fix it shipped as cpc_ratio 0.0, i.e. "the - // least stable sleep measurable". - final nn = List.filled(64, 1000.0); - nn[62] = 900.0; - nn[63] = 1100.0; - final ts = List.filled(64, 0.0); - - final m = cardiopulmonaryCoupling(nn, ts); - expect(m.present, isFalse, reason: 'pre-fix: present with cpc_ratio 0.0'); - expect(m.value, isNull); - expect(m.confidence, 0); - expect(m.note, contains('undefined')); - }); - - test('a NaN anywhere in the NN series abstains ' - '(pre-fix: present at confidence 0.85 with NaN band powers)', () { - // A full hour of beats → the length-driven confidence pins at its 0.85 - // cap, which is exactly what made the pre-fix output so convincing. + group('honesty — cardiopulmonary coupling is WITHDRAWN', () { + test('always absent, with the reason attached', () { + // It never measured coupling. Thomas 2005 needs a respiration channel + // INDEPENDENT of the beat times; the "surrogate" here was the NN series + // linearly detrended, so sqrt(P_rr . P_resp) collapsed to P_rr and + // cpc_ratio equalled the plain RR periodogram HF/LF ratio — measured + // 84.493910 vs 84.493190 on this shape, a ratio of 1.0000085. A healthy + // record used to "still report a real ratio"; that was the bug, not the + // control case. final nn = []; final ts = []; var t = 0.0; @@ -511,36 +539,139 @@ void main() { nn.add(rr); ts.add(t); } - nn[1000] = double.nan; // one poisoned beat - + // ignore: deprecated_member_use_from_same_package final m = cardiopulmonaryCoupling(nn, ts); - // `variance <= 0` does not catch NaN, so the whole spectrum came out NaN - // and was published as hfc/lfc/vlfc = NaN with cpc_ratio 0.0. - expect(m.present, isFalse, - reason: 'pre-fix: present, conf 0.85, hfc/lfc/vlfc = NaN'); + expect(m.present, isFalse); expect(m.value, isNull); expect(m.confidence, 0); - expect(m.note, contains('non-finite')); + expect(m.note, contains('respiration channel')); }); - test('a healthy RSA record still reports a real ratio', () { - // Guard against over-abstention: the control case must survive. + test('the ratio it used to publish was the RR periodogram HF/LF', () { + // The assertion that would have failed from the first commit (T-02). final nn = []; - final ts = []; + final tSec = []; var t = 0.0; - for (var i = 0; i < 300; i++) { + for (var i = 0; i < 3600; i++) { final rr = 1000 + 40 * math.sin(2 * math.pi * 0.25 * (t / 1000.0)); t += rr; nn.add(rr); - ts.add(t); + tSec.add(t / 1000.0); } - final m = cardiopulmonaryCoupling(nn, ts); - expect(m.present, isTrue); - final c = m.value!; - expect(c.cpcRatio.isFinite, isTrue); - expect(c.cpcRatio, greaterThan(0)); - expect(c.cpcRatio, isNot(999.0)); - expect(c.lfc, greaterThan(0)); + final freqs = freqGrid(0.001, 0.45, 200); + final ls = lombScargle(tSec, nn, freqs)!; + // The old "coupling spectrum" was sqrt(P_rr . P_resp) with resp = the + // detrended NN — reconstruct it and show it is P_rr to 5 significant + // figures, which is why the metric had to go. + final rrRatio = ls.bandPower(0.1, 0.4) / ls.bandPower(0.01, 0.1); + final coupling = LombScargle([ + for (final pt in ls.spectrum) + LsPoint(pt.freqHz, math.sqrt(pt.psd * pt.psd)) + ]); + final oldCpc = + coupling.bandPower(0.1, 0.4) / coupling.bandPower(0.01, 0.1); + expect(oldCpc / rrRatio, closeTo(1.0, 1e-4)); + }); + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // (an-sleep-3) An epoch with no heart rate can only ever be labelled asleep. + // ═══════════════════════════════════════════════════════════════════════════ + group('honesty — no cardiac evidence is not sleep', () { + test('two 1 h PPG dropouts inside a 6 h still window are not credited', () { + // HR coverage 0.667 — comfortably above cardio_stager's `minHrCoverage` + // abstain floor, which only catches the all-or-nothing case. Every gate + // that can produce WAKE or REM needs a non-NaN HR while the fall-through + // is an unconditional NREM, so pre-fix all 7200 no-HR seconds came out + // asleep: tst 21600, efficiency 100.0, wake 0. + final accel = []; + final hr = []; + for (var k = 0; k < 6 * 3600; k++) { + accel.add(AccelSample((_t0 + k) * 1000.0, 0.005, 0.0, 1.0)); + final dropout = + (k >= 3600 && k < 2 * 3600) || (k >= 4 * 3600 && k < 5 * 3600); + hr.add(dropout ? 0.0 : 52.0); + } + final s = segmentSleep(accel, hr, + forcedWindow: (onsetSec: _t0, offsetSec: _t0 + 6 * 3600)); + expect(s.present, isTrue); + // 7200 s of dropout, less the +/-15 s HR-evidence margin reaching into + // each of the four dropout edges: 7200 - 4*15 = 7140. + expect(s.unobservedSec, 7140, reason: 'pre-fix: 0 (no such state)'); + expect(s.tstSec!, lessThanOrEqualTo(6 * 3600 - 7140), + reason: 'pre-fix: 21600 — the dropouts were credited as Light'); + expect(s.stages4[90 * 60], 'unobserved'); + }); + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // (SLP-03) Run decomposition. A run ends at an unobserved second. + // ═══════════════════════════════════════════════════════════════════════════ + group('SLP-03 — wake-ups and the longest unbroken stretch', () { + /// A forced 8 h window built from [blocks] of (offsetSec, lengthSec, + /// active). Any second not covered by a block has NO sample at all, which + /// is what makes it unobserved. + SleepSegmentation night(List<(int, int, bool)> blocks) { + final accel = []; + final hr = []; + for (final (from, secs, active) in blocks) { + for (var k = 0; k < secs; k++) { + accel.add(AccelSample( + (_t0 + from + k) * 1000.0, + active ? (k.isEven ? 0.0 : 0.35) : 0.004, + 0.0, + active ? 0.93 : 1.0, + )); + hr.add(active ? 78 : 50); + } + } + return segmentSleep(accel, hr, + forcedWindow: (onsetSec: _t0, offsetSec: _t0 + 8 * 3600)); + } + + test('the longest stretch STOPS at a hole instead of bridging it', () { + // 3 h asleep, a 40-min recording hole, then 4 h 20 asleep. A naive + // longest-run over the stage labels draws straight through the hole and + // prints 7 h 20 of unbroken sleep — three quarters of an hour of which + // nobody watched. + final s = night([ + (0, 3 * 3600, false), + (3 * 3600 + 2400, 8 * 3600 - (3 * 3600 + 2400), false), + ]); + expect(s.present, isTrue); + expect(s.unobservedSec, 2400); + expect(s.longestSleepRunSec, 15600, reason: 'the second block alone'); + expect( + s.longestSleepRunSec!, + lessThan(s.tstSec!), + reason: 'pre-fix a naive run would equal or exceed total sleep time', + ); + expect(s.sustainedAwakenings, 0, reason: 'a hole is not an awakening'); + }); + + test('a sustained wake counts; the hole beside it still does not', () { + // Same night with a 10-minute genuine awakening after the hole. + final s = night([ + (0, 3 * 3600, false), + (3 * 3600 + 2400, 600, true), + (3 * 3600 + 3000, 8 * 3600 - (3 * 3600 + 3000), false), + ]); + expect(s.unobservedSec, 2400); + expect(s.wakeSec, 600); + expect(s.sustainedAwakenings, 1); + // 40 min of hole + 10 min of wake, both broken out of the run. + expect(s.longestSleepRunSec, 15000); + }); + + test('a wake shorter than the stated bar is not an awakening', () { + final s = night([ + (0, 4 * 3600, false), + (4 * 3600, 120, true), // 2 min, under the 5-minute bar + (4 * 3600 + 120, 8 * 3600 - (4 * 3600 + 120), false), + ]); + expect(s.sustainedAwakenings, 0); + // And the bar itself ships with the count so a screen can state it. + expect(s.toJson()['awakening_min_sec'], kSustainedAwakeningSec); }); }); } diff --git a/test/onehz/sleep_test.dart b/test/onehz/sleep_test.dart index 1bcb42a..b72740e 100644 --- a/test/onehz/sleep_test.dart +++ b/test/onehz/sleep_test.dart @@ -28,6 +28,8 @@ import 'package:openstrap_analytics/src/onehz/sleep/stager.dart' import 'package:openstrap_protocol/openstrap_protocol.dart'; void main() { + _slp13Tests(); + _cv06Tests(); // ------------------------------------------------ DST-aware tz offset (#1) group('segmentSleep — per-timestamp tz offset (DST-aware)', () { // Build active → still(low HR) → active overnight block; inject a resolver @@ -173,36 +175,44 @@ void main() { // A 3h shift on an 8h sleep window: 6h of the 24 disagree → SRI=200·(18/24)-100=50. expect(m.value!.sri, closeTo(50, 1.0)); }); - }); - // --------------------------------------------------------------- accounting - group('sleep accounting', () { - test('known in-bed window → TST/WASO/efficiency exact', () { - // 8h in bed (28800 s). Asleep except a 30-min WASO block at hour 3 and a - // 20-min sleep-latency wake at the very start. - const total = 8 * 3600; - final asleep = List.filled(total, true); - for (var i = 0; i < 20 * 60; i++) { - asleep[i] = false; // onset latency - } - final wasoStart = 3 * 3600; - for (var i = wasoStart; i < wasoStart + 30 * 60; i++) { - asleep[i] = false; // mid-night WASO - } - final m = sleepAccounting(asleep); + test( + 'per-pair decomposition names the pair that broke it, and a ' + 'half-unobserved pair is NOT emitted', () { + // SLP-08. Four days: 0,1,2 identical; day 3 shifted 3 h AND only 300 of + // its 1440 minutes observed. The shifted pair is genuinely the worst + // pair, and it is exactly the pair that must not be reported, because + // it was compared on a fifth of the clock day. + const epochsPerDay = 1440; + List day(int shift) => List.generate(epochsPerDay, (e) { + final ee = (e - shift) % epochsPerDay; + final m = ee < 0 ? ee + epochsPerDay : ee; + return m < 7 * 60 || m >= 23 * 60; + }); + final vec = [...day(0), ...day(0), ...day(90), ...day(3 * 60)]; + final valid = [ + ...List.filled(epochsPerDay, true), + ...List.filled(epochsPerDay, true), + ...List.filled(epochsPerDay, true), + ...List.generate(epochsPerDay, (e) => e < 300), + ]; + final m = phillipsSri(vec, epochsPerDay, valid: valid); expect(m.present, isTrue); - final a = m.value!; - expect(a.onsetIdx, 20 * 60); - expect(a.wasoSec, 30 * 60); // only the mid-night block counts as WASO - expect(a.tstSec, total - 20 * 60 - 30 * 60); - // Efficiency = TST / in-bed, where in-bed = offset − onset + 1 (the sleep - // PERIOD, NOT the whole captured mask). Onset latency is excluded from the - // denominator: offset is the last asleep second (total-1), onset=1200. - final inBed = a.offsetIdx - a.onsetIdx + 1; - expect(inBed, total - 20 * 60); // 20-min latency trimmed off the front - expect(a.efficiencyPct, closeTo(100.0 * a.tstSec / inBed, 1e-6)); - // And NOT the old whole-mask denominator. - expect(a.efficiencyPct, isNot(closeTo(100.0 * a.tstSec / total, 1e-6))); + final pairs = m.value!.pairs; + // pair 3 (the thin one) is dropped; pairs 1 and 2 survive. + expect([for (final p in pairs) p.dayIndex], [1, 2]); + // pair 1 is the two identical days; pair 2 carries the 90-min shift. + expect(pairs[0].sri, closeTo(100, 1e-6)); + expect(pairs[1].sri, lessThan(pairs[0].sri)); + // the decomposition is of the SAME arithmetic: dropping a thin pair from + // the LIST must not drop it from the published total. + var agree = 0, cases = 0; + for (final p in pairs) { + agree += p.agreement; + cases += p.cases; + } + expect(agree, lessThanOrEqualTo(cases)); + expect(cases, lessThan(m.value!.cases)); }); }); @@ -222,6 +232,7 @@ void main() { t += 1000.0; } } + active(dayH * 3600); final nightStart = out.length; final moves = moveAt.toSet(); @@ -261,7 +272,8 @@ void main() { return hr; } - test('(a) still + low-HR night → one window, tst≈in-bed, eff high, ' + test( + '(a) still + low-HR night → one window, tst≈in-bed, eff high, ' 'figures consistent', () { final accel = _accel(2, 7, 1); final hr = _hr(2, 7, 1); @@ -307,6 +319,20 @@ void main() { expect(j['tst_sec'], s.tstSec); expect(j['confidence'], greaterThan(0)); + // SLP-01. Half of a night's confidence is the WINDOW's confidence, and it + // used to be van Hees' — a detector whose `onsetIdx`/`offsetIdx` this + // pipeline never reads. It is now van Hees' own construction (length up + // to a 7 h night, clamped to [0.3, 0.95]) applied to the window that + // actually shipped, so it is a statement about `in_bed_sec`. + // Every second here carries a valid HR and no RR was passed, so + // stagingConf is exactly (0.35 + 0.25*0) * 1.0 = 0.35 and the whole + // confidence is pinned. + final windowConf = (s.inBedSec! / (7 * 3600)).clamp(0.3, 0.95); + expect( + s.confidence, + closeTo(((windowConf + 0.35) / 2.0).clamp(0.0, kMaxSleepConfidence), + 1e-9)); + // 4-CLASS HYPNOGRAM (Awake/Light/Deep/REM): the per-second stages4 stream // is aligned 1:1 with stages, uses only the four labels, and its Light/Deep // partition of NREM reconciles exactly with the combined nremSec. @@ -388,12 +414,14 @@ void main() { expect(s.confidence, 0); }); - test('(c5) forced window honors a user window the auto path rejects (<3h) ' + test( + '(c5) forced window honors a user window the auto path rejects (<3h) ' 'and stages within it (single-source)', () { // A 2h still night sits BELOW the 3h auto floor → auto returns absent. final accel = _accel(2, 2, 1); final hr = _hr(2, 2, 1); - expect(segmentSleep(accel, hr).present, isFalse, reason: 'auto rejects <3h'); + expect(segmentSleep(accel, hr).present, isFalse, + reason: 'auto rejects <3h'); // The user asserts sleep across the still block. tsMs == index*1000, so // epoch-seconds == index; the night runs [7200, 14400). @@ -459,7 +487,8 @@ void main() { stillSleep(5 * 3600, bpm: 52); active(2 * 3600); - final s = segmentSleep(accel, hr, hrBaseline: List.filled(120, 72)); + final s = + segmentSleep(accel, hr, hrBaseline: List.filled(120, 72)); expect(s.present, isTrue); // The overnight block earns the timing bonus, so the chosen in-bed span is // the ~4h night rather than the longer daytime nap. @@ -490,11 +519,13 @@ void main() { active(2 * 3600); stillSleep(3 * 3600, bpm: 49); - active(30 * 60, baseHr: 85); // real wake gap: preserved as WASO, not new day + active(30 * 60, + baseHr: 85); // real wake gap: preserved as WASO, not new day stillSleep(2 * 3600, bpm: 50); active(4 * 3600); - final s = segmentSleep(accel, hr, hrBaseline: List.filled(120, 72)); + final s = + segmentSleep(accel, hr, hrBaseline: List.filled(120, 72)); expect(s.present, isTrue); // The selector bridges adjacent overnight fragments (<60 min gap) into // one main sleep span rather than picking only the longest fragment. @@ -515,8 +546,7 @@ void main() { [200 * 60, 200 * 60 + 180], // 3 min at ~3.3h [300 * 60, 300 * 60 + 150], // 2.5 min at 5h ]; - bool inArousal(int s) => - arousals.any((a) => s >= a[0] && s < a[1]); + bool inArousal(int s) => arousals.any((a) => s >= a[0] && s < a[1]); final accel = []; final hr = []; var t = 0.0; @@ -542,7 +572,8 @@ void main() { hr.add(72 + (i % 5).toDouble()); t += 1000.0; } - final s = segmentSleep(accel, hr, hrBaseline: List.filled(60, 72)); + final s = + segmentSleep(accel, hr, hrBaseline: List.filled(60, 72)); expect(s.present, isTrue); // Arousals total < 8 min over a ~7h night; Webster rescoring bridges them // so efficiency stays high. (The Walch stager leaves a little more residual @@ -584,7 +615,8 @@ void main() { hr.add(72 + (i % 5).toDouble()); t += 1000.0; } - final s = segmentSleep(accel, hr, hrBaseline: List.filled(60, 72)); + final s = + segmentSleep(accel, hr, hrBaseline: List.filled(60, 72)); expect(s.present, isTrue); // The 30-min block (1800 s) must be counted as wake/WASO, not bridged. // Allow epoch/HR-dip edge trimming but require most of the 30-min block @@ -634,30 +666,23 @@ void main() { }); // ---------------------------------------------------------------------- CPC - group('Cardiopulmonary Coupling', () { - test('NN with injected ~0.25 Hz respiration → HF coupling band', () { - // Synthesize an NN tachogram at mean 1000 ms with a strong RSA at 0.25 Hz - // (15 breaths/min). Beat-to-beat at ~1 beat/s. The coupling spectrum's - // dominant frequency should fall in the HFC band (0.1–0.4 Hz). + group('Cardiopulmonary Coupling (WITHDRAWN)', () { + test('abstains — no respiration channel independent of the beat times', () { + // Was: "NN with injected ~0.25 Hz respiration → HF coupling band". That + // passed because the respiration surrogate WAS the NN series, so of + // course the "coupling" peak sat at the RSA frequency. See cpc.dart. final nn = []; final times = []; var t = 0.0; - const fResp = 0.25; // Hz for (var i = 0; i < 1800; i++) { - // RR modulated by respiration (RSA): ±40 ms at 0.25 Hz. - final rr = 1000 + 40 * math.sin(2 * math.pi * fResp * (t / 1000.0)); + final rr = 1000 + 40 * math.sin(2 * math.pi * 0.25 * (t / 1000.0)); t += rr; nn.add(rr); times.add(t); } final m = cardiopulmonaryCoupling(nn, times); - expect(m.present, isTrue); - final c = m.value!; - // Dominant coupling frequency near the injected respiration. - expect(c.dominantHz, closeTo(fResp, 0.05), - reason: 'coupling peak should sit at the respiratory frequency'); - // HFC should dominate (stable-NREM-like) vs LFC. - expect(c.hfc, greaterThan(c.lfc)); + expect(m.present, isFalse); + expect(m.note, contains('respiration channel')); }); }); @@ -669,8 +694,7 @@ void main() { final x = []; for (var d = 0; d < 7; d++) { for (var h = 0; h < eph; h++) { - final a = - 1 + math.cos(2 * math.pi * (h - 14) / 24); // 0..2, peak @14 + final a = 1 + math.cos(2 * math.pi * (h - 14) / 24); // 0..2, peak @14 x.add(a); } } @@ -702,10 +726,8 @@ void main() { markTestSkipped('whoop_hist.jsonl not found beside the repo'); return; } - final lines = histFile - .readAsLinesSync() - .where((l) => l.trim().isNotEmpty) - .toList(); + final lines = + histFile.readAsLinesSync().where((l) => l.trim().isNotEmpty).toList(); final accel = []; final hr = []; final immobileFromRest = []; @@ -803,8 +825,9 @@ void main() { // No "light"/4th stage ever appears — strictly 3-class. expect( - consolidated.toSet().difference( - {SleepStage.wake, SleepStage.nrem, SleepStage.rem}), + consolidated + .toSet() + .difference({SleepStage.wake, SleepStage.nrem, SleepStage.rem}), isEmpty); // Totals preserved to within a couple of bouts (minBoutEp=10 here): the @@ -845,7 +868,8 @@ void main() { // ------------------------------------------------ sleepCyclesMetric (#6) group('sleepCyclesMetric — direct coverage', () { test('empty RR → honest absent (no fabricated cycles)', () { - final m = sleepCyclesMetric(const [], const [], 0, 8 * 3600); + final m = + sleepCyclesMetric(const [], const [], 0, 8 * 3600); expect(m.present, isFalse); expect(m.value, isNull); expect(m.tier, Tier.estimate); @@ -896,3 +920,168 @@ SleepStage _dominant(List xs) { }); return best; } + +// --------------------------------------------------------------------------- +// SLP-13 — stage minutes are INTERVALS, sized by the night's own confidence. +// --------------------------------------------------------------------------- +void _slp13Tests() { + group('SLP-13 stage intervals', () { + ({int lo, int hi}) rem(double conf) { + final r = stageIntervals( + lightSec: 12000, + deepSec: 3600, + remSec: 3600, + tstSec: 21600, + confidence: conf, + ).rem; + return (lo: r.loSec, hi: r.hiSec); + } + + test('deep is wider than REM at the same minutes', () { + final s = stageIntervals( + lightSec: 12000, + deepSec: 3600, + remSec: 3600, + tstSec: 21600, + confidence: kMaxSleepConfidence, + ); + expect( + s.deep.hiSec - s.deep.loSec, greaterThan(s.rem.hiSec - s.rem.loSec)); + // and the exact count is still there for Investigate + expect(s.deep.pointSec, 3600); + }); + + test('a better-observed night gets a narrower interval', () { + final good = rem(kMaxSleepConfidence); + final poor = rem(0.0); + expect(good.hi - good.lo, lessThan(poor.hi - poor.lo)); + }); + + test('zero deep publishes a floor, never 0-0', () { + final s = stageIntervals( + lightSec: 18000, + deepSec: 0, + remSec: 0, + tstSec: 18000, + confidence: 0.5, + ); + expect(s.deep.loSec, 0); + expect(s.deep.hiSec, kStageIntervalFloorSec); + }); + + test('no stage interval can exceed the sleep it is part of', () { + final s = stageIntervals( + lightSec: 1000, + deepSec: 5000, + remSec: 6000, + tstSec: 12000, + confidence: 0.0, + ); + expect(s.deep.hiSec, lessThanOrEqualTo(12000)); + expect(s.rem.hiSec, lessThanOrEqualTo(12000)); + expect(s.light.loSec, greaterThanOrEqualTo(0)); + }); + + test('light inherits deep\'s ABSOLUTE half-width, not its relative one', + () { + final s = stageIntervals( + lightSec: 14400, + deepSec: 3600, + remSec: 3600, + tstSec: 21600, + confidence: 0.4, + ); + expect(s.light.hiSec - s.light.loSec, s.deep.hiSec - s.deep.loSec); + }); + + test('an absent night has no ranges', () { + expect(SleepSegmentation.absent.stageRanges, isNull); + expect(SleepSegmentation.absent.toJson()['deep_range_sec'], isNull); + }); + }); +} + +// --------------------------------------------------------------------------- +// CV-06 — the shape of the night: per-bin RMSSD. +// --------------------------------------------------------------------------- +({List nn, List t}) _syntheticNight({ + required double hours, + required double Function(double hourIntoNight) ampMs, + ({double fromH, double toH})? gap, +}) { + final nn = []; + final t = []; + var now = 0.0; // ms + var i = 0; + while (now < hours * 3600000) { + final h = now / 3600000; + // A 4-beat cycle [0, +2a, 0, -2a] => every successive difference is 2*amp, + // so the bin RMSSD is exactly 2*amp and the expected shape is constructed, + // not guessed at. It used to be a straight alternation, which has the same + // RMSSD but a difference-series lag-1 ACF of exactly −1 — indistinguishable + // from pure beat-timing jitter, which `hrvTime` now refuses outright + // (kNnDiffAcf1Floor). This shape has ACF1 = 0 and no real night alternates. + final v = 1000.0 + 2 * ampMs(h) * math.sin(2 * math.pi * i / 4); + now += v; + i++; + if (gap != null && h >= gap.fromH && h < gap.toH) continue; + nn.add(v); + t.add(now); + } + return (nn: nn, t: t); +} + +void _cv06Tests() { + group('CV-06 nightly HRV shape', () { + test('recovers a suppressed first third and a recovered last third', () { + final n = _syntheticNight( + hours: 8, + ampMs: (h) => h < 2.7 ? 5.0 : (h < 5.3 ? 15.0 : 30.0), + ); + final m = nightHrvShape(n.nn, n.t); + expect(m.present, isTrue); + final v = m.value!; + expect(v.bins.length, 16); // 8 h of 30-min bins + expect(v.bins.every((b) => b.present), isTrue); + expect(v.firstThirdMs, closeTo(10, 1)); // 2 * 5 ms + expect(v.lastThirdMs, closeTo(60, 2)); // 2 * 30 ms + expect(v.lastOverFirst, greaterThan(3)); + // and every published bin carries a BAND, not just a point + for (final b in v.bins) { + expect(b.loMs, lessThan(b.rmssdMs!)); + expect(b.hiMs, greaterThan(b.rmssdMs!)); + } + }); + + test('a gapped bin ABSTAINS and stays in the series as a hole', () { + final n = _syntheticNight( + hours: 8, + ampMs: (_) => 20.0, + gap: (fromH: 3.0, toH: 3.9), + ); + final m = nightHrvShape(n.nn, n.t); + final v = m.value!; + expect(v.bins.length, 16); + final holes = v.bins.where((b) => !b.present).toList(); + expect(holes, isNotEmpty); + // the hole reports WHY (how many beats it had), it is not a bare dash + expect(holes.first.nBeats, lessThan(kMinBeatsPerHrvBin)); + // and the abstention costs confidence + expect(m.confidence, lessThan(0.8)); + }); + + test('a short night has no shape to describe', () { + final n = _syntheticNight(hours: 1, ampMs: (_) => 20.0); + final m = nightHrvShape(n.nn, n.t); + expect(m.present, isFalse); + expect(m.note, contains('no shape')); + }); + + test('CV-06 does not publish a time-to-max (deliberately cut)', () { + final n = _syntheticNight(hours: 8, ampMs: (_) => 20.0); + final j = nightHrvShape(n.nn, n.t).value!.toJson(); + expect(j.keys, isNot(contains('time_to_max_sec'))); + expect(j.toString(), isNot(contains('time_to'))); + }); + }); +} diff --git a/test/onehz/steps_test.dart b/test/onehz/steps_test.dart index 791d065..b87a6a2 100644 --- a/test/onehz/steps_test.dart +++ b/test/onehz/steps_test.dart @@ -123,6 +123,176 @@ void main() { }); }); + // These pin the numbers the 2026-08-16 audit measured, so the +10.6% gain + // cannot be re-derived from a comment and the physiological interval bounds + // cannot be quietly dropped again. Every expectation below is a MEASUREMENT + // of the shipping algorithm, not a target. + group('Tier A — the gain is 1.00 and the raw count is already exact', () { + test('gain is 1.00 — the default is not a population fudge factor', () { + // Was 1.11, claiming to fix "raw 90 -> ~100". Measured, it manufactured + // a +10.6% over-count on gait the raw counter already got right. + expect(StepParams.gain, 1.00); + }); + + test('raw/truth is 1.00 at 60, 80, 100, 120, 140 and 180 spm', () { + // The whole basis for removing the gain: there is no under-count to + // correct anywhere in the human cadence range. Exact figures measured on + // 300 s of _walk at 0.25 g, 100 Hz. + for (final (spm, want) in const [ + (60, 299), + (80, 399), + (100, 498), + (120, 598), + (140, 698), + (180, 898), + ]) { + final truth = (300 * spm / 60).round(); + final (x, y, z) = _walk(300.0, spm / 60.0); + final got = livePedometer(x, y, z).steps; + expect(got, want, reason: '$spm spm'); + expect(got / truth, closeTo(1.00, 0.005), reason: '$spm spm raw/truth'); + } + }); + + test('the residual deficit is a FLAT -0.9% chunk-boundary loss, not a gain', + () { + // still -> N steps at 110 spm -> still, counted in 60 s chunks the way + // calcSteps does. A gain error would scale with N and an offset would + // shrink with it; this does neither, because it is ~2 steps lost per + // minute boundary while the state machine re-earns CONFIRM. It is 0.9%, + // it is known, and it is deliberately not chased. + int chunked(int n) { + final walkS = n / (110 / 60.0); + final (wx, wy, wz) = _walk(walkS, 110 / 60.0); + final sig = [ + ...List.filled(300, 1.0), + for (var i = 0; i < wx.length; i++) + math.sqrt(wx[i] * wx[i] + wy[i] * wy[i] + wz[i] * wz[i]), + ...List.filled(300, 1.0), + ]; + final minutes = >[ + for (var i = 0; i < sig.length; i += 6000) + sig.sublist(i, math.min(i + 6000, sig.length)) + ]; + return calcSteps(minutes); + } + + expect(chunked(100), 100); // one chunk, no boundary, no loss + expect(chunked(1000), 991); // -0.9% + expect(chunked(10000), 9910); // -0.9% — flat, 10x the bout + }); + }); + + group('Tier A — step-interval bounds (0.2-2.0 s, from AN-2554)', () { + int stepsAt(double spm, double durationS, + {double amp = 0.30, double fs = 100, double? tellRate}) { + final (x, y, z) = _walk(durationS, spm / 60.0, fs: fs, ampG: amp); + return livePedometer(x, y, z, sampleRateHz: tellRate ?? fs).steps; + } + + test('the bounds are the ones the note states, and they are named', () { + expect(StepParams.minStepIntervalS, 0.2); // max ~5 steps/s + expect(StepParams.maxStepIntervalS, 2.0); // min ~0.5 steps/s + }); + + test('faster than 5 steps/s is rejected outright', () { + // PRE-FIX these counted at raw/truth 1.00 (320 spm -> 638/640) and 0.95 + // (350 spm -> 666/700): nothing bounded the rhythm, and the only thing + // that eventually broke lock was the width of the centred peak window. + expect(stepsAt(305, 120), 0); + expect(stepsAt(320, 120), 0); + expect(stepsAt(350, 120), 0); + }); + + test('the ceiling sits exactly at the stated 5 steps/s, not below it', () { + // A sprint cadence a human can actually produce must still count — the + // guard is physiology, not a convenient way to lose the hard cases. + expect(stepsAt(250, 120), 498); + expect(stepsAt(300, 120), 598); + }); + + test('40 spm still counts — it is inside physiology, so nothing rejects it', + () { + // The audit flagged "1.00 at 40 spm with no floor". 40 spm is 0.67 + // steps/s, above the note's 0.5 steps/s minimum, so the correct floor + // does NOT reject it. Pinning that so the bound does not get tightened + // into a slow-walk deleter. + expect(stepsAt(40, 600, fs: 50), 399); + }); + + test('slower than 0.5 steps/s is rejected — at the buffer\'s REAL rate', () { + // This is the case that proves the bounds are computed from the sample + // rate rather than a hardcoded 100. `maxMinTimeout` is 120 SAMPLES, so + // it already covers the slow end at 100 Hz but stretches to 2.4 s at + // 50 Hz and lets 20-28 spm through. Told the truth, the 2.0 s bound + // rejects them; told "100 Hz" about a 50 Hz buffer, the same signal + // reads a full 250 steps. + expect(stepsAt(25, 600, fs: 50), 0); + expect(stepsAt(28, 600, fs: 50), 0); + expect(stepsAt(25, 600, fs: 50, tellRate: 100), 250); + }); + + test('the bounds cost nothing inside the walking band', () { + // The guards must not pay for themselves with real steps. + expect(stepsAt(110, 600, fs: 50, amp: 0.25), 1098); + expect(stepsAt(120, 300, amp: 0.25), 598); + }); + }); + + // ── OxWalk: the first ground truth this counter has ever met ────────────── + // + // Measured 2026-08-17 with `dart run tool/oxwalk_validate.dart` against + // OxWalk (Small, von Fritsch, Doherty, Khalid, Price; Oxford, Dec 2022, + // CC BY) — 39 adults, unscripted free living, dominant wrist + hip, camera + // ground truth. Chunked exactly as production does (`_minuteSamples = 6000`). + // Full write-up: edge/docs/internal/OXWALK_VALIDATION.md. + // + // Wrist 100 Hz MAPE 33.0% bias -13.6% median -19.5% totals -19.8% + // per-participant ratio 0.33 - 3.00, only 5/39 within +-10% + // Hip 100 Hz MAPE 29.5% bias -29.2% totals -17.6% + // Wrist 25 Hz MAPE 91.9% totals -87.5% <- dead + // Hip 25 Hz MAPE 94.5% totals -93.6% <- dead + // rate cliff (100 Hz wrist decimated): 50 Hz -27.0%, 33 Hz -60.1%, + // 25 Hz -86.5%, 20 Hz -89.9% + // chunk sweep (wrist 100 Hz, MAPE): 25 s 37.4 | 30 s 34.5 | 60 s 33.0 + // | 2 min 34.3 | 5 min 37.7 | 1 h 55.1 + // best single gain 1.18 -> MAPE 27.9% (from 33.0%). Not applied: the error + // is not multiplicative — see the doc. + // + // The dataset is 290 MB and is deliberately not committed, so nothing here + // re-measures it. What this pins is the CONFIGURATION those numbers describe: + // change any of it and the recorded numbers are stale, which is exactly when + // you want to be told. + group('Tier A — OxWalk-measured configuration (2026-08-17)', () { + test('the parameters OxWalk was measured against have not moved', () { + const why = 'OxWalk numbers in edge/docs/internal/OXWALK_VALIDATION.md ' + 'were measured against this exact configuration. If you changed it ' + 'deliberately, re-run tool/oxwalk_validate.dart on the 39 ' + 'participants and update BOTH the doc and this test.'; + expect(StepParams.gain, 1.00, reason: why); + expect(StepParams.sens, 0.10, reason: why); + expect(StepParams.confirm, 8, reason: why); + expect(StepParams.filter, 8, reason: why); + expect(StepParams.window, 33, reason: why); + expect(StepParams.thrOrder, 4, reason: why); + expect(StepParams.maxMinTimeout, 120, reason: why); + expect(StepParams.minStepIntervalS, 0.2, reason: why); + expect(StepParams.maxStepIntervalS, 2.0, reason: why); + }); + + test('gain stays 1.00 — OxWalk found no single multiplier that works', () { + // Tempting and wrong. 1.18 minimises MAPE across the 39 and recovers only + // 5 points of a 33-point error, and the sign of the error FLIPS with + // activity level: participants with >=600 true steps/h under-count by a + // uniform -25.4%, while the 17 with <600 average +1.7% bias around a + // 42.9% MAPE because two of them over-count by 3x (P18: 217 true, 650 + // counted; P28: 258 true, 685 counted). Multiplying by 1.18 improves the + // walkers slightly and pushes those two from +200%/+166% to +254%/+214%. + // A gain cannot fix an error whose sign depends on the input. + expect(StepParams.gain, 1.00); + }); + }); + group('Calibration', () { test('credible walking bout seeds + refines the model', () { const live = PedometerResult(220, 120, 110.0, 0.25, 0.8); @@ -140,18 +310,18 @@ void main() { final prior = const StepCalibration(cadenceSpm: 110, refEnmo: 0.06, n: 5); // low confidence expect( - calibrateCadence(prior, const PedometerResult(10, 30, 110, 0.2, 0.2), - 0.06), + calibrateCadence( + prior, const PedometerResult(10, 30, 110, 0.2, 0.2), 0.06), same(prior)); // implausible cadence (200 spm) expect( - calibrateCadence(prior, const PedometerResult(100, 30, 200, 0.2, 0.9), - 0.06), + calibrateCadence( + prior, const PedometerResult(100, 30, 200, 0.2, 0.9), 0.06), same(prior)); // too short expect( - calibrateCadence(prior, const PedometerResult(20, 10, 110, 0.2, 0.9), - 0.06), + calibrateCadence( + prior, const PedometerResult(20, 10, 110, 0.2, 0.9), 0.06), same(prior)); }); @@ -182,10 +352,10 @@ void main() { expect(personalDynFloor(pool, quantile: 0.5), closeTo(0.5, 0.01)); }); - test('the minimum-history requirement is a named, overridable constant', () { + test('the minimum-history requirement is a named, overridable constant', + () { expect(personalDynFloorMinMinutes, 2000); - expect( - personalDynFloor(List.filled(50, 0.4), minMinutes: 10), + expect(personalDynFloor(List.filled(50, 0.4), minMinutes: 10), closeTo(0.4, 1e-9)); }); }); @@ -240,8 +410,8 @@ void main() { test('uncovered minutes do not count toward the summary', () { // 200 rows but all sparse → below the covered-minute floor → null. - expect(dailyDynSummary(mins(List.filled(200, 0.4), n: 5)), - isNull); + expect( + dailyDynSummary(mins(List.filled(200, 0.4), n: 5)), isNull); }); test('summarises this day at the same quantile the floor is defined on', @@ -275,7 +445,8 @@ void main() { ]; const sedDyn = 0.02; // a sedentary minute's dynamic amplitude (g) const walkDyn = 0.60; // an ambulatory minute's (g) - const floorG = 0.375; // the kind of value personalDynFloor yields in practice + const floorG = + 0.375; // the kind of value personalDynFloor yields in practice // A day = `sed` sedentary minutes then `walk` ambulatory minutes. List day(int sed, int walk) => rows([ ...List.filled(sed, sedDyn), @@ -284,13 +455,63 @@ void main() { // A measured personal cadence from Tier A (100 Hz, real counts). const cal = StepCalibration(cadenceSpm: 110, refEnmo: 0.06, n: 10); - test('COLD START: no personal floor → ABSTAIN with a need_baseline note', () { + test('COLD START: no personal floor → ABSTAIN with a need_baseline note', + () { final m = dailyActiveMinutes(day(120, 30), - personalDynFloorG: null, pooledMinutesAvailable: 640); + personalDynFloorG: null, historyDaysAvailable: 3); expect(m.present, isFalse, reason: 'no constant fallback is permitted'); expect(m.confidence, 0); expect(m.tier, Tier.estimate); - expect(m.note, 'need_baseline:have=640,need=$personalDynFloorMinMinutes'); + // DAYS, not minutes: the only floor builder a storage-bound caller can + // use (personalDynFloorFromDailySummaries) is gated on days, and edge + // passes a day count. This used to read have=3,need=2000. + expect(m.note, 'need_baseline:have=3,need=$personalDynFloorMinDays'); + }); + + test('an off-wrist HOUR between moving minutes does not make one bout', () { + // an-motion-2. Four isolated moving minutes an hour apart. enmoSeries + // emits NO MotionMinute for a fully-absent minute, so the old test + // `idx[end+1] == idx[end] + 1` was adjacency in the gap-COMPACTED list + // and these four sat "consecutive". PRE-FIX: activeMinutes 4, bouts 1. + final scattered = [ + for (final min in [0, 60, 120, 180]) + MotionMinute(min * 60000.0, 60, 0.055, 0.02, 1.055, walkDyn), + ]; + final m = dailyActiveMinutes(scattered, + personalDynFloorG: floorG, minBoutMin: 3); + expect(m.value!.activeMinutes, 0); + expect(m.value!.boutCount, 0); + }); + + test('a run broken by ONE absent minute is two sub-bout stretches', () { + // Same root cause, minimum case: minutes 0,1 then 3,4 (minute 2 absent). + // Adjacent in the list, an hour apart or one minute apart it makes no + // difference — the clock says they are not one run of 4. + final split = [ + for (final min in [0, 1, 3, 4]) + MotionMinute(min * 60000.0, 60, 0.055, 0.02, 1.055, walkDyn), + ]; + final m = + dailyActiveMinutes(split, personalDynFloorG: floorG, minBoutMin: 3); + expect(m.value!.activeMinutes, 0, reason: 'neither stretch reaches 3'); + expect(m.value!.boutCount, 0); + }); + + test('a partially-worn day reports coverage against the DAY, not the span', + () { + // an-motion-5. 4 h worn out of 24. enmoSeries(expectedMinutes: 1440) + // reports 0.167 for this substrate; dailyActiveMinutes reported 1.000 + // (and confidence 0.30, its ceiling) because it divided by the WORN span. + final worn = day(210, 30); // 240 contiguous covered minutes + final honest = dailyActiveMinutes(worn, + personalDynFloorG: floorG, expectedMinutes: 1440); + expect(honest.value!.coverage, closeTo(240 / 1440, 1e-9)); // 0.1667 + expect(honest.confidence, closeTo(0.1, 1e-9), + reason: 'confidence keys off coverage; 0.30 was the pre-fix value'); + // Without expectedMinutes the span denominator still applies — that is a + // different, documented claim, not the fraction of a day. + final spanOnly = dailyActiveMinutes(worn, personalDynFloorG: floorG); + expect(spanOnly.value!.coverage, 1.0); }); test('a non-positive floor is treated as absent, not as "pass everything"', @@ -300,7 +521,8 @@ void main() { expect(m.note, startsWith('need_baseline:')); }); - test('REGRESSION: a day whose sedentary minutes sit just above an ABSOLUTE ' + test( + 'REGRESSION: a day whose sedentary minutes sit just above an ABSOLUTE ' '0.05 g floor produces no active minutes', () { // The measured failure shape: a calibration excursion lifted every // sedentary minute of one day above the old absolute 0.05 g gate, and the @@ -356,8 +578,7 @@ void main() { test('more movement → more active minutes', () { final few = dailyActiveMinutes(day(120, 10), personalDynFloorG: floorG); final many = dailyActiveMinutes(day(120, 40), personalDynFloorG: floorG); - expect(many.value!.activeMinutes, - greaterThan(few.value!.activeMinutes)); + expect(many.value!.activeMinutes, greaterThan(few.value!.activeMinutes)); }); test('NO CEILING: the highest-amplitude minutes still COUNT', () { @@ -386,10 +607,11 @@ void main() { final m = dailyActiveMinutes(day(120, 30), personalDynFloorG: floorG); expect(m.value!.activeMinutes, 30); expect(m.inputs_used, isNot(contains('hr_per_min')), - reason: 'the metric must not claim an HR input it never reads'); + reason: 'the metric must not claim an HR input it never reads'); }); - test('BOUT GATE: an isolated elevated minute does not count on its own', () { + test('BOUT GATE: an isolated elevated minute does not count on its own', + () { final d = List.filled(60, sedDyn); d[30] = walkDyn; final m = dailyActiveMinutes(rows(d), personalDynFloorG: floorG); @@ -402,22 +624,25 @@ void main() { final d2 = List.filled(60, sedDyn); d2[30] = walkDyn; d2[31] = walkDyn; - expect(dailyActiveMinutes(rows(d2), personalDynFloorG: floorG) - .value! - .activeMinutes, + expect( + dailyActiveMinutes(rows(d2), personalDynFloorG: floorG) + .value! + .activeMinutes, 0); final d3 = List.filled(60, sedDyn); d3[30] = walkDyn; d3[31] = walkDyn; d3[32] = walkDyn; - expect(dailyActiveMinutes(rows(d3), personalDynFloorG: floorG) - .value! - .activeMinutes, + expect( + dailyActiveMinutes(rows(d3), personalDynFloorG: floorG) + .value! + .activeMinutes, 3); }); - test('BOUT GATE: a coverage gap breaks the run rather than stitching two ' + test( + 'BOUT GATE: a coverage gap breaks the run rather than stitching two ' 'short bouts together', () { // 4 elevated minutes total but never 3 adjacent, so none of it counts. final d = List.filled(60, sedDyn); @@ -507,8 +732,8 @@ void main() { final x = (axis == 0 ? p : 0.0); final y = (axis == 1 ? p : 0.0) - 0.05; final z = (axis == 2 ? p : 0.0) + 1.03; - out.add(AccelSample( - i * 1000.0, gx * x + bx, gy * y + by, gz * z + bz)); + out.add( + AccelSample(i * 1000.0, gx * x + bx, gy * y + by, gz * z + bz)); } } return out; @@ -538,11 +763,9 @@ void main() { amp: poolAmp, gx: gx, gy: gy, gz: gz, bx: bx, by: by, bz: bz)); final today = enmoSeries(synth(720, amp: dayAmp, gx: gx, gy: gy, gz: gz, bx: bx, by: by, bz: bz)); - final floor = - personalDynFloor([for (final m in pool.minutes) m.dynAmp]); + final floor = personalDynFloor([for (final m in pool.minutes) m.dynAmp]); expect(floor, isNotNull, reason: '2400 pooled minutes is enough history'); - final est = - dailyActiveMinutes(today.minutes, personalDynFloorG: floor); + final est = dailyActiveMinutes(today.minutes, personalDynFloorG: floor); expect(est.present, isTrue); return (active: est.value!.activeMinutes, floor: floor!); } @@ -586,23 +809,61 @@ void main() { weightKg: 80, heightCm: 180, age: 30, sex: 'male'); // 1440 minutes at a low HR (40% HRmax → below active surplus) final hr = List.filled(1440, 70.0); - final e = Calories.dailyEnergy(hr, profile: profile); + final e = Calories.dailyEnergy(hr, profile: profile, hrmax: 190); expect(e.basal, closeTo(1780.0, 1.0)); expect(e.active, closeTo(0.0, 60.0)); // tiny if any expect(e.total, greaterThanOrEqualTo(e.basal)); - // no hrmax passed, so this ran on the fallback anchor - flagged. - expect(e.usedDefaultHrmax, isTrue); }); - test('usedDefaultHrmax is false once a real profile + hrmax are both given', () { + test('the flex gate moves with the CALLER-SUPPLIED ceiling, not 220−age', + () { + // TS-03a: there is no `220 − age` fallback left in here — `hrmax` is + // required, and it is the only thing that sets the flex gate. Two + // ceilings for the same 30 y/o (220−30 = 190 vs Tanaka 208−0.7·30 = 187) + // put the gate at 123.5 vs 121.55 bpm, so a day spent at 122 bpm is + // entirely basal on one and entirely active on the other. That divergence + // is the bug the single dispatched definition exists to remove; this pins + // that the number in front of it is what decides. final profile = const WorkoutUserProfile( weightKg: 80, heightCm: 180, age: 30, sex: 'male'); - final hr = List.filled(1440, 70.0); - final e = Calories.dailyEnergy(hr, profile: profile, hrmax: 190); - expect(e.usedDefaultHrmax, isFalse); + final hr = List.filled(1440, 122.0); + final wide = Calories.dailyEnergy(hr, profile: profile, hrmax: 190); + final tanaka = Calories.dailyEnergy(hr, profile: profile, hrmax: 187); + expect(wide.active, 0.0); + expect(tanaka.active, greaterThan(0.0)); + }); + + test('MOT-02: the flex point is 0.65·HRmax, not 0.50 — a 100 bpm day is ' + 'not "active"', () { + // 0.50 put the gate at 93.5 bpm for a 30 y/o, inside the region where + // Keytel is extrapolating off the end of its own fitted exercise data. + // MEASURED on whoop-4.db (9 days, 70 kg/170 cm/30 y male stand-in, Tanaka + // 187): billed wake minutes 39.4 % → 4.9 %, daily ACTIVE energy + // min/median/max 769/1955/4062 → 9/48/1917 kcal, daily TOTAL + // 1544–5582 → 793–3437 kcal. Everyone's active energy drops; the quiet + // days lose nearly all of it, which is the point. + expect(Calories.defaultActiveFraction, 0.65); + final profile = const WorkoutUserProfile( + weightKg: 70, heightCm: 170, age: 30, sex: 'male'); + // 16 h of ordinary waking at 100 bpm — clears the old gate, not the new. + final quiet = List.filled(960, 100.0); + expect(Calories.dailyEnergy(quiet, profile: profile, hrmax: 187).active, + 0.0); + expect( + Calories.dailyEnergy(quiet, + profile: profile, hrmax: 187, activeFraction: 0.50) + .active, + greaterThan(4000.0), + reason: 'the number the old gate published for sitting around'); + // A real session still bills: 45 min at 145 bpm. + final session = [ + ...List.filled(915, 100.0), + ...List.filled(45, 145.0), + ]; + expect(Calories.dailyEnergy(session, profile: profile, hrmax: 187).active, + closeTo(553.0, 5.0)); }); - test('an exercise block adds active calories on top of basal', () { final profile = const WorkoutUserProfile( weightKg: 80, heightCm: 180, age: 30, sex: 'male'); @@ -640,9 +901,10 @@ void main() { final trackedCounts = []; for (final k in const [1.0, 1.5, 2.0, 3.0]) { final d = gradedDay(k); - frozenCounts.add(dailyActiveMinutes(rowsOf(d), personalDynFloorG: frozen) - .value! - .activeMinutes); + frozenCounts.add( + dailyActiveMinutes(rowsOf(d), personalDynFloorG: frozen) + .value! + .activeMinutes); // What a self-referential floor converges to: this day's own p90. final sorted = [...d]..sort(); final p90 = sorted[(sorted.length * 0.9).floor()]; @@ -683,8 +945,8 @@ void main() { test('enrollment window is longer than the bare minimum for the median', () { - expect(enrollmentDaysForFrozenFloor, - greaterThan(personalDynFloorMinDays)); + expect( + enrollmentDaysForFrozenFloor, greaterThan(personalDynFloorMinDays)); }); }); } diff --git a/test/onehz/strain_calibration_test.dart b/test/onehz/strain_calibration_test.dart index ef8d817..4e6a418 100644 --- a/test/onehz/strain_calibration_test.dart +++ b/test/onehz/strain_calibration_test.dart @@ -81,6 +81,32 @@ void main() { expect(s, lessThanOrEqualTo(21.0)); }); + test('MOT-04: the scale saturates at ~3.25 h at 160 bpm, not 5 h', () { + // The docstring's "5 h at 160 bpm → 21" was loose: with the baseline + // subtraction included, 195 min already tops out, so every session past + // that is the same number. Regenerate this with `quietWakingHrr` if that + // constant ever moves (MOT-03) — it moves every anchor in the table. + expect(strainOfDay(dayHr(960, 85, [(190, 160)])), lessThan(21.0)); + expect(strainOfDay(dayHr(960, 85, [(195, 160)])), closeTo(21.0, 1e-9)); + }); + + test('MOT-04/MOT-03: at this user MEASURED quiet level a nothing-day still ' + 'scores ~12 — the known defect, pinned', () { + // The anchors above put quiet waking at exactly `quietWakingHrr` = 0.20. + // whoop-4.db says this user's real wake minutes sit at p50 0.274 HRR + // (RHR 55 / HRmax 187), and at that level a full-wear day with no + // exercise at all scores in the band the table calls "90 min hard". + // Measured on the real corpus: 6.93 / 11.17 / 11.38 / 11.97 / 12.14 over + // the five quiet days. THIS TEST IS EXPECTED TO FAIL WHEN MOT-03 LANDS — + // when it does, that is the fix arriving, and the anchor table in + // load_trimp.dart has to be regenerated in the same change. + const rhr = 55.0, hrMax = 187.0; + final quietBpm = rhr + 0.274 * (hrMax - rhr); // 91.2 bpm + final trimp = banisterTrimp(List.filled(960, quietBpm), + restingHr: rhr, maxHr: hrMax, sex: Sex.male); + expect(strainScore(trimp.value!, wakeMinutes: 960), closeTo(11.9, 0.6)); + }); + test('short wear with no activity is not scored as effort', () { // Real bundle 2026-07-10: band worn ~135 waking minutes, 23 steps. // The baseline must scale with wear, or a 2-hour inactive wear window diff --git a/test/onehz/util_test.dart b/test/onehz/util_test.dart index 16ae90a..177c50a 100644 --- a/test/onehz/util_test.dart +++ b/test/onehz/util_test.dart @@ -109,6 +109,72 @@ void main() { }); }); + group('multiplicity — benjaminiHochberg', () { + test('known answer, and the step-up keeps q monotone in p', () { + // m = 5, q_(k) = p_(k)·m/k with a running minimum applied from the + // largest p downwards. The raw ratios here are + // [0.04, 0.025, 0.04, 0.05, 0.6] in p order — NOT monotone, and the + // smallest p carries the largest raw ratio. The step-up pulls it down to + // its neighbour, which is the part everyone drops: without it the + // strongest test in the family is refused while a weaker one publishes. + final q = benjaminiHochberg([0.01, 0.008, 0.024, 0.04, 0.6]); + expect(q[1]!, closeTo(0.025, 1e-12)); // raw 0.04, stepped down + expect(q[0]!, closeTo(0.025, 1e-12)); // 0.01·5/2 + expect(q[2]!, closeTo(0.04, 1e-12)); // 0.024·5/3 + expect(q[3]!, closeTo(0.05, 1e-12)); // 0.04·5/4 + expect(q[4]!, closeTo(0.6, 1e-12)); + // Monotone in p, which is the property the step-up buys. + final byP = [0, 1, 2, 3, 4]..sort((a, b) => [ + 0.01, + 0.008, + 0.024, + 0.04, + 0.6 + ][a] + .compareTo([0.01, 0.008, 0.024, 0.04, 0.6][b])); + for (var k = 1; k < byP.length; k++) { + expect(q[byP[k]]!, greaterThanOrEqualTo(q[byP[k - 1]]!)); + } + }); + + test('a family of one is the identity', () { + expect(benjaminiHochberg([0.03]).single!, closeTo(0.03, 1e-12)); + }); + + test('a null p stays null and does NOT count toward the family size', () { + // A test we abstained from is not a test we performed. Counting it would + // punish every other test for a comparison that never ran. + final q = benjaminiHochberg([0.01, null, 0.02]); + expect(q[1], isNull); + expect(q[0]!, closeTo(0.02, 1e-12), reason: 'm = 2, not 3'); + }); + + test('q never goes above 1 or below its own p', () { + final q = benjaminiHochberg([0.9, 0.95, 0.99]); + for (final v in q) { + expect(v!, lessThanOrEqualTo(1.0)); + } + }); + + test('empty in, empty out', () { + expect(benjaminiHochberg(const []), isEmpty); + }); + }); + + group('normalTwoSidedP', () { + test('matches the standard normal at the landmarks', () { + expect(normalTwoSidedP(0), closeTo(1.0, 1e-6)); + expect(normalTwoSidedP(1.959964), closeTo(0.05, 1e-5)); + expect(normalTwoSidedP(2.575829), closeTo(0.01, 1e-5)); + expect(normalTwoSidedP(-1.959964), closeTo(0.05, 1e-5), + reason: 'two-sided: the sign cannot matter'); + }); + + test('a non-finite z is no evidence, not a certainty', () { + expect(normalTwoSidedP(double.nan), 1.0); + }); + }); + test('RrSeries beat-time reconstruction', () { final rr = RrSeries([1000, 1800, 2600], [1000, 800, 800]); expect(rr.beatTimesMs(0), [1000.0, 1800.0, 2600.0]); diff --git a/test/onehz/weekday_effect_test.dart b/test/onehz/weekday_effect_test.dart new file mode 100644 index 0000000..3546762 --- /dev/null +++ b/test/onehz/weekday_effect_test.dart @@ -0,0 +1,134 @@ +// MIND-12 — which day of the week costs you. +// +// The load-bearing test in this file is the FALSE-POSITIVE RATE one. Everything +// downstream of this function — a card, a copy line, a row on the circadian +// screen — is worth exactly nothing if a two-stage gate at alpha 0.05 publishes +// a "worst weekday" out of pure noise more often than about 5% of the time. If +// that test starts failing, the feature does not get retuned, it gets deleted. + +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +/// [n] consecutive real calendar dates from a Monday. +List _dates(int n) { + final start = DateTime(2026, 1, 5); // a Monday + return [ + for (var i = 0; i < n; i++) + () { + final d = start.add(Duration(days: i)); + return '${d.year}-${d.month.toString().padLeft(2, '0')}' + '-${d.day.toString().padLeft(2, '0')}'; + }(), + ]; +} + +/// Box-Muller, so the noise is actually normal rather than uniform-shaped. +double _gauss(math.Random r) => + math.sqrt(-2 * math.log(1 - r.nextDouble())) * + math.cos(2 * math.pi * r.nextDouble()); + +void main() { + group('weekdayEffect', () { + test('PURE NOISE clears at roughly the nominal rate', () { + // 150 independent 12-week histories with no weekday structure at all. + // One of the seven weekdays is always the worst — that is arithmetic, not + // physiology — so a naive "show the worst day" publishes 150 findings + // here. The two-stage gate has to publish about 7. + const trials = 150; + const days = 84; // 12 weeks + final dates = _dates(days); + var published = 0; + var computed = 0; + for (var t = 0; t < trials; t++) { + final rng = math.Random(1000 + t); + final m = weekdayEffect( + dates, + [for (var i = 0; i < days; i++) 50.0 + 8 * _gauss(rng)], + permutations: 199, + seed: 7 + t, + ); + expect(m.present, isTrue, reason: '12 weeks clears the history floor'); + computed++; + if (m.value!.meaningful) published++; + } + expect(computed, trials); + final rate = published / trials; + expect( + rate, + lessThanOrEqualTo(0.10), + reason: 'a two-stage gate at alpha 0.05 publishing $published/$trials ' + 'noise findings means nothing downstream is worth building', + ); + }); + + test('a real weekday shift IS detected', () { + // Every Sunday runs 12 units low against a spread of 4 — a difference a + // person would actually notice, over the same 12 weeks. + const days = 84; + final dates = _dates(days); + final rng = math.Random(3); + final vals = [ + for (var i = 0; i < days; i++) + 50.0 + + 4 * _gauss(rng) + + (DateTime.parse(_dates(days)[i]).weekday == DateTime.sunday + ? -12.0 + : 0.0), + ]; + final m = weekdayEffect(dates, vals, permutations: 199); + expect(m.present, isTrue); + final v = m.value!; + expect(v.meaningful, isTrue); + expect(v.peakWeekday, DateTime.sunday); + expect(v.peakDelta, lessThan(0), reason: 'Sunday is LOW, and says so'); + expect(v.omnibusP, lessThanOrEqualTo(0.05)); + expect(v.peakP, lessThanOrEqualTo(0.05)); + expect(v.nByWeekday[DateTime.sunday], 12); + }); + + test('under eight weeks it is absent, not weak', () { + final dates = _dates(35); // 5 weeks + final m = weekdayEffect( + dates, + [for (var i = 0; i < 35; i++) 50.0 + i % 7], + ); + expect(m.present, isFalse); + expect(m.note, startsWith('need_history:')); + }); + + test('a weekday with too few days blocks the whole test', () { + // 12 weeks of history, but Sundays are almost all missing. A verdict + // about the worst weekday cannot come out of a group of two. + const days = 84; + final dates = _dates(days); + final vals = [ + for (var i = 0; i < days; i++) + (DateTime.parse(dates[i]).weekday == DateTime.sunday && i > 20) + ? null + : 50.0 + (i % 5), + ]; + final m = weekdayEffect(dates, vals); + expect(m.present, isFalse); + expect(m.note, contains('min_per_weekday')); + }); + + test('a misaligned series is refused, not truncated', () { + expect( + weekdayEffect(_dates(84), const [1.0, 2.0]).present, + isFalse, + ); + }); + + test('the same history always gives the same answer', () { + const days = 84; + final dates = _dates(days); + final vals = [for (var i = 0; i < days; i++) 50.0 + (i % 11) * 0.7]; + final a = weekdayEffect(dates, vals, permutations: 199); + final b = weekdayEffect(dates, vals, permutations: 199); + expect(a.value!.omnibusP, b.value!.omnibusP); + expect(a.value!.peakP, b.value!.peakP); + }); + }); +} diff --git a/test/onehz/wellness_test.dart b/test/onehz/wellness_test.dart index fcf93ea..f631964 100644 --- a/test/onehz/wellness_test.dart +++ b/test/onehz/wellness_test.dart @@ -29,24 +29,102 @@ void main() { for (var i = 0; i < 3 * 24 * 6; i++) { final tMs = i * 10 * 60 * 1000.0; final tHours = tMs / 3.6e6; - final adc = 2000 + - 300 * math.cos(2 * math.pi / 24 * (tHours - peakHour)); + final adc = + 2000 + 300 * math.cos(2 * math.pi / 24 * (tHours - peakHour)); samples.add(AdcSample(tMs, adc)); } - final m = tempCircadian(samples, epochMin: 60); + final m = tempCircadian(samples, deviceFamily: 'gen4', epochMin: 60); expect(m.present, isTrue); expect(m.tier, Tier.relative); final fit = m.value!.cosinorFit!; expect(fit.acrophaseHours, closeTo(peakHour, 0.3)); expect(fit.amplitude, closeTo(300, 15)); expect(fit.r2, greaterThan(0.95)); - // Nonparametric: strong clean rhythm => high IS, low IV, high RA. + // Nonparametric: strong clean rhythm => high IS, low IV. final np = m.value!.nonparam!; expect(np.interdailyStability, greaterThan(0.7)); expect(np.intradailyVariability, lessThan(0.5)); - expect(np.relativeAmplitude, greaterThan(0.05)); - // The warmest 10-h window should centre near the peak. - expect(np.m10, greaterThan(np.l5)); + }); + + test('RD-14: IS is taken about the GRAND mean, not the profile mean', () { + // Unbalanced epoch-of-day coverage — the normal case on real data, where + // per-day 24-bin coverage runs 14/24 to 24/24 — plus a level drift across + // days. Hours 12-23 exist only on the highest day, so the 24-h profile's + // own mean sits above the grand mean. + final samples = []; + for (var day = 0; day < 3; day++) { + final hours = day == 2 ? 24 : 12; + for (var h = 0; h < hours; h++) { + for (var k = 0; k < 6; k++) { + final tHours = day * 24.0 + h + k / 6.0; + final adc = 2000 + + 100.0 * day + + 150 * math.cos(2 * math.pi / 24 * (tHours - 4)); + samples.add(AdcSample(tHours * 3.6e6, adc)); + } + } + } + final np = tempCircadian(samples, deviceFamily: 'gen4', epochMin: 60) + .value! + .nonparam!; + // van Someren's IS puts BOTH variances about the grand mean. Using the + // profile's own mean minimises the numerator by construction and gives + // 0.5299 on this fixture — biased low, always in the same direction. + expect(np.interdailyStability, closeTo(0.5582, 0.0005)); + }); + + test('M10/L5/RA are WITHHELD, not merely null (MT-09)', () { + final samples = []; + for (var i = 0; i < 3 * 24 * 6; i++) { + final tMs = i * 10 * 60 * 1000.0; + final tHours = tMs / 3.6e6; + samples.add( + AdcSample(tMs, 2000 + 300 * math.cos(2 * math.pi / 24 * tHours))); + } + final m = tempCircadian(samples, deviceFamily: 'gen4', epochMin: 60); + final j = m.value!.nonparam!.toJson(); + // Not "absent when unobserved" — absent from the shape entirely, because + // RA on a median-centred series divides by a quantity that crosses zero. + for (final k in [ + 'm10', + 'l5', + 'm10_onset_hour', + 'l5_onset_hour', + 'relative_amplitude' + ]) { + expect(j.containsKey(k), isFalse, reason: k); + } + expect(j.containsKey('interdaily_stability'), isTrue); + }); + + test('the family decides the unit and the gate; unknown refuses (MT-09)', + () { + final samples = []; + for (var i = 0; i < 3 * 24 * 6; i++) { + final tMs = i * 10 * 60 * 1000.0; + samples.add(AdcSample( + tMs, 2000 + 300 * math.cos(2 * math.pi / 24 * (tMs / 3.6e6)))); + } + expect(tempCircadian(samples, deviceFamily: 'gen4').value!.unit, + 'adc_counts'); + expect( + tempCircadian(samples, deviceFamily: 'gen5').value!.unit, 'centi_c'); + for (final id in [null, '', 'imported']) { + final m = tempCircadian(samples, deviceFamily: id); + expect(m.present, isFalse, reason: 'id=$id'); + expect(m.note, unknownFamilyNote(id)); + } + + // The de-mask gate is per-family: 0.06 g of wrist motion is inside gen4's + // own resting noise floor and outside gen5's. + final accel = [ + for (final s in samples) AccelSample(s.tsMs, 0, 0, 1.06), + ]; + expect(tempCircadian(samples, deviceFamily: 'gen4', accel: accel).note, + contains('dropped=0')); + expect(tempCircadian(samples, deviceFamily: 'gen5', accel: accel).present, + isFalse, + reason: 'gen5 masks all of it, leaving nothing to fit'); }); test('activity de-masking drops high-motion epochs', () { @@ -59,13 +137,14 @@ void main() { final motion = i % 3 == 0 ? 0.5 : 0.0; accel.add(AccelSample(tMs, 1.0 + motion, 0, 0)); } - final m = tempCircadian(samples, accel: accel, motionGate: 0.08); + final m = tempCircadian(samples, + deviceFamily: 'gen4', accel: accel, motionGate: 0.08); expect(m.inputs_used, contains('accel')); expect(m.note, contains('demasked')); }); test('absent on too-few epochs', () { - final m = tempCircadian([AdcSample(0, 2000)]); + final m = tempCircadian([AdcSample(0, 2000)], deviceFamily: 'gen4'); expect(m.present, isFalse); expect(m.confidence, 0); }); @@ -95,11 +174,15 @@ void main() { test('luteal phase suppresses the illness flag (confound tag)', () { final temp = [ for (var i = 0; i < 14; i++) 2000.0 + (i.isEven ? 3 : -3), - 2060, 2065, 2070, + 2060, + 2065, + 2070, ]; final luteal = [ for (var i = 0; i < 14; i++) false, - true, true, true, + true, + true, + true, ]; final out = tempIllnessFlag(dates(temp.length), temp, luteal: luteal, baselineDays: 21, zThresh: 2.0, persistDays: 2); @@ -142,7 +225,9 @@ void main() { }); test('no shift => no confirmation event', () { - final temp = [for (var i = 0; i < 20; i++) 2000.0 + (i.isEven ? 1 : -1)]; + final temp = [ + for (var i = 0; i < 20; i++) 2000.0 + (i.isEven ? 1 : -1) + ]; final m = menstrualCoverline( [for (var i = 0; i < temp.length; i++) 'd$i'], temp, threshold: 1.5); @@ -226,15 +311,152 @@ void main() { reason: 'no regression-to-mean false split'); }); - test('cusumChangePoints fires on an online upward step', () { - final x = [ - for (var i = 0; i < 30; i++) 10.0 + (i.isEven ? 0.2 : -0.2), - for (var i = 0; i < 15; i++) 15.0, // sustained upward shift - ]; - final dets = cusumChangePoints(x, k: 0.5, h: 4.0); + // an-wellness-3. The old case here used a deliberately IMBALANCED 30-low / + // 15-high split, which put the whole-series median inside the low regime + // and kept the MAD small — so it passed while hiding the actual behaviour. + // Standardizing by the whole series folds the post-change data into the + // scale, which pins |z| at 1/1.4826 ~ 0.675 for ANY balanced two-regime + // series whatever the step size: detection depended only on how LONG the + // regimes were, never on how big the shift was. + + // A deterministic 10-day noise cycle: median 0, MAD 1.4826. + const noise = [0, 1, -1, 2, -2, 1, -1, 0, 2, -2]; + List twoRegime(double base, double step, {int perRegime = 30}) => [ + for (var i = 0; i < perRegime; i++) base + noise[i % 10], + for (var i = 0; i < perRegime; i++) base + step + noise[i % 10], + ]; + + test('a BALANCED step is detected at every size, once, on the high side', + () { + for (final step in [3.0, 7.0, 15.0, 60.0, 145.0]) { + final dets = cusumChangePoints(twoRegime(55.0, step), h: 5.0); + expect(dets, hasLength(1), reason: 'step $step announced once'); + expect(dets.single.direction, 1); + expect(dets.single.index, inInclusiveRange(30, 36), + reason: 'step $step detected near the change, not drifted'); + } + // Downward steps too. + final down = cusumChangePoints(twoRegime(200.0, -60.0), h: 5.0); + expect(down, hasLength(1)); + expect(down.single.direction, -1); + }); + + test('a large step in a SHORT window is not invisible', () { + // 30 days, 55 -> 200. The whole-series scale made this return []. + final dets = + cusumChangePoints(twoRegime(55.0, 145.0, perRegime: 15), h: 5.0); expect(dets, isNotEmpty); + expect(dets.first.index, inInclusiveRange(15, 17)); expect(dets.first.direction, 1); - expect(dets.first.index, greaterThanOrEqualTo(30)); + }); + + test('one shift is announced ONCE under the caller`s latest-day gate', () { + // derivation_engine.dart replays a growing window and notifies only when + // `dets.last.index == n - 1`. On the whole-series scale that fired on + // n = 31,32,34,36,39,42,46,50,54,58 — ten critical-priority notifications + // about one shift. + final x = twoRegime(55.0, 10.0); + var fires = 0; + for (var n = 10; n <= x.length; n++) { + final dets = cusumChangePoints(x.sublist(0, n), h: 5.0); + if (dets.isNotEmpty && dets.last.index == n - 1) fires++; + } + expect(fires, 1); + }); + + test('no detection when a robust scale cannot be estimated', () { + // A perfectly constant baseline has no dispersion to standardize + // against; an absent change-point beats a fabricated one. + final x = [ + for (var i = 0; i < 30; i++) 55.0, + for (var i = 0; i < 30; i++) 200.0, + ]; + expect(cusumChangePoints(x, h: 5.0), isEmpty); + }); + + test('no detection on a series that never shifts', () { + final x = [for (var i = 0; i < 60; i++) 55.0 + noise[i % 10]]; + expect(cusumChangePoints(x, h: 5.0), isEmpty); + }); + + test('a series shorter than the baseline requirement yields nothing', () { + expect(cusumChangePoints([for (var i = 0; i < 10; i++) 55.0 + i], h: 5.0), + isEmpty); + }); + + // MT-10. The BIC penalty is a significance test; the MDC gate is an + // effect-size test. A step the instrument cannot resolve is not a finding + // however clean it is, and this is the guard that gets quietly dropped. + test('a statistically clean step SMALLER than the MDC is not reported', () { + // Noisy pre-segment (MAD ~1.5 bpm ⇒ MDC ~4 bpm) with a perfectly clean + // 2 bpm step: binary segmentation loves it, the body cannot resolve it. + const noise10 = [0, 1, -1, 2, -2, 1, -1, 0, 2, -2]; + final x = [ + for (var i = 0; i < 30; i++) 55.0 + noise10[i % 10], + for (var i = 0; i < 30; i++) 57.0 + noise10[i % 10], + ]; + final m = segmentChangePoints(x, minSeg: 7); + expect(m.value!.changePoints, isEmpty); + expect(m.note, contains('dropped=1')); + expect(m.note, contains('UNDER-SPLITS'), + reason: 'an empty result is not evidence of stability'); + + // The same series with a 6 bpm step clears the MDC and is reported. + final big = [ + for (var i = 0; i < 30; i++) 55.0 + noise10[i % 10], + for (var i = 0; i < 30; i++) 61.0 + noise10[i % 10], + ]; + expect(segmentChangePoints(big, minSeg: 7).value!.changePoints, + hasLength(1)); + }); + + // STAT-05. The penalty shipped at HALF the BIC it claimed to be + // (`penaltyK = 1.0` where BIC = (diffparam+1)·log n = 2·σ̂²·ln n for the + // Normal change-in-mean), so the search proposed boundaries the criterion + // it names would never have proposed — and only the MDC gate downstream + // stopped them reaching the user. + test('the penalty is BIC, not half of it (STAT-05)', () { + const noise10 = [0, 1, -1, 2, -2, 1, -1, 0, 2, -2]; + final x = [ + for (var i = 0; i < 30; i++) 55.0 + noise10[i % 10], + for (var i = 0; i < 30; i++) 56.0 + noise10[i % 10], + ]; + // At the shipped-then half-penalty the search DID split here. + expect(segmentChangePoints(x, minSeg: 7, penaltyK: 1.0).note, + contains('dropped=1')); + // At BIC it never proposes the boundary at all. + expect(segmentChangePoints(x, minSeg: 7).note, contains('dropped=0')); + expect(segmentChangePoints(x, minSeg: 7).value!.changePoints, isEmpty); + }); + + // STAT-07. `cusumChangePoints` was purely positional, and its only + // production caller feeds it a COMPACTED series (days with no nocturnal RHR + // are skipped — "most days for some users"). So a pre-change regime spanned + // wear gaps and the accumulator carried evidence across months. + test('a wear gap breaks the regime and the accumulator (STAT-07)', () { + const noise10 = [0, 1, -1, 2, -2, 1, -1, 0, 2, -2]; + final x = [ + for (var i = 0; i < 30; i++) 55.0 + noise10[i % 10], + for (var i = 0; i < 30; i++) 70.0 + noise10[i % 10], + ]; + String day(int n) => DateTime.utc(2026, 1, 1) + .add(Duration(days: n)) + .toIso8601String() + .substring(0, 10); + // 30 consecutive days, 60 days off-wrist, then 30 consecutive days at a + // different level. Positionally that is one clean step. + final dates = [ + for (var i = 0; i < 30; i++) day(i), + for (var i = 0; i < 30; i++) day(90 + i), + ]; + expect(cusumChangePoints(x, h: 5.0), isNotEmpty, + reason: 'positional: the gap is invisible'); + expect(cusumChangePoints(x, dates: dates, h: 5.0), isEmpty, + reason: 'no observations across the gap ⇒ no evidence to carry'); + // Consecutive dates leave the old behaviour exactly as it was. + final contiguous = [for (var i = 0; i < 60; i++) day(i)]; + expect(cusumChangePoints(x, dates: contiguous, h: 5.0), + hasLength(cusumChangePoints(x, h: 5.0).length)); }); }); @@ -253,7 +475,7 @@ void main() { hrvInput(40.0, around(60.0)), // far below baseline => bad rhrInput(55.0, around(55.0)), // at baseline respInput(14.0, around(14.0)), - tempInput(2000.0, around(2000.0)), + tempInput(2000.0, around(2000.0), settledFraction: 0.97), ]); expect(m.present, isTrue); expect(m.drivers, isNotNull); @@ -261,25 +483,27 @@ void main() { expect(m.drivers!.first.label, 'HRV'); // HRV dropped => negative contribution => below 50. expect(m.value!.score, lessThan(50)); - expect(m.value!.meaningful, isTrue); expect(m.inputs_used, contains('HRV')); }); - test('all-at-baseline => ~50 and NOT meaningful (SWC gate)', () { + test('all-at-baseline => ~50', () { final m = readinessComposite([ hrvInput(60.0, around(60.0)), rhrInput(55.0, around(55.0)), respInput(14.0, around(14.0)), - tempInput(2000.0, around(2000.0)), + tempInput(2000.0, around(2000.0), settledFraction: 0.97), ]); expect(m.value!.score, closeTo(50, 8)); - expect(m.value!.meaningful, isFalse); + expect(m.value!.toJson().containsKey('meaningful'), isFalse); }); test('weights renormalize over present inputs; absent => "—"', () { - final present = readinessComposite([hrvInput(70.0, around(60.0))]); + final present = readinessComposite([ + hrvInput(70.0, around(60.0)), + rhrInput(55.0, around(55.0)), + ]); expect(present.present, isTrue); - expect(present.inputs_used, ['HRV']); + expect(present.inputs_used, ['HRV', 'RHR']); final none = readinessComposite([ hrvInput(null, around(60.0)), @@ -293,7 +517,8 @@ void main() { expect(base.length, 14); }); - test('degenerate-MAD baseline is rescued by mean/SD z (no intermittent "—")', + test( + 'degenerate-MAD baseline is rescued by mean/SD z (no intermittent "—")', () { // A quantized RHR whose recent baseline clusters tight enough that the // median-absolute-deviation collapses to 0 (deviations from the median 52 @@ -301,7 +526,10 @@ void main() { // before the fallback the WHOLE composite blanked to "—" here — the // intermittent "readiness sometimes disappears (with sleep present)" bug. final tightBase = [52, 52, 52, 52, 52, 52, 53]; - final m = readinessComposite([rhrInput(56.0, tightBase)]); + // minInputs:1 — the subject here is the MAD==0 rescue, not the RD-04 + // minimum-inputs gate (which has its own test below). + final m = readinessComposite([rhrInput(56.0, tightBase)], + minInputs: 1, minWeightSum: 0.0); expect(m.present, isTrue, reason: 'mean/SD z should rescue a MAD==0 (but SD>0) baseline'); expect(m.inputs_used, ['RHR']); @@ -310,7 +538,9 @@ void main() { // A TRULY constant baseline (SD == 0 too) still honestly abstains — we // only rescue degenerate MAD, never fabricate against zero dispersion. - final flat = readinessComposite([rhrInput(56.0, [52, 52, 52, 52])]); + final flat = readinessComposite([ + rhrInput(56.0, [52, 52, 52, 52]) + ]); expect(flat.present, isFalse); }); }); @@ -326,10 +556,8 @@ void main() { markTestSkipped('whoop_hist.jsonl not found beside the repo'); return; } - final lines = histFile - .readAsLinesSync() - .where((l) => l.trim().isNotEmpty) - .toList(); + final lines = + histFile.readAsLinesSync().where((l) => l.trim().isNotEmpty).toList(); final temp = []; final accel = []; var firstTs = 0, lastTs = 0; @@ -355,7 +583,8 @@ void main() { } // Cosinor runs (the ~9-min span has no full day => low R²/short, but it // MUST NOT crash and MUST return an honest Metric). - final m = tempCircadian(temp, accel: accel, epochMin: 1); + final m = + tempCircadian(temp, deviceFamily: 'gen4', accel: accel, epochMin: 1); expect(m.tier, Tier.relative); // ignore: avoid_print print('REAL temp cosinor: present=${m.present} ' @@ -371,7 +600,8 @@ void main() { }); group('baseline-need signals (need_baseline convention)', () { - test('readinessComposite: value present but 1-day baseline -> absent + need', + test( + 'readinessComposite: value present but 1-day baseline -> absent + need', () { final inputs = [ hrvInput(50.0, [48.0]), // value present, baseline length 1 (< min 3) @@ -379,10 +609,12 @@ void main() { final m = readinessComposite(inputs); expect(m.present, isFalse); expect(m.confidence, 0); - expect(m.note, 'need_baseline:have=1,need=$readinessCompositeMinBaseline'); + expect( + m.note, 'need_baseline:have=1,need=$readinessCompositeMinBaseline'); // With >= minBaseline points it computes. final ok = readinessComposite([ hrvInput(60.0, [48.0, 49.0, 50.0, 51.0, 52.0]), + rhrInput(55.0, [54.0, 55.0, 56.0, 55.0, 54.0]), ]); expect(ok.present, isTrue); }); @@ -424,7 +656,8 @@ void main() { // not floored to an epsilon scale. // ------------------------------------------------------------------------- group('multivariateAnomaly — degenerate baseline (regression)', () { - test('an exactly-constant baseline column is DROPPED, never floored to 1e-6', + test( + 'an exactly-constant baseline column is DROPPED, never floored to 1e-6', () { // Ten baseline nights whose skin-temp z is an exactly-constant quantized // 0.0 (MAD == 0 AND SD == 0), alongside a real HRV column, then a night @@ -456,8 +689,7 @@ void main() { final feats = [ for (var i = 0; i < 12; i++) AnomalyFeatures( - hrv: 40.0 + (i.isEven ? 1.0 : -1.0), - temp: i.isEven ? 0.1 : -0.1), + hrv: 40.0 + (i.isEven ? 1.0 : -1.0), temp: i.isEven ? 0.1 : -0.1), ]; final dates = [for (var i = 0; i < feats.length; i++) 'd$i']; final out = @@ -477,7 +709,8 @@ void main() { // robustZ abstains and the deliberate `?? z(v, base)` fallback (#26) // produced the contribution. PRE-FIX the detail still said "robust-z". final base = [55, 55, 55, 55, 55, 55, 55, 58]; - final m = readinessComposite([rhrInput(60, base)]); + final m = readinessComposite([rhrInput(60, base)], + minInputs: 1, minWeightSum: 0.0); expect(m.present, isTrue, reason: m.note); final d = m.drivers!.single; expect(d.detail, contains('mean+SD fallback')); @@ -486,16 +719,234 @@ void main() { test('a dispersed baseline still discloses robust-z (median+MAD)', () { final base = [50, 52, 54, 56, 58, 60, 62]; - final m = readinessComposite([rhrInput(70, base)]); + final m = readinessComposite([rhrInput(70, base)], + minInputs: 1, minWeightSum: 0.0); expect(m.present, isTrue, reason: m.note); expect(m.drivers!.single.detail, contains('robust-z (median+MAD)')); }); test('a fully constant baseline (MAD=0 AND SD=0) still abstains', () { // The fallback is NOT a licence to score against zero dispersion. - final m = readinessComposite([rhrInput(60, List.filled(8, 55.0))]); + final m = + readinessComposite([rhrInput(60, List.filled(8, 55.0))]); + expect(m.present, isFalse); + expect(m.toJson()['value'], '—'); + }); + }); + + // ------------------------------------------------------------------------- + // RD-15 / RD-05 — a warming strap must not read as a recovered person. + // + // Numbers pinned from whoop-4.db, the 8 real gen4 nights the audit measured + // (sleep windows from v_hypnogram, skin_temp_raw from decoded_onehz). Nightly + // means recomputed here match the shipped `skin_temp_adc` series exactly. + // ------------------------------------------------------------------------- + group('nightlySkinTemp (RD-15 settled fraction)', () { + // One night, 1 Hz: 8 h at 805 counts with a 2 h segment ~100 counts colder + // in the middle — the shape of 2026-08-14, whose real settled fraction is + // 0.787 and whose plain mean (784.20) sat -3.17 robust-z below the trailing + // baseline, i.e. +7.6 readiness points of "recovery" from a cold strap. + List night({required int coldSec, int totalSec = 28800}) => [ + for (var i = 0; i < totalSec; i++) + AdcSample( + i * 1000.0, (i >= 6000 && i < 6000 + coldSec) ? 700.0 : 805.0), + ]; + + test('a settled night keeps its samples and its mean', () { + final m = nightlySkinTemp(night(coldSec: 0), deviceFamily: 'gen4'); + expect(m.present, isTrue, reason: m.note); + expect(m.value!.settledFraction, 1.0); + expect(m.value!.mean, closeTo(805.0, 1e-9)); + expect(m.value!.unit, 'adc_counts'); + }); + + test('a cold segment is trimmed out of the mean', () { + // 5 % cold: passes the 0.80 floor, but the plain mean would be 799.75. + final m = nightlySkinTemp(night(coldSec: 1440), deviceFamily: 'gen4'); + expect(m.present, isTrue, reason: m.note); + expect(m.value!.settledFraction, closeTo(0.95, 1e-9)); + expect(m.value!.mean, closeTo(805.0, 1e-9)); + }); + + test('a night that spent 25 % below skin temperature REFUSES', () { + final m = nightlySkinTemp(night(coldSec: 7200), deviceFamily: 'gen4'); expect(m.present, isFalse); + expect(m.note, startsWith('unsettled_skin_temp:settled=0.75')); expect(m.toJson()['value'], '—'); }); + + test('a FEVER is not trimmed — the band is one-sided', () { + // The exact mirror of the refused night above: same 25 % excursion, same + // 100-count size, only UPWARD. It survives whole, because "not skin" is a + // statement about cold readings and a fever is the signal this channel + // exists to carry. + final hot = [ + for (var i = 0; i < 28800; i++) + AdcSample(i * 1000.0, (i >= 6000 && i < 13200) ? 905.0 : 805.0), + ]; + final m = nightlySkinTemp(hot, deviceFamily: 'gen4'); + expect(m.present, isTrue, reason: m.note); + expect(m.value!.settledFraction, 1.0); + expect(m.value!.mean, closeTo(830.0, 1e-9)); + }); + + test('gen5 has no measured band and unknown families refuse', () { + for (final id in ['gen5', null, '', 'gen6']) { + final m = nightlySkinTemp(night(coldSec: 0), deviceFamily: id); + expect(m.present, isFalse, reason: 'family $id must not borrow gen4'); + expect(m.note, startsWith('unknown_device_family:')); + } + }); + + test('under the 60-sample floor it says how short it was', () { + final m = nightlySkinTemp( + [for (var i = 0; i < 30; i++) AdcSample(i * 1000.0, 805.0)], + deviceFamily: 'gen4'); + expect(m.present, isFalse); + expect(m.note, 'need_baseline:have=30,need=60'); + }); + }); + + group('tempInput settled gate (RD-05)', () { + List around(double c) => + [for (var i = 0; i < 14; i++) c + (i.isEven ? 1.0 : -1.0)]; + + // The whoop-4 2026-08-14 shape: temp reads far BELOW baseline and + // goodSign = -1 turns that into "good", so readiness goes UP. + List inputs(double? settled) => [ + hrvInput(60.0, around(60.0)), + rhrInput(55.0, around(55.0)), + tempInput(784.2, around(805.0), settledFraction: settled), + ]; + + test('an unsettled night drops the temp driver, it never pays out', () { + final ungated = readinessComposite(inputs(0.97)); + final gated = readinessComposite(inputs(0.787)); // the real 08-14 value + expect(ungated.inputs_used, contains('temp')); + expect(gated.inputs_used, isNot(contains('temp'))); + expect(gated.value!.score, lessThan(ungated.value!.score), + reason: 'a cold strap was buying readiness points'); + expect(gated.note, contains('unsettled_skin_temp:settled=0.787')); + }); + + test('an unmeasured settled fraction is a refusal, not a pass', () { + final m = readinessComposite(inputs(null)); + expect(m.inputs_used, isNot(contains('temp'))); + expect(m.note, contains('no settled fraction measured')); + }); + + test('the channel itself is untouched — a settled night still drives', () { + final m = readinessComposite(inputs(0.90)); + expect(m.inputs_used, contains('temp')); + expect(m.drivers!.map((d) => d.label), contains('temp')); + }); + }); + + group('readinessComposite minimum inputs (RD-04)', () { + List around(double c) => + [for (var i = 0; i < 14; i++) c + (i.isEven ? 1.0 : -1.0)]; + + test('one surviving input is not a composite', () { + // temp alone: its disclosed 0.10 would renormalise to an effective 1.0. + final m = readinessComposite( + [tempInput(760.0, around(805.0), settledFraction: 0.99)]); + expect(m.present, isFalse); + expect(m.note, startsWith('need_inputs:have=1,need=2')); + expect(m.inputs_used, ['temp']); + }); + + test('RR + temp clear the count but not the weight floor', () { + final m = readinessComposite([ + respInput(14.0, around(14.0)), + tempInput(805.0, around(805.0), settledFraction: 0.99), + ]); + expect(m.present, isFalse, reason: '0.20 + 0.10 = 0.30 < 0.5'); + expect(m.note, contains('need_weight=0.5')); + }); + + test('HRV + RHR compute', () { + final m = readinessComposite([ + hrvInput(60.0, around(60.0)), + rhrInput(55.0, around(55.0)), + ]); + expect(m.present, isTrue, reason: m.note); + }); + }); + + // ------------------------------------------------------------------------- + // RD-07 — the chi-square gate assumed a KNOWN scale. Pinned against the + // 200k-trial Monte Carlo in the docstring of `_madInflationTable`. + // ------------------------------------------------------------------------- + group('multivariateAnomaly small-sample gate (RD-07)', () { + /// Deterministic pseudo-normal noise — no dart:math Random seeding games, + /// just a fixed sequence with mean 0 and unit-ish spread. + double noise(int i) => + math.sin(i * 1.7) + 0.6 * math.sin(i * 0.31) + 0.4 * math.cos(i * 2.9); + + test('the gate widens when the baseline is small and relaxes as it grows', + () { + // The audit's measured false-candidate rate against the bare chi2(0.999) + // gate was 0.197/night at n=10, dof=4 — 196x nominal. The widening below + // is the empirical 99.9 %ile of d2 over that quantile. + List run(int n) { + final feats = [ + for (var i = 0; i < n + 1; i++) + AnomalyFeatures( + rhr: 55 + noise(i), + hrv: 4.0 + 0.1 * noise(i + 40), + temp: 805 + 6 * noise(i + 80), + resp: 15 + noise(i + 120), + ), + ]; + final dates = [ + for (var i = 0; i < feats.length; i++) + DateTime.utc(2026, 1, 1) + .add(Duration(days: i)) + .toIso8601String() + .substring(0, 10) + ]; + return multivariateAnomaly(dates, feats, + baselineDays: 400, minBaseline: 10, persistDays: 2); + } + + // Same night, same distance — only the gate moves with the baseline size. + final small = run(10).last; + final large = run(120).last; + expect(small.mahalanobis, isNotNull); + expect(large.mahalanobis, isNotNull); + // An explicit gate bypasses the widening entirely (unchanged contract). + final fixed = multivariateAnomaly( + [for (var i = 0; i < 12; i++) 'd$i'], + [ + for (var i = 0; i < 12; i++) + AnomalyFeatures(rhr: 55 + noise(i), hrv: 4.0 + 0.1 * noise(i + 40)), + ], + minBaseline: 10, + chiSqGate: 0.0, + ); + expect(fixed.where((d) => d.candidate), isNotEmpty, + reason: 'chiSqGate must still be honoured verbatim'); + }); + + test('a d2 that clears bare chi2 at n=10 no longer becomes a candidate', + () { + // dof 2 => bare gate 13.82. Ten baseline nights of tight noise then one + // night ~4 robust-z out on both features: d2 lands well over 13.82 but + // under 13.82 x 13.23, which is where the measured 99.9 %ile actually is. + final feats = [ + for (var i = 0; i < 10; i++) + AnomalyFeatures(rhr: 55 + noise(i), hrv: 4.0 + 0.1 * noise(i + 40)), + const AnomalyFeatures(rhr: 59.0, hrv: 3.65), + ]; + final dates = [for (var i = 0; i < feats.length; i++) 'd$i']; + final bare = + multivariateAnomaly(dates, feats, minBaseline: 10, chiSqGate: 13.82); + final widened = multivariateAnomaly(dates, feats, minBaseline: 10); + expect( + bare.last.mahalanobis! * bare.last.mahalanobis!, greaterThan(13.82)); + expect(bare.last.candidate, isTrue); + expect(widened.last.candidate, isFalse, + reason: 'a MAD over ten nights is not a known scale'); + }); }); } diff --git a/test/onehz/workout_test.dart b/test/onehz/workout_test.dart deleted file mode 100644 index 7fc683b..0000000 --- a/test/onehz/workout_test.dart +++ /dev/null @@ -1,677 +0,0 @@ -import 'package:openstrap_analytics/onehz.dart'; -import 'package:test/test.dart'; - -// Build a 1 Hz HR series: [restBpm] for [restBeforeS], [workBpm] for [workS], -// then [restBpm] again for [restAfterS], starting at epoch [t0]. -({List ts, List bpm}) _hrDay({ - int t0 = 0, - int restBeforeS = 600, - int restBpm = 60, - int workS = 900, - int workBpm = 140, - int restAfterS = 600, -}) { - final ts = []; - final bpm = []; - var t = t0; - for (var i = 0; i < restBeforeS; i++, t++) { - ts.add(t); - bpm.add(restBpm); - } - for (var i = 0; i < workS; i++, t++) { - ts.add(t); - bpm.add(workBpm); - } - for (var i = 0; i < restAfterS; i++, t++) { - ts.add(t); - bpm.add(restBpm); - } - return (ts: ts, bpm: bpm); -} - -void main() { - group('AutoWorkoutDetector (suggestion)', () { - test('detects a sustained elevated-HR bout in the right window', () { - final d = _hrDay(); // 60 bpm rest, 140 bpm for 900 s (15 min) - final out = AutoWorkoutDetector.detect( - hrTs: d.ts, - hrBpm: d.bpm, - restingBpm: 60, - ); - expect(out, hasLength(1)); - final w = out.first; - // floor = max(60+40, 60+.45*(190-60)) = 119; work span 140 bpm. window = [600, 1499]. - expect(w.startSec, 600); - expect(w.endSec, 1499); - expect(w.avgBpm, 140); - expect(w.peakBpm, 140); - expect(w.durationMin, (1499 - 600) ~/ 60); // = 14 - expect(w.sport, 'detected'); // default seam - }); - - test('below-threshold day → no suggestion', () { - // Elevated for only 5 min (< 12 min minSustained). - final d = _hrDay(workS: 300); - final out = AutoWorkoutDetector.detect( - hrTs: d.ts, - hrBpm: d.bpm, - restingBpm: 60, - ); - expect(out, isEmpty); - }); - - test('HR never above RHR+30 → none', () { - final d = _hrDay(workBpm: 85); // floor 90 → never elevated - final out = - AutoWorkoutDetector.detect(hrTs: d.ts, hrBpm: d.bpm, restingBpm: 60); - expect(out, isEmpty); - }); - - test('overlap-dedup drops a bout overlapping a saved span', () { - final d = _hrDay(); - final saved = [const SavedWorkoutSpan(900, 1000)]; // inside the bout - final out = AutoWorkoutDetector.detect( - hrTs: d.ts, - hrBpm: d.bpm, - restingBpm: 60, - savedSpans: saved, - ); - expect(out, isEmpty); - }); - - test('motion confirmation gate: high motion keeps it', () { - final d = _hrDay(); - final highMotion = [ - for (var t = 600; t <= 1499; t++) MotionPoint(t, 0.5), - ]; - final kept = AutoWorkoutDetector.detect( - hrTs: d.ts, - hrBpm: d.bpm, - restingBpm: 60, - motion: highMotion, - ); - expect(kept, hasLength(1)); - }); - - test('motion gate + onset bypass: sharp HR onset from rest keeps a ' - 'low-motion bout (cycling/rowing — wrist stays still)', () { - // Same shape as _hrDay(): 60 bpm rest for 600 s, then a sharp step to - // 140 bpm — a genuine exercise onset even though the wrist motion stays - // near-zero throughout (handlebar/oar grip). - final d = _hrDay(); - final lowMotion = [ - for (var t = 600; t <= 1499; t++) MotionPoint(t, 0.01), - ]; - final out = AutoWorkoutDetector.detect( - hrTs: d.ts, - hrBpm: d.bpm, - restingBpm: 60, - motion: lowMotion, - ); - expect(out, hasLength(1)); - expect(out.first.startSec, 600); - }); - - test('motion gate + onset bypass: a slow-drifting elevation with no ' - 'discernible start (fever/heat/anxiety) stays dropped even at low ' - 'motion', () { - // Ramp gradually from 60 to 140 over 10 min (well under onsetRiseBpm's - // 25 bpm/3 min bar at any point), then hold — no sharp onset anywhere, - // so the low-motion gate must still reject it. - final ts = []; - final bpm = []; - var t = 0; - for (; t < 600; t++) { - ts.add(t); - bpm.add(60); - } - // 600 s ramp: +8 bpm/min over 10 min => 60 -> 140. - for (var i = 0; i < 600; i++, t++) { - ts.add(t); - bpm.add(60 + (i * 80 / 600).round()); - } - for (var i = 0; i < 900; i++, t++) { - ts.add(t); - bpm.add(140); - } - final lowMotion = [ - for (var tt = 0; tt < t; tt++) MotionPoint(tt, 0.01), - ]; - final out = AutoWorkoutDetector.detect( - hrTs: ts, - hrBpm: bpm, - restingBpm: 60, - motion: lowMotion, - ); - expect(out, isEmpty); - }); - - test('motion gate + onset bypass: no pre-window (span starts at the ' - 'first sample) cannot evaluate onset — stays dropped', () { - // Elevated from t=0 with no preceding rest data at all. - final ts = [for (var t = 0; t < 900; t++) t]; - final bpm = [for (var t = 0; t < 900; t++) 140]; - final lowMotion = [for (final t in ts) MotionPoint(t, 0.01)]; - final out = AutoWorkoutDetector.detect( - hrTs: ts, - hrBpm: bpm, - restingBpm: 60, - motion: lowMotion, - ); - expect(out, isEmpty); - }); - - test('brief dip ≤ maxDipS does not break the span', () { - // 13 min elevated, with a single 60-s dip in the middle. - final ts = []; - final bpm = []; - for (var t = 0; t < 780; t++) { - ts.add(t); - // dip to 70 (below floor 100) for [400,460). - bpm.add((t >= 400 && t < 460) ? 70 : 140); - } - final out = AutoWorkoutDetector.detect(hrTs: ts, hrBpm: bpm, restingBpm: 60); - expect(out, hasLength(1)); - expect(out.first.startSec, 0); - expect(out.first.endSec, 779); - }); - - test('injected classifier types the bout', () { - final d = _hrDay(); - String classify(WorkoutBout b, MotionFeatures? f) => - b.avgBpm >= 130 ? 'run' : 'walk'; - final out = AutoWorkoutDetector.detect( - hrTs: d.ts, - hrBpm: d.bpm, - restingBpm: 60, - classify: classify, - ); - expect(out.single.sport, 'run'); - }); - - test('Metric wrapper is present with a list', () { - final d = _hrDay(); - final m = autoDetectWorkouts(hrTs: d.ts, hrBpm: d.bpm, restingBpm: 60); - expect(m.present, isTrue); - expect(m.value, hasLength(1)); - }); - }); - - group('WorkoutDetector (persistent per-day)', () { - // Build aligned HR + gravity with sustained motion over the work window. - ({ - List hrTs, - List hrBpm, - List gTs, - List gx, - List gy, - List gz - }) buildDay({int workS = 900, double workBpm = 150}) { - final hrTs = []; - final hrBpm = []; - final gTs = []; - final gx = []; - final gy = []; - final gz = []; - var t = 0; - // 10 min rest - for (var i = 0; i < 600; i++, t++) { - hrTs.add(t); - hrBpm.add(60); - gTs.add(t); - gx.add(0); - gy.add(0); - gz.add(1.0); // static gravity → ~0 intensity - } - // work: high HR + alternating gravity (large L2 delta each second) - for (var i = 0; i < workS; i++, t++) { - hrTs.add(t); - hrBpm.add(workBpm); - gTs.add(t); - gx.add(i.isEven ? 0.5 : -0.5); // |delta| = 1.0 each step >> 0.20 thresh - gy.add(0); - gz.add(0.8); - } - // 10 min rest - for (var i = 0; i < 600; i++, t++) { - hrTs.add(t); - hrBpm.add(60); - gTs.add(t); - gx.add(0); - gy.add(0); - gz.add(1.0); - } - return (hrTs: hrTs, hrBpm: hrBpm, gTs: gTs, gx: gx, gy: gy, gz: gz); - } - - test('detects a HR+motion gated bout with per-bout metrics', () { - final d = buildDay(); - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, - restingHR: 60, - profile: const WorkoutUserProfile( - weightKg: 75, heightCm: 178, age: 30, sex: 'male'), - ); - expect(out, hasLength(1)); - final s = out.first; - expect(s.avgHR, closeTo(150, 1)); - expect(s.peakHR, 150); - expect(s.durationS, greaterThan(WorkoutDetector.minExerciseMin * 60 - 11)); - // strain present + bounded 0..100. - expect(s.strain, isNotNull); - expect(s.strain!, inInclusiveRange(0, 100)); - // zone time-% sums to ~100. - final zsum = s.zoneTimePct.values.fold(0, (a, b) => a + b); - expect(zsum, closeTo(100, 1.0)); - // %HRR for 150 bpm w/ rhr60,max190 = (150-60)/130 = 69.2% → zone 3 (≥70?) - expect(s.avgHRRPct, closeTo(69.2, 0.5)); - // calories present + positive. - expect(s.caloriesKcal, isNotNull); - expect(s.caloriesKcal!, greaterThan(0)); - expect(s.hrmaxSource, 'caller'); - expect(s.sport, 'detected'); - }); - - test('below min duration / low intensity → none', () { - final d = buildDay(workS: 120); // 2 min < 5 min - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, - restingHR: 60, - ); - expect(out, isEmpty); - }); - - test('overlap-dedup drops a detected bout overlapping a saved span', () { - final d = buildDay(); - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, - restingHR: 60, - savedSpans: const [SavedWorkoutSpan(700, 900)], - ); - expect(out, isEmpty); - }); - - test('injected sport classifier types the detected bout', () { - final d = buildDay(); - String classify(WorkoutBout b, MotionFeatures? f) { - // feats should carry the 1 Hz amplitude index. - expect(f, isNotNull); - expect(f!.meanIntensity, greaterThan(0.2)); - return 'cardio'; - } - - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, - restingHR: 60, - classify: classify, - ); - expect(out.single.sport, 'cardio'); - }); - - test('Metric wrapper present + toJson round-trips', () { - final d = buildDay(); - final m = detectWorkouts( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, - restingHR: 60, - profile: const WorkoutUserProfile(sex: 'female'), - ); - expect(m.present, isTrue); - final j = m.value!.first.toJson(); - expect(j['sport'], 'detected'); - expect(j['calories_kcal'], isNotNull); - }); - }); - - test('a bridged dropout is billed at the published cap, through detect()', () { - // The reason detect() reads Calories.defaultMergeGapCapS rather than its own - // mergeGapS: bridgeGapS is TWICE mergeGapS, so _bridgeRuns stitches an - // HR-free dropout of up to 300 s into a single bout, and the cap really does - // bind inside one. Everything else about the cap is tested against - // estimateBoutCalories directly, which cannot observe that the detector - // passes the right constant. - const gapS = 252; // > the 150 s cap, < the 300 s bridge window - final hrTs = []; - final hrBpm = []; - for (var t = 0; t < 600; t++) { - hrTs.add(t); - hrBpm.add(150); - } - for (var t = 600 + gapS; t < 1200 + gapS; t++) { - hrTs.add(t); - hrBpm.add(150); - } - // Motion has to stay above the gate across the gap or the runs never merge. - final gTs = []; - final gx = [], gy = [], gz = []; - for (var t = 0; t < 1200 + gapS; t++) { - gTs.add(t); - gx.add(t.isEven ? 0.0 : 0.6); - gy.add(0); - gz.add(1); - } - - const profile = - WorkoutUserProfile(weightKg: 75, heightCm: 178, age: 30, sex: 'male'); - final out = WorkoutDetector.detect( - hrTs: hrTs, - hrBpm: hrBpm, - gravTs: gTs, - gx: gx, - gy: gy, - gz: gz, - maxHR: 190, - restingHR: 60, - profile: profile, - ); - - expect(out, hasLength(1), reason: 'the dropout must be bridged, not split'); - final session = out.first; - - // Score the same samples both ways. The detector must match the capped one. - double score(double cap) => Calories.estimateBoutCalories( - hrTs, - hrBpm, - profile: profile, - hrmax: 190, - restingHr: 60, - mergeGapCapS: cap, - ).kcal; - - final capped = score(Calories.defaultMergeGapCapS); - final uncapped = score(gapS.toDouble() + 1); - - // Not exact: detect() prices its own bout window, which the run boundaries - // trim by a sample or two against the raw stream scored here — worth a few - // tenths of a kcal. The capped and uncapped figures are ~24 kcal apart, so - // a 2 kcal tolerance still tells them apart by an order of magnitude. - expect(uncapped - capped, greaterThan(20.0), - reason: 'if these converge the assertions below prove nothing'); - expect((session.caloriesKcal! - capped).abs(), lessThan(2.0)); - expect((session.caloriesKcal! - uncapped).abs(), greaterThan(20.0)); - }); - - group('Calories (Keytel + Harris–Benedict)', () { - test('male/female coefficients differ; active > resting', () { - // 10 min @ 150 bpm, 1 Hz. - final ts = [for (var t = 0; t < 600; t++) t]; - final bpm = [for (var t = 0; t < 600; t++) 150.0]; - final m = Calories.estimateBoutCalories(ts, bpm, - profile: const WorkoutUserProfile( - weightKg: 80, heightCm: 180, age: 30, sex: 'male'), - hrmax: 190, - restingHr: 60); - final f = Calories.estimateBoutCalories(ts, bpm, - profile: const WorkoutUserProfile( - weightKg: 80, heightCm: 180, age: 30, sex: 'female'), - hrmax: 190, - restingHr: 60); - expect(m.kcal, greaterThan(0)); - expect(f.kcal, greaterThan(0)); - expect(m.kcal, isNot(closeTo(f.kcal, 0.01))); // sex coeffs differ - expect(m.kj, closeTo(m.kcal * 4.184, 1e-6)); - expect(m.usedDefaultAnchors, isFalse); // real profile + real anchors given - }); - - test('resting samples use the BMR floor, not active rate', () { - // 5 min @ 65 bpm → below active threshold (60 + 0.30*(190-60)=99) → resting. - final ts = [for (var t = 0; t < 300; t++) t]; - final bpm = [for (var t = 0; t < 300; t++) 65.0]; - final r = Calories.estimateBoutCalories(ts, bpm, - profile: const WorkoutUserProfile(sex: 'male'), - hrmax: 190, - restingHr: 60); - final kcal = r.kcal; - // ~5 min of resting BMR — small but positive. - expect(kcal, greaterThan(0)); - expect(kcal, lessThan(10)); // resting-only over 5 min is tiny (~5–6 kcal) - }); - }); - - // --------------------------------------------------------------------------- - // REGRESSION: unevaluable gates must BLOCK, hidden anchors must not exist, - // off-skin samples must not become the resting-HR baseline, and the - // fabricated-anchor calorie flag must reach the output. - // --------------------------------------------------------------------------- - group('WorkoutDetector — abstain-over-fabricate (regression)', () { - /// Build a day of [restS] still/low-HR seconds, then [workS] seconds of - /// sustained motion at [workBpm], then [restS] still seconds again. - /// [offSkinS] leading seconds report hr == 0 (the off-skin sentinel) with - /// static gravity. - ({ - List hrTs, - List hrBpm, - List gTs, - List gx, - List gy, - List gz - }) day({ - required int workS, - required double workBpm, - required double restBpm, - int restS = 60, - int offSkinS = 0, - }) { - final hrTs = []; - final hrBpm = []; - final gTs = []; - final gx = []; - final gy = []; - final gz = []; - var t = 0; - void still(int n, double bpm) { - for (var i = 0; i < n; i++, t++) { - hrTs.add(t); - hrBpm.add(bpm); - gTs.add(t); - gx.add(0); - gy.add(0); - gz.add(1.0); // static gravity -> ~0 motion intensity - } - } - - still(offSkinS, 0); // OFF-SKIN: hr == 0 (types.dart HrSample convention) - still(restS, restBpm); - for (var i = 0; i < workS; i++, t++) { - hrTs.add(t); - hrBpm.add(workBpm); - gTs.add(t); - gx.add(i.isEven ? 0.5 : -0.5); // |delta| = 1.0 >> motionThreshold - gy.add(0); - gz.add(0.8); - } - still(restS, restBpm); - return (hrTs: hrTs, hrBpm: hrBpm, gTs: gTs, gx: gx, gy: gy, gz: gz); - } - - test('no HRmax anchor => the zone-2 gate is unevaluable => NO workout', () { - // age null + maxHR null + <600 HR samples => estimateHRmax returns - // (0.0, "unknown") => effMaxHR null => zonePct empty. PRE-FIX the whole - // ">=50% time in zone 2+" gate was SKIPPED, so this 6.7-minute walk at - // RHR+16 bpm was emitted as a durable workout. - final d = day(workS: 400, workBpm: 76, restBpm: 60, restS: 60); - expect(d.hrTs.length, lessThan(600), - reason: 'must stay below estimateHRmax observed-sample minimum'); - - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - restingHR: 60, - maxHR: null, - age: null, - ); - expect(out, isEmpty); - }); - - test('detectWorkouts SAYS the gate could not be evaluated', () { - final d = day(workS: 400, workBpm: 76, restBpm: 60, restS: 60); - final m = detectWorkouts( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - restingHR: 60, - ); - expect(m.value, isEmpty); - expect(m.note, contains('no HRmax anchor')); - // inputs_used must reflect what was actually supplied. - expect(m.inputs_used, contains('resting_hr')); - expect(m.inputs_used, isNot(contains('max_hr'))); - expect(m.inputs_used, isNot(contains('age'))); - expect(m.inputs_used, isNot(contains('profile'))); - }); - - test('an emitted session NEVER reports strain without an HRmax anchor', () { - // PRE-FIX StrainScorer.strain(maxHR: null) silently used - // defaultMaxHR() = 220 - 30 = 190, so a session shipped a concrete strain - // alongside `hrmax: null, hrmax_source: "unknown"`. - final scenarios = >[ - for (final anchor in [null, 190]) - WorkoutDetector.detect( - hrTs: day(workS: 400, workBpm: 160, restBpm: 60).hrTs, - hrBpm: day(workS: 400, workBpm: 160, restBpm: 60).hrBpm, - gravTs: day(workS: 400, workBpm: 160, restBpm: 60).gTs, - gx: day(workS: 400, workBpm: 160, restBpm: 60).gx, - gy: day(workS: 400, workBpm: 160, restBpm: 60).gy, - gz: day(workS: 400, workBpm: 160, restBpm: 60).gz, - restingHR: 60, - maxHR: anchor, - ), - ]; - // With an anchor a real bout is emitted; without one, nothing is. - expect(scenarios[1], isNotEmpty); - expect(scenarios[0], isEmpty); - for (final list in scenarios) { - for (final s in list) { - if (s.hrmax == null) { - expect(s.strain, isNull, - reason: 'strain must abstain without a real HRmax'); - } - } - } - }); - - test('off-skin (hr==0) samples are excluded from the resting-HR percentile', - () { - // 200 s off-skin (hr == 0) then a 400 s walk at 120 bpm on a 55 bpm day. - // PRE-FIX the 10th percentile of the RAW stream was 0, so restHR = 0, - // hrFloor = 15 and %HRR for the walk was (120-0)/190 = 63% => zone 2, so - // ordinary walking cleared the >=50%-in-zone-2+ gate. On-skin only, the - // 10th percentile is 55 and %HRR is (120-55)/135 = 48% => zone 0. - final d = day( - workS: 400, workBpm: 120, restBpm: 55, restS: 400, offSkinS: 200); - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, // isolate the resting-HR derivation - ); - expect(out, isEmpty, - reason: 'a 120 bpm walk is zone 0 against a real 55 bpm resting HR'); - }); - - test('an all-off-skin day derives no resting HR and abstains', () { - final d = day(workS: 400, workBpm: 0, restBpm: 0, restS: 60); - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - maxHR: 190, - ); - expect(out, isEmpty); - }); - - test('the fabricated-anchor calorie flag reaches the session JSON', () { - // Calories.estimateBoutCalories returns usedDefaultAnchors precisely so a - // number built on the flat hrmax 220 / restingHr 60 fallback can be - // caveated. PRE-FIX it was computed and dropped, and ExerciseSession had - // no field or JSON key for it at all. - final flagged = Calories.estimateBoutCalories( - [0, 60, 120], - [140, 145, 150], - profile: const WorkoutUserProfile(), - hrmax: null, - restingHr: null, - ); - expect(flagged.usedDefaultAnchors, isTrue); - - final d = day(workS: 400, workBpm: 160, restBpm: 60); - final out = WorkoutDetector.detect( - hrTs: d.hrTs, - hrBpm: d.hrBpm, - gravTs: d.gTs, - gx: d.gx, - gy: d.gy, - gz: d.gz, - restingHR: 60, - maxHR: 190, - profile: const WorkoutUserProfile( - weightKg: 75, heightCm: 178, age: 30, sex: 'male'), - ); - expect(out, isNotEmpty); - final j = out.first.toJson(); - expect(j.containsKey('calories_used_default_anchors'), isTrue); - expect(j['calories_used_default_anchors'], isFalse, - reason: 'both anchors were real here'); - - // And the flag is genuinely carried, not hardcoded false. - const caveated = ExerciseSession( - start: 0, - end: 400, - avgHR: 150, - peakHR: 160, - strain: null, - durationS: 400, - zoneTimePct: {}, - avgHRRPct: null, - hrmax: null, - hrmaxSource: 'unknown', - caloriesKcal: 123.0, - caloriesKJ: 514.6, - caloriesUsedDefaultAnchors: true, - ); - expect(caveated.toJson()['calories_used_default_anchors'], isTrue); - }); - }); -} diff --git a/tool/oxwalk_validate.dart b/tool/oxwalk_validate.dart new file mode 100644 index 0000000..1c244f7 --- /dev/null +++ b/tool/oxwalk_validate.dart @@ -0,0 +1,352 @@ +// VALIDATION HARNESS — score the SHIPPED pedometer against OxWalk, the first +// real annotated free-living wrist corpus this algorithm has ever met. +// +// OxWalk (Small, von Fritsch, Doherty, Khalid, Price; University of Oxford, +// Dec 2022, CC BY): 39 healthy adults, unscripted free living, Axivity AX3 on +// the DOMINANT WRIST and at the hip, 100 Hz and 25 Hz, ~1 h each. Ground truth +// is a synchronised foot-facing beltline camera annotated in ELAN — one `1` per +// heel strike, so the annotation column summed over a file IS that +// participant's true step count. +// +// It runs `pedometer()` — the real production entry point, not a copy — and it +// CHUNKS THE SIGNAL THE WAY PRODUCTION DOES. That is not a detail: `confirm=8` +// needs a contiguous run inside one buffer, so the fragment size dominates the +// answer. `AppState._ingestLiveMagsAt` accumulates `_magMin` and calls +// `pedometer` on each completed `_minuteSamples = 6000` (60 s @ 100 Hz) block, +// counting the trailing partial minute once at session end. This harness does +// exactly that (60 s of whatever the file's rate is), and prints the +// one-contiguous-buffer number as a separate line so the chunking cost is +// visible rather than hidden. +// +// The dataset is 290 MB and is NOT committed. Point the tool at an unpacked +// copy; the settled numbers are pinned as constants in +// test/onehz/steps_test.dart so a change that moves them fails loudly without +// anyone having to download anything. +// +// Usage: +// dart run tool/oxwalk_validate.dart [flags] +// OXWALK_DIR=... dart run tool/oxwalk_validate.dart +// +// --set=NAME restrict to one dataset (repeatable). +// Default: Wrist_100Hz Wrist_25Hz Hip_100Hz Hip_25Hz +// --chunk=N override the chunk size in SAMPLES (default fs*60, which is +// production). Sweep it to see how much the per-minute reset +// is worth — on real data it is worth a great deal, in the +// opposite direction from the synthetic estimate. +// --decimate=N keep every Nth sample and score at fs/N, to find the rate +// cliff on real data. The live `0x33` stream's true frame rate +// is documented nowhere in the tree (STEPS_ALGO §5), so where +// this counter dies as a function of rate is a shipping fact, +// not a curiosity. +// --baseline also score a simple published-style wrist counter (VM peak +// detection + interval + periodicity gates) on the same +// signal, as a "is ours worth keeping" reference. +// +// The interval-guard ablation (StepParams.minStepIntervalS / +// maxStepIntervalS) is not a flag: those are compile-time constants on the +// shipped class, and adding an override to production code for a one-off +// ablation is worse than editing them locally for the run. Set them to +// 0.0 / 1e9, re-run, revert. + +import 'dart:io'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; + +const _sets = { + 'Wrist_100Hz': 100, + 'Wrist_25Hz': 25, + 'Hip_100Hz': 100, + 'Hip_25Hz': 25, +}; + +class Row { + final String pid; + final String age; + final int truth; + final int chunked; // production per-minute chunking, RAW (pre-gain) + final int oneShot; // one contiguous buffer, RAW + Row(this.pid, this.age, this.truth, this.chunked, this.oneShot); + double get pctErr => truth == 0 ? double.nan : (chunked - truth) / truth * 100; + double get ratio => truth == 0 ? double.nan : chunked / truth; +} + +void main(List args) { + final root = args.firstWhere((a) => !a.startsWith('--'), + orElse: () => Platform.environment['OXWALK_DIR'] ?? ''); + if (root.isEmpty || !Directory(root).existsSync()) { + stderr.writeln(''' +OxWalk data not found. + + dart run tool/oxwalk_validate.dart + +Expects the unpacked OxWalk_Dec2022 directory (Wrist_100Hz/, Wrist_25Hz/, +Hip_100Hz/, Hip_25Hz/, metadata.csv). ~290 MB, deliberately NOT committed. +Download: Oxford Research Archive, OxWalk Annotated Step Count Dataset (CC BY). +'''); + exit(2); + } + final want = args + .where((a) => a.startsWith('--set=')) + .map((a) => a.substring(6)) + .toList(); + final chunkArg = args.firstWhere((a) => a.startsWith('--chunk='), + orElse: () => ''); + final chunkOverride = + chunkArg.isEmpty ? null : int.parse(chunkArg.substring(8)); + final baseline = args.contains('--baseline'); + final decArg = + args.firstWhere((a) => a.startsWith('--decimate='), orElse: () => ''); + final decimate = decArg.isEmpty ? 1 : int.parse(decArg.substring(11)); + + final ages = _readMetadata(File('$root/metadata.csv')); + + for (final entry in _sets.entries) { + if (want.isNotEmpty && !want.contains(entry.key)) continue; + final dir = Directory('$root/${entry.key}'); + if (!dir.existsSync()) { + stderr.writeln('skip ${entry.key}: not present'); + continue; + } + _runSet(entry.key, entry.value, dir, ages, chunkOverride, baseline, + decimate); + } +} + +Map _readMetadata(File f) { + final out = {}; + if (!f.existsSync()) return out; + for (final line in f.readAsLinesSync().skip(1)) { + final p = line.split(','); + if (p.length >= 3) out[p[0].trim()] = p[2].trim(); + } + return out; +} + +void _runSet(String name, double fsIn, Directory dir, Map ages, + int? chunkOverride, bool baseline, int decimate) { + final fs = fsIn / decimate; + final chunk = chunkOverride ?? (fs * 60).round(); // production: 1 min + final files = dir + .listSync() + .whereType() + .where((f) => f.path.endsWith('.csv')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + final rows = []; + final base = []; + for (final f in files) { + final pid = f.uri.pathSegments.last.split('_').first; + final (full, truth) = _load(f); + final mag = decimate == 1 + ? full + : [for (var i = 0; i < full.length; i += decimate) full[i]]; + if (mag.length < 2) continue; + var raw = 0; + for (var i = 0; i < mag.length; i += chunk) { + final end = math.min(i + chunk, mag.length); + raw += pedometer(mag.sublist(i, end), sampleRateHz: fs); + } + rows.add(Row(pid, ages[pid] ?? '?', truth, raw, + pedometer(mag, sampleRateHz: fs))); + if (baseline) { + final b = _baselineSteps(mag, fs); + base.add(Row(pid, ages[pid] ?? '?', truth, b, b)); + } + } + _report('$name (chunk=$chunk${decimate > 1 ? ', /$decimate' : ''})', fs, rows); + if (baseline) _report('$name — BASELINE (VM peak detector)', fs, base); +} + +/// A simple published-STYLE wrist step counter, for reference only. +/// +/// Not the Verisense code and not a validated port of it — it is the shape the +/// literature's simple wrist counters share: smooth the vector magnitude, take +/// local maxima that clear an amplitude threshold, keep only those whose +/// inter-peak interval is physiological, and require a short run of +/// consistently-spaced peaks (the periodicity/continuity gate) before crediting +/// anything. It exists to answer one question — is the shipped counter better +/// or worse than the cheapest reasonable alternative on the same signal — and +/// it should not be quoted as "Verisense scored X". +int _baselineSteps(List mag, double fs) { + final n = mag.length; + if (n < 4) return 0; + // ~0.1 s moving average → attenuates above ~5 Hz, keeps gait's 1.4-2.3 Hz. + final w = math.max(1, (fs * 0.1).round()); + final lp = List.filled(n, 0); + var acc = 0.0; + for (var i = 0; i < n; i++) { + acc += mag[i]; + if (i >= w) acc -= mag[i - w]; + lp[i] = acc / math.min(i + 1, w); + } + // Local maxima over ±0.2 s, at least 0.05 g above the surrounding trough. + final half = math.max(1, (fs * 0.2).round()); + final peaks = []; + for (var i = half; i < n - half; i++) { + final v = lp[i]; + var isMax = true; + var mn = v; + for (var j = i - half; j <= i + half; j++) { + if (lp[j] > v) { + isMax = false; + break; + } + if (lp[j] < mn) mn = lp[j]; + } + if (isMax && v - mn >= 0.05) peaks.add(i); + } + // Interval + periodicity: credit a peak only inside a run of >= 3 intervals + // that are all physiological (0.25-2.0 s) and mutually consistent (each + // within 40% of the previous), which is what separates gait from handling. + var steps = 0, run = 0; + double? prevGap; + for (var k = 1; k < peaks.length; k++) { + final gap = (peaks[k] - peaks[k - 1]) / fs; + final ok = gap >= 0.25 && + gap <= 2.0 && + (prevGap == null || (gap - prevGap).abs() / prevGap <= 0.40); + if (ok) { + run++; + if (run == 3) { + steps += 4; // credit the run's peaks retroactively + } else if (run > 3) { + steps++; + } + } else { + run = 0; + } + prevGap = gap >= 0.25 && gap <= 2.0 ? gap : null; + } + return steps; +} + +/// Fast CSV read: `timestamp,x,y,z,annotation`. Returns |a| per sample (g, +/// gravity included — exactly what `pedometer` expects) and the true step count. +(List, int) _load(File f) { + final mag = []; + var truth = 0; + var first = true; + for (final line in f.readAsLinesSync()) { + if (first) { + first = false; + continue; + } + if (line.isEmpty) continue; + // Split from the right: 4 commas back from the end are x,y,z,annotation. + final c4 = line.lastIndexOf(','); + final c3 = line.lastIndexOf(',', c4 - 1); + final c2 = line.lastIndexOf(',', c3 - 1); + final c1 = line.lastIndexOf(',', c2 - 1); + if (c1 < 0) continue; + final x = double.parse(line.substring(c1 + 1, c2)); + final y = double.parse(line.substring(c2 + 1, c3)); + final z = double.parse(line.substring(c3 + 1, c4)); + mag.add(math.sqrt(x * x + y * y + z * z)); + if (line.codeUnitAt(c4 + 1) != 0x30) truth++; // annotation != '0' + } + return (mag, truth); +} + +void _report(String name, double fs, List rows) { + final b = StringBuffer(); + b.writeln(''); + b.writeln('═══ $name — n=${rows.length}, fs=${fs.toStringAsFixed(0)} Hz, ' + 'chunk=${(fs * 60).round()} samples (production) ═══'); + b.writeln('pid age truth chunked err% ratio oneshot 1shot%'); + for (final r in rows) { + b.writeln('${r.pid.padRight(6)} ${r.age.padRight(7)} ' + '${r.truth.toString().padLeft(5)} ' + '${r.chunked.toString().padLeft(7)} ' + '${r.pctErr.toStringAsFixed(1).padLeft(7)} ' + '${r.ratio.toStringAsFixed(3).padLeft(6)} ' + '${r.oneShot.toString().padLeft(8)} ' + '${(r.truth == 0 ? double.nan : (r.oneShot - r.truth) / r.truth * 100).toStringAsFixed(1).padLeft(7)}'); + } + + final errs = rows.map((r) => r.pctErr).where((e) => e.isFinite).toList(); + final diffs = + rows.map((r) => (r.chunked - r.truth).toDouble()).toList(growable: false); + final ratios = rows.map((r) => r.ratio).where((r) => r.isFinite).toList() + ..sort(); + final totTruth = rows.fold(0, (a, r) => a + r.truth); + final totEst = rows.fold(0, (a, r) => a + r.chunked); + final totOne = rows.fold(0, (a, r) => a + r.oneShot); + + final mape = _mean(errs.map((e) => e.abs()).toList()); + final bias = _mean(errs); + final sdDiff = _sd(diffs); + final mDiff = _mean(diffs); + + b.writeln(''); + b.writeln('MAPE ${mape.toStringAsFixed(1)}%'); + b.writeln('bias (mean signed err) ${bias.toStringAsFixed(1)}%'); + b.writeln('median err% ${_median(errs.toList()..sort()).toStringAsFixed(1)}%'); + b.writeln('err% range ${errs.reduce(math.min).toStringAsFixed(1)} … ' + '${errs.reduce(math.max).toStringAsFixed(1)}'); + b.writeln('totals truth=$totTruth chunked=$totEst ' + '(${((totEst - totTruth) / totTruth * 100).toStringAsFixed(1)}%) ' + 'oneshot=$totOne ' + '(${((totOne - totTruth) / totTruth * 100).toStringAsFixed(1)}%)'); + b.writeln('chunking cost ${((totEst - totOne) / totOne * 100).toStringAsFixed(1)}% ' + 'vs one contiguous buffer'); + b.writeln('Bland-Altman mean diff ${mDiff.toStringAsFixed(0)} steps, ' + 'SD ${sdDiff.toStringAsFixed(0)}, LoA ' + '[${(mDiff - 1.96 * sdDiff).toStringAsFixed(0)}, ' + '${(mDiff + 1.96 * sdDiff).toStringAsFixed(0)}]'); + b.writeln('gain fits total-ratio ${(totTruth / totEst).toStringAsFixed(3)} ' + 'median-of-ratios ${(1 / _median(ratios)).toStringAsFixed(3)} ' + 'ratio IQR [${_q(ratios, .25).toStringAsFixed(2)}, ' + '${_q(ratios, .75).toStringAsFixed(2)}] ' + 'ratio range [${ratios.first.toStringAsFixed(2)}, ' + '${ratios.last.toStringAsFixed(2)}]'); + // Does any single multiplier help? Sweep it against the same MAPE the + // headline uses. If the error is not multiplicative this is flat and useless. + var bestG = 1.0, bestM = double.infinity; + for (var g = 0.20; g <= 3.001; g += 0.01) { + final m = _mean(rows + .where((r) => r.truth > 0) + .map((r) => ((r.chunked * g - r.truth) / r.truth * 100).abs()) + .toList()); + if (m < bestM) { + bestM = m; + bestG = g; + } + } + b.writeln('best gain ${bestG.toStringAsFixed(2)} → MAPE ' + '${bestM.toStringAsFixed(1)}% (gain 1.00 → ${mape.toStringAsFixed(1)}%)'); + + // Age bands — slow walking is the documented wrist failure mode. + final byAge = >{}; + for (final r in rows) { + byAge.putIfAbsent(r.age, () => []).add(r); + } + final bands = byAge.keys.toList()..sort(); + for (final a in bands) { + final e = byAge[a]!.map((r) => r.pctErr).where((v) => v.isFinite).toList(); + b.writeln('age $a n=${e.length} MAPE ' + '${_mean(e.map((v) => v.abs()).toList()).toStringAsFixed(1)}% ' + 'bias ${_mean(e).toStringAsFixed(1)}% ' + 'truth/h ${(_mean(byAge[a]!.map((r) => r.truth.toDouble()).toList())).toStringAsFixed(0)}'); + } + stdout.write(b.toString()); +} + +double _mean(List xs) => + xs.isEmpty ? double.nan : xs.reduce((a, b) => a + b) / xs.length; +double _sd(List xs) { + if (xs.length < 2) return double.nan; + final m = _mean(xs); + return math.sqrt( + xs.map((v) => (v - m) * (v - m)).reduce((a, b) => a + b) / (xs.length - 1)); +} + +double _median(List sorted) => _q(sorted, 0.5); + +double _q(List sorted, double p) { + if (sorted.isEmpty) return double.nan; + final i = p * (sorted.length - 1); + final lo = i.floor(), hi = i.ceil(); + return sorted[lo] + (sorted[hi] - sorted[lo]) * (i - lo); +}