From 22429e705f479a34a2cb9cce6d03838b01d3fb11 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 06:53:58 -0500 Subject: [PATCH 1/3] Pre-register the Fara-7B replay: 57 first-half rows on stack 9820, checked before the token fix --- .../PREREGISTRATION-FARA.md | 55 +++ .../leaderboard_rescore/SHA256SUMS-FARA | 18 + .../inputs/frozen-9820.jsonl.gz | Bin 0 -> 38786 bytes .../inputs/rows-fara.jsonl.gz | Bin 0 -> 13781 bytes .../leaderboard_rescore/inputs/times.json | 404 +++++++++++++++++- .../leaderboard_rescore/rescore.py | 10 +- .../leaderboard_rescore/score_rescore.py | 4 +- packages/flow/public-artifacts.json | 2 + 8 files changed, 489 insertions(+), 4 deletions(-) create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/PREREGISTRATION-FARA.md create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/SHA256SUMS-FARA create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9820.jsonl.gz create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-fara.jsonl.gz diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/PREREGISTRATION-FARA.md b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/PREREGISTRATION-FARA.md new file mode 100644 index 0000000000..fa5472f9d0 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/PREREGISTRATION-FARA.md @@ -0,0 +1,55 @@ +# Pre-registration addendum: Fara-7B's first half + +Written and committed before any re-score of Fara's rows. It extends +`PREREGISTRATION.md` to a third agent. Everything there applies unchanged, +including deviations 1, 2 and 4 (the horizon, the retried seed and the review +fixes), except what this file states. The commit that adds this file fixes +the code, the contracts and the inputs below. + +## Population + +Fara-7B's 57 scored rows from its first half (cases P01 to A06, 3 runs each, +less the signed-out runs), on stack 9820, as committed on PR #1737 at +01854c344 and extracted to `inputs/rows-fara.jsonl.gz` (each row keeps its +source path and sha256). The server state is +`inputs/frozen-9820.jsonl.gz`, the bench project's resource history and +audit events from that stack. The run folders' times are merged into +`inputs/times.json` under `fara/`. + +Fara's live checks got a fresh token at 2026-11-04 11:55 UTC. `rescore.py` +skips any Fara row checked at or after that time (`CHECKED_BEFORE`), since +its live verdict stands. Every row in this population was checked before it. + +## Columns + +The same three as the main analysis: **main**, **operations** (v1, +`between: consistent`) and **operations-v2** (`between: either`). Here +**operations-v2** is registered, not exploratory: its rule was fixed before +any Fara verdict was computed. The primary comparison stays **operations**. + +## Fidelity gate + +Four Fara rows (r1-P01 to r1-P04) were judged live (no `HTTP 401`). The +**main** column must equal them as defined in `PREREGISTRATION.md`. + +## What was seen before registering + +- The oracle labels of all 57 rows. +- Which rows the live verifier judged, and the live verdict of r1-P01 + (`needs_review`, no record and no agent write), seen while checking the + row format. +- No re-scored verdict of any Fara row, under any contract. + +## Commands + +```bash +for c in main operations operations-v2; do + python rescore.py --agent fara --contracts $c --out results/rescore-$c-fara.json +done +python score_rescore.py --agent fara results/rescore-{main,operations,operations-v2}-fara.json \ + --out results/score-fara.json +``` + +The claude and openai results in `results/` are not re-run. Their files' +`inputs` field lists the inputs as they were then; `RESULTS-SHA256SUMS` +still pins what produced them. diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/SHA256SUMS-FARA b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/SHA256SUMS-FARA new file mode 100644 index 0000000000..c7ea9b703d --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/SHA256SUMS-FARA @@ -0,0 +1,18 @@ +d009dbce1ea5e1802fd2ff1c4ab5e93ce9aa4cfec489f17af463220d13bdd1b5 packages/flow/openadapt_flow/ensure_record.py +6b6accdd943039a6f9d46d05a23f37b596ba7dd0557414922d478385989dcc34 packages/flow/benchmark/fhir_medplum/app_semantics/medplum-v2.yaml +2468ee74ab10c53deb6f4a0ddd3f491b7cea1c2a795407f88da6edf7b9a37fe8 packages/flow/benchmark/fhir_medplum/app_semantics/medplum.yaml +66718fbd2e421adb465e442d789a267990babc0df77f7003d313b8738426a343 packages/flow/benchmark/fhir_medplum/public/contracts/allergy.yaml +c0041e371828ffa6a5ded7aef1176b3a06977ababd66b17002099aa4e5c9448b packages/flow/benchmark/fhir_medplum/public/contracts/appointment.yaml +8c293f75e94a0ee032722a7285af9cf07c6bebefaf040126d0a6d26a9d1695df packages/flow/benchmark/fhir_medplum/public/contracts/patient.yaml +213dbaac11196b380b3833229fdfdb5cf55878ad63f173bbdfbf42705042aeee packages/flow/benchmark/fhir_medplum/leaderboard_rescore/contracts_operations/allergy.yaml +9494eec870f446f316b07d6389f73e7d2321179bd0a8ed5aa30cee6d6973bdae packages/flow/benchmark/fhir_medplum/leaderboard_rescore/contracts_operations/appointment.yaml +775d8328df5d4b9f99ee69d9522ed05a5f376f1f50b28ce944ed0dd5b3ec719a packages/flow/benchmark/fhir_medplum/leaderboard_rescore/contracts_operations/patient.yaml +bcd0a8287c27c1f33e0963da654bfd8358f45f9fa37a70b276e8311ff5a0c9b4 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/contracts_operations_v2/allergy.yaml +a2dc1b912b6285eda2300374aa81c73d8079ab5795586631e3f5e42f93b67c78 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/contracts_operations_v2/appointment.yaml +12c89175598ac4d3f0ea65674ef5908f271c6eaf3425e1178fbca56d958fbeaa packages/flow/benchmark/fhir_medplum/leaderboard_rescore/contracts_operations_v2/patient.yaml +c1be1c05f95e01427205c559e2a6f48f7ef926c9076deed0de6143193a23c74f packages/flow/benchmark/fhir_medplum/leaderboard_rescore/frozen.py +c2c5af9b81f516a394da9275dde9da3ce55b7b58787ebb60e50e333f756b92c4 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/rescore.py +3341946b9a18fe24c53ebd17a5e94d424e38760c191b93167df0d66993e02d85 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/score_rescore.py +6bbd6fe5f848a6508a4916e1804ec29565a2a18a286dcd30abaefe6a2d15a585 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9820.jsonl.gz +3cbcb0b41730cf66f50d9674ee17984c6283dec1799ca672973cd7e94969e6b9 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-fara.jsonl.gz +1ee03f4bb4c2d27199072e9f4b371ba46667f37ac55938f525761f7761b2080e packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/times.json diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9820.jsonl.gz b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9820.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..20f2fb2c80d83bca1bce59aa21591b36460749d3 GIT binary patch literal 38786 zcmZ5{19v4%ux)HloJ=s8*tTukwylY6+qRvPWMbQ%*!BrtzI)eO>#g?-s=90Ms$Esx zgi$asqj=|5U|(iV_FkrTw45AF40Kk`_I5TP7XaP3VsSe&}q^`y^!kk1}* z5TW0jkjl;sGGPx#&zs-p!;r_vZQRoa*4O(baK*k?tL_-@W6!VqZExzlZDHpIa<=O0 z_3h*3ZHYe|aMybwztaP}QQUst;(lcJK?VKwMt(6ASEl~wIg9>tV`oLb_v_`+{#rj+ z%{m-i>F)Ut>&}_abzR+SOn1iTy7i-9+2~JaIpQBeM8q(%Ap_$u5K0`>FI~MEghSh3 zW%LJKar(X9es8M!elJfk*=F}$6+1nzS6S-4bM|_@kI%h%);)RlBbE%ke)RkM=W%sA zfSl{z$&$f#FNBU|Oo^gkHYwx|YbyieG2Gr4dtl`4RhIqX$B6yWMx49!bJDfX^Vihv z?ZjGsfDA0@KbT-sC`rn&Kwi~RS2AsOU}xSR9loAc`cJ2YclimR7@yC3n*Gx9^Phhg z#300bgue)2V^@(lSH3pLzb>~{-)*x(%?V$hUq6FtW_I#=vN+m!6u=QL13INn9AiP6 zj)pmKpWUZ|VwWR1m zfKnJ*+HCaZXA%h@x1ht2&;{r*-EKj>U(~+AeVOX?^~S_>>f^rIkRs&s zd8($O*5P2WP5FV-We*mVv`29mNX{~14200IQT&Le`XOB97+&)2J+-^o91jc}7CIB)*2|OsqZ7m<_ zC~s-juZ8ndga7O13h(tspF4LGXZ&MqYcCFI7e2MrIT_fTmy9&5;I=lihWSh(*55cS zAgX4;VI{HdGKnw#^D+YZhGeii;`cOVNM_h2+!0zO93)Gzd5YmEcdaIXeabJ%tBd2n zcTf-$9Y-T5)^rc#hYY}*>12czZbXThZv^!HH|(#ZjZD#K+RQ;ER`0?QV&dafC+RA~ z%oQ5JtfOAdK&p#hf{du+OtG`x#z{onr`FL=!tlvl;x$End<$2W!|Ra33mN8+1SI(5 zav}_tU~+C|bog`OM2!v3#R||f60A##3q|HbLseJGYVCV`<4x&BA6pzLxF@*fS)3Qq zVzY9_DbwJulx2=g93R11)4_MuTa#4oe}5Z0v(}Wm7*Rth2V^fyIdO zWNemzU>AaFlbKk_OqF0-V?XVsGmmVnt6co7B4W{A;jAu%c`#hjdPdIiFh;DM0orjr zjBOJxo?$jmAWDf%;3VQ6tMTGDctRBq!yTeuQQSLID<(qoe+p%_tma>cn~oOS%WGb7+|qC}zc*s}2-m4vGO%B=JB`;PnhC)ia}ueF2d^F{+TV~mM(vlH?d z3{>DzyV+;=8_=4hxQ2o72Lar-?^;JRo`nGZ4qatqhBv4H@o^Khg3gOeRH5a(kF4fn^Y(XGuit6w?oNI0Hgl&7e`&#ow3_g6+k<24%_xoX>M*VdXX zKjT2Pc!l@^z3bf3Lbsg~P4*Rg>G3CG=@eisamHm&OSv2p+EB4*WREG_d7ao|JuMn8 zoARuV>>>v+ZPtmLYXyrVKhoWt5U(`$?>EKr6W;dt_{PkFlrfBXNpOR~pQl-Z73=;3 z_?P2j4^~)X90&SxraBeqVvt!MI-n|Ho|Pg{0M}zioPLk3a0B6QD_Us;?QLUGFJ+jK zv92x|(SquZm`%nP8*HGkj3wc46fM{j#`?I~Ty?I{53PcA&jiGvDyg2$jc|QsB>n2c zmMHvIW0|XujdQhXEagckYHV#ocM6;geRBT%)yOJf z+50X3ZN5GXDy}`NYG-dHIn@>&HT7`d`RD}MmT`-*L*g} zL;!;f4q-`KbtTTZm)3=r0HNv{BMA$BiY!6(3h7x9#|D;-j87h^b?9Z&Q5>%`q`skD zLGzXmVq<>|t)nWndD(kE9Z6mM=%jtVvO+cE12H(E$kr&97#bCWq*xYCcGk+}(oPw)aUtf^mNDAN&1#d@PE5&i9>*uO zii3X^SSce=53Ae&Hkn_>Qf~kbkP8x|ic-a}3`*xxQ*liKG9+dwosuJyJBF2eiqj+q zKEd%dWwCaTO>_&Sez<}(i@Ie>n2*Y&_tr_jCYa0QNfO>vVzMc>?v<|S+|+tVlk`Q# z4bJ#n)MI*Vwdn!RO7SD%aiC5H$O%o$lLEOKGC^g5EAThmlWCd0%BmN%&tI+QVhlQ< z$XyGhsZ0%V>-ey}7`&_tWCOis6MGW32SMkA@55#@q;9R_320|&KmpFaEJ0B?U(RlqM_~ z!jAGUI~`|@nk{{`$1G@#(+)EJIi^m|P6mvRRtS-DNMAA&LZq4TNYrDf&JMcwaPd>- zP0sOoOD@%yd{)4Y{>6MWYbjVRQZZSbdAF>Q2fy4|R_(Ag)|Iu~M~4WWY*Wf53Zp~` zRc@>X)M(m&@RXbHx1MH;!4&m#0(;`1gPkbNO4J;IQ~!C{^v&nLyU2SWK!}w?h&@h| zQq1@m`&N@Z1rCe(O-984##~gW|01nCFCrc+eB}Egi*;$HV;GOU1&7kqzRp4$!O6EZy}WmzrAh4w2jaP6b7^mz>RP7Nf>GMxY4ez;+7ofzXYPAC&@kv%vn0jfV+q-y zfS<~Z#B^@+ifZM1&7~w>7Ug4q09IFBrk`;Y8_)d;9%Wn-QpIEE=BjaJf^jlK6`+i_ zG*f)bN1~;ry7eqMWt$gEl2d`(;vFkANv28~b8ZPLUgfLA(V0@$($ptDxr@!2G3bIa z!|FsGSUixHt*qFbt{DHo)Gv?LS-wV=fO7axa77w@U`cYE+hj=2%R~$;lZGSI4LSZ; z$(d?*XTmW*);yK2pPHU6Ht-tKRkt_rS9~kJ{mUXPzXGRNL(m6D=H7W#v28D&&5?Ty z^(|&q*F2wxKh$Ml$#GU=kr$S5AnMN5ObfFetXo*5T9Y}m_7oppvQ3kxFnLvCRJ8BV zTdQhQTfb=%V9ZU%Q`(vpW>K0db1^o!y#P~DJL@54tnK_yXoz5f*(>}BH9}wiRLKol zXqz%IW0`=wjy^?Tx+~ukm^-+A`{=Hc{eDSN)LqQ^q#_!bmLPQ)H7Yrus3VN6(*BlT~jkIdHj#Laq@&M9OI23mP+O~!PNW`^`++SJxb^w+r*M&~;@hTT8;jfOQU zgDg@tJ!O^xCg`O~4v92=D?m{xuYR_+Ut>PnNxBk92aXk ze?U3qjmcBy%pWS3*VsH_*tMyNO%Z(vid7NeOOf@V530=o_3spua?ER*IvqYWVoFNN zFZ?$B1$#fh{cMk%gc6B(p!Tc=Rtdg_YCSzcJ8pO{w~E3NS|x{uSl;Q%%DDVVih|lQ zl7(fYimv3Ff_|!!gpmVskNzkXkEZsvV|Qhgmp~_TlM9l}`CbPpF+Zr?R5`PHYA3(@4bu8Mg z`Yg4`f2&SL8h?^QlH;JYXrKWo)?s!LYU|NQ3%5sf0)!YV0l8=GYArR%Q{d1I&r5-_ zmqeg1lTiuoopf-#*zF0%ozX8^$a+PqR6V@q^yYKZmmT4)mEXfttzg3tKnOyS{3}lx#q}YLh&R|`8{Qc#a8{243Md(o` zE2uMI;d95$<``+Ptf~Z4vM8AdW4)xm9h1e8dGx&Z*SQs)rwi2NDuC5GtwNYX*Y*82 z^_zZir!YRnP7HHf$7n>%zmw5%6aO-&U=)iYRSfsfPtNOUb8lTW_x83rt-*Q6KZ8OCQpzI^{JD`+R+5^$Plj*8>09uvOZ+ zh`vdH+Qc)u?yGmU91iR5I0!oLXxdP*sOh~a>4P~?uhR0bDM&{u!c|e!Tlg4~&Saym__iHITi%u>^Ypz2A($ycUgEzv+JsGvvlWM{*CH zKQWro8>uXwFG$+8YM3dok~}l1ix6}<`sY;p>u*d`i3gLk2yap1{y+p>jjo)1MR7PG|^gMp&l!ch^|LysFMXKVQ<>Qok%Ga%-gKQPzd3c&Ifc*JM0V7p;Wg z+pZq(l>vWgL+#pyaQ%0X&Md^8DnOYyCAl<%e=KJRb(C9dYPZANnpQufCS}sSkSndJ zAGqs4^)oEaesIqw32;h{cc^0lv$cW!nDJ{<>Jzf|U3TW;iJ6gvAt_kM85+ z$2TR0dCyT3N1RMnFFV`T@DN55%L){pO==E9{(8t6jm@a`9_*R!;gj-Wq{ocW1m?Wi z&!jk-=_f!D816R=Q-)M7aeGbUANvrHsv0XZIw7NWE9B=tQ^({Al>0j?;C)m&53zl> z;{WOQ<{6C2(@@2(2ty;eE8-KuZX&>31G=Y+o}_NEuvY9 zy$0401$4|0O0pUez7c2lDqzeu&APMbWE{vfyFFks1mHSY6XlWWddkiN?}Wl*lqQss*S7pG6lKLx6*{}uLtYdnjgLqtfh|{()ZNPWG3W`Iy)6_ThJL`gWgWpNx8L&q6EiBxO(q1)(>U$%$1&lT zHRpN+veX0FPbcSgDlLYeaRa*mjtdp%$^{i7h7fB#*lJHf5gFV_{j1v(=hEYctN(4( z9HPsRBpl1$NUL*JaC5;JrEh*`Dgd$}EH1N)BXjg-ii(Sc5n&jTMYop#9pp39B}|M7 zEVxXy+(Awg{{{Mb2-K}pbvplNy!oipFL+w0qHbCfV-#Aa7`Ig3sKi+Ix}6?^yQ4h* z@sVr3Y&OsCh3{QJCw-)|{^qdz{f_*=XR_G+<4D{NLGSzBzK=cH&DB&_-KWRPo9u0O z20egmnm=#*rKH=FdB^W@WW}Dpr%8X1jl}0mM!t^o^6eDI@AYqMdb%;skP}aQ&(5ar zc~#uCo-g<1gkG?C(M8Lgm{xean(ROCuVD8tP$~UQE>4T4M#L)rOUfN6{swv2m^FLB>=B zBtPAmU<@6lhJOb2-k)6knTuwb3A0 zmYaw34U0bVX)M!d%?flafSvG+?Sl8(iRa&6K7TfdbvG+O&S;UBPTo}%=2uzmN>HZ^!7Lgf2rEg1)gmu@Ktv$+H1$< zPBxAwILzVhrmjU0tRJmKcsreaS#azC{Jh@^^DF+8;cR=zyF zPTQ{AV2;4cC8SGn1$IxZ=)HVo8l7iWA?Ok&xA{%f*BJjU_u{M-c!WFFMOR@8M&BXV z?`PpUS`e6K6#IMpkdfpkjJiYaIpI&t*_nyE_6vA;;oo^>_r0b61l%nlQ@^?W;VD?N zH1<1*y&1Y~zwi?e4?A;&3tPb09|6{V_0+6>eQmwp-2R4rK6vBr`SW~X*p#aI1b@X5sO?ocdNa@H&Of_O;XNeed;tJ0TJ8 zeE0UY?Yfs-y|(DgZaZR(rw}%I9d@5bc)Wq^PHjW-HpbuU{WaYC{s-`>=cmwceb3X# z#z+d4gtcPnE%H1}V!h5>E{g$Z%De6zHDf%=^OcR@%4+@X+RP<34`#gPXfGB=9Xfpv*qHR`k=6FCt*axC*Y(lbZ? zHKQ=JzovjFoT+b!rHx+eISk}-xb6X3EOc8-Igu~!Z+U-EgBN#gr0_HI=t_Uq`SsG5 zdOWu+?Dd=z9s&LG+^Ii`1`g%(FfYt>tvI+cfdC-!>;n~PZ>X_^LP&OuK3=^s{^># zd=ire)2Lm0mJ{Jy3PF7=fPcc$z+Bd%BKdOs592h~y&)(=p(w)uzjLD@XTOuKH#vTG~>GU`miC8pkZa5um`{bG`cqyz%3y$VsC)DpIxC8sPfPX z-3vj8u?UBW5VvL6RH*#}7BAPVzFPWw3{2y>eO|7;2rvucqZP{TF!0h1xD+@@`7u_C zCAJw{Vn@ohr^oO}W!oO44AoZ^&pYehDmAq7^{t1*Ge{w6bxWq@2)SKU<7XyIBo{h1 zInME) z_o-HY57hD>goN`d=rUU*_e^V1@x}(wgM){HJi=vY%kSl=`?d7E=^I%#3YW-j^TuId?TygLH^-U7NZHJnHE z@0lLpQvcD_3tQ&j7UBI^;kT)8M?uiBkyaW%HemxBpl-ik&jc2LKCQGik;PLtgAE)xOd3F(!D!COj)O=B^kZelYwj_E3#`qs!QhI{Mt&(B-Iyk->)0Na3 z-gf$#GBLAr)^j{HScsD?>6qx=4V$4SMA)swZh&Rz3AgG`8#AzkP&471{A2V%D4g|+ zPAiCD$J~Y81rYiHO9lkjujrRQUbkXCGXr~D+-=<)G)LnKWNikE?HeYQKN&TxPplg% zlmB+u^zDT(i=DBuIhYI8DRe0|Q3)g|Yhs;?P$1X0ON!|w^lOc^t*Vs-vM-|)WgCz? z3iephurcQ$rMpon&yHx}Ze1U8CbrNatAlf|iD?g8YtA6Bys-3nplZB7T92ghe?pKG zc6$l$-_O2;MO8VFgBkkpj`V)uh?g=wQnWf?IgZ79}{ZskxD!w!`Bnw_(_KXy|m7|)oRkhmK`Voc49lPk&NL`AQ% z-TRK@v9(MXy~F$JN$>v1ldQfeNc^~T$Y=3G~J9}P}1E@c>(c_O|r`{|(| zRWOdRWC6_&D(vge-rJw@ior*c_Az04qTFO(wN+(eou53J7()$7J0%Q%u2FLBa+yLt zjft=cV1K(oDj3LbaQRj>!s~my5^Qtph2DOWQ0!*M*wi?=98pGqrThO`IDH|;5 z^C|%T$c!H6!F+&Otriw?yF~jI8EssbV-)B*@pW>#U30JUKV01NXIG~sIq(NJOrFq5 zCp({&50C`=s~|i>fLlM6ki;CG+SbxWip?y(EHpLtj>rP{YNG4z&{u>f3kj!mrKPEM zxLJ`gM?oEtqrKv)xX*eh{i`I0C%eCmyeYS-4gZ7)o$Q!=KvObtEl} z*wHWjR7IzOu$!|8zF~7OXnUwo;!t<%MNj(ngv-&hRhc1$tP%A}E9YVV8kL`Gtf0-T zt)ZH%yB!gnPe{zU*ZN|W8{{7!e`C|Z+A+L$GrVu$A^4eL=SD@`Qw2ml36vTaOpFp^ z;R#Hz(yj|>dmfdm7~!ZIy5&}RaSRyHU;CZ?@hD(;M+w_-n#ZeBm(vXJ5N(>6fN_32 zlCf)sccr)pAGn-#qjEEPLLz1U6c%RS(y`^S^%zNO6a$@GQt}2v0)o(!pgh+`@5&q8 zEkv;sUbPrFipENIafo?+9^d`>SrZJpNM1M2v_~hZFB6qO5n66^#8Ive@#u6JS_sY* z1Dj5~VT>@&+rqauG#vU*n1asHO_Q>CJd{I*b+i(NMGfsIm(ds& z?9hgZDK&aoj+48&WYel+Y0z91tS9NYBxu@d!ipgdxCbsedH1WEeSu*fJy650!zYMz)RCXH~u6dOD(?CZ+@xl+9sbFiGhEJPj^EcuDS3O?CvCX~FgIjwda# z!Q6{h8_ua3K$mpumL;-sLX!LE6F-=qv!R5#0r0zWKQ^PTE{ zEaGXd-B?j>cO&0jRD#jw#buU(&QFzj-=FGX=>>+RmlcAhj!X;T6=2QB2N)A1D^oRt z(x)^M(mRU?%fu+~} z480H7=Zm=)tId|2q{=+$K&WsQ^n%a;c-z}M&Ee7fE5^o%s|q0@qapp;-< z8ZTMq-V|4m5LT2Kp(CS>^*|e8BJ9oX3zoP#mF@g#L4~PEEqjuNHxWNR;Kh>eZ8(ju zRAct9l~K5ABA(h@>?wH>r>?V8h}Vd7usiM?6VxedZ)+W(F`|QCnIF0it&Xeu+B3I@ zW7y#iKIq}RwcQQroz}!SKcCDICTbB}#fWWhC?8yy+`?j44?iNyYIu+LmYlPy$V9aV zRp?YxTF)fCiPmPorMVD)Sc2U(&VZl+dltFmVJ8cGgZM=rGj^h^cGvi)OqzE85Cxi^ zMi2Ck>3rH|V^KW50iW@wGMB>|rQ)A)F!*$3m0iL52V=+5BF^FPJu0iB-BO)n=J*MYIOA&ZycrH4`NN zC=EqrqNg?T4`9!In^TloIgM77rf`Y`E??KxO@o<(+hmix+zkm7+bG(@D~?fqq!zr) z5mHIkz~|p2Odugt`Wd2)y+IfG>K{5dsvl5^G2dM;&ip^!Y^>0f04?^5;?AM7xb27` z8cs%`a8#Vd>wqDdBfSXMODREU3Hzle%0A0U=g?^)VkFxi>S2;zALuOP+ZA~`44jA9 zN6{vPQj{*XoF1N$73gWccarEVW!e#ecvRaIC-BAj$l6$D@vH4=4CiB#psdZKlzo}t zEOo@$pHLhW*FmiYekf0RFl6v#nr4q}5C=FxYtvKsoMmN>3UY0;71}X@KdsTs}ID_VnbT2sG=*<4qyiVj+XET z26X_UR>-Uwpc8Hqs+GD@JzgNg!Ueupg{cSgoqw+;=`^;T3+0$ejBK8K2V0&#AwZr$ z`w{)^i{0x)0{y(61@%7MdS0a-wr0pzg~mNikn2y!l4_h#t|8Z@o7DkLpbx%h{WR&+ zLjGEVT|OB@fJ~g-n#N_rIF-#rgN=d{rJ`Pq$}o4)J_`wYu}bopW_x1crrXKd9KjAx z_nZ`ARcM8{7S*_-8Z51NgN4#h*VSMb2oAKL`IpL?(s+k0Or8~N4-kAoWN3ePOV<37gK?89%GIMx}fP2xe+xPHae+k`E6e1q6g0{RN3 z$Sl*1YH<+6E!&rzCuVv>{?(eUhmeWv`3-=mq2`#JHX%dq<#Y(e^1;*$H`8jps(OSC zcxEZy6BbeKPdFTizs-m9OSeqYZU3gDQ;h*EA5sv5h}ICH+a|~k5=v_B9F3=?R8tDkw&`z|3eEjs z>%fp~Aq(0})&nyhvU)K<*gjdhVcNV{VX_dZnxCKdX0!9VRlk^7lopA>=m`_-p~&; zhm&kA1yPi`d#MOIw6HeV36h5to;|`71ve?la@92#Az>k)U`GI@%>8l%4SYxQ{5194 zou&V&8vX7OZ#&v^O>KGx4}<=k{n};3gr(=9Wp~p_JZ(zrC6CsyS6YIearv-Marzi7 zDq}72+plhD+Gn=;$NGLo%WNpbwj?^?HhPmp#JwtIq5_midy@L7{cm-`H4$vWpz|Omsi>YZO!t7X-EjnKkvvZC?l~IT_@o=cw#DH{ zyo_I9HD!+1I( zH>sVx^ ze&5iFLc%A?0_r$p3V1)CD%g}(_%=Da0mk#6VcfU9A7i#vBvH4cm*p=8{->#@Ue*gv zwt^r{O=PFEyLa(80<7gSWw2-&YnP8-Y`dmY9J3xKiHZ#?O#Y##nudBMusSx&)FaRw zJb%5Na1u!)cmE0F;;vy<3n?ez+&A8(K!Yl9p9NpDxW5;pHiCY zv0xg06$JT)tCsx^_L&^!2gz`EdH(zzwhcWY2Ow@#1NZKsYe|RH%W+B{X#i4cEJ$5| zSC!QY#p|I*GgM?~p-G|*olt|yIRUY~N={;cPxaShvTb=v`L*`2s@XVXW6UDPrgx|` zt1c_8`v|#LN@-N^#P@~3*t$x2nD8a%cvUB}u1B-tILXuV2hcu%qOp`H9yJ%W&ucb6 zE`Th=v~C-z`*GTS5KHVXSyqmymzg#gj-RvF`A`pZwb_5ZnE(It!6tI)Tc-mkpn`H{ zZrd60vjU=u8>NS7w%P~ghO7cTF!SS2kT45kg`^|mmeL=F4H#~hITbvNIivLKjXgTo zbWJ`zp>vx?XQD$;;w1GyLfqt80g?=8r2yJ+CcmZCN^uKIE?wU?VM)ibRh8CLdYB3i z2bH)ByZe7zh`{7T)PLQ2YWxT&>$Gf-uMJQ7pK4y3TVt=+N9uHag?iqDP}Uw<22vnGWSY&gAE?ngR7XY3t`LU z*-}NdlGqG0>6(*;`XqW1+Q+U1u1|<%K7hx5#mYQn*Z)3B7k5Gyf>jLx%N(1ua@roJ z6OxY*f@;2FbwmB(#yyzVM>AEi}!aTkq$cvD!?k$L&l_L5QcrkUk&4 z{#ph_liHQIq8lwUnIkAv{M0NiTcvBtJ!jKgJJY50F6kU@6rubrI|MPoKZsyRMwFDw zws#_bUz@5ey{sMll~SyM$ozDyfavM)WLAdT*=8jRk8w`Hb(9M8RcYIbU@S|;Qo3S^ z6C$8x$r`Csjpt9$XK!0&$Yh%5ZdtocR=H+YcvQjb3fl9g7CfY4k=Ij~!TtM3u)hMY ze&OJ(hPB3`vJy8Y+1`&|@4o)8nE(9t`@J3i?ezFPX4yw0`lZ)>y)gr7BE=a)LkPTe zWtb3m)JNyEvFkvqR&?L`S^io{VRolPt=zFNu{y#3NgGr(N^35uJYRYQpIx26KmSy z(>UOzdV?G8QC`q7`uQk19ilc7Gq(-m$}aHn!4mE9h+MaGrte2pOe$9c#x_$CEtV-O z!je7}F5S)W*w0I(3QK{-T&?^}wov7OV*iMRBa&)9`%Ft*w?N(Fg{(N%ahMK+DUY8o zb_ac_^nA8P%<8Rc9~Xe@jJuQPPmmZdj5&%Zz^dHoB2nTI?hiaDIO<;lU$2ZX%5SY~ zSd4JnbIJtqbUis1KC|?1Ac}U8EfQF4Ht2f_@ z+|*^xW=1v?eT@?`I&0B9;eu_Glv4A#-l;4}!!5PJ5UIbaOu9HKKyn~pneI76BBv#P zzoOPDcsr%S&e9qJrW@*c9o{9H zdgc?Y{GvIioO}{ACKWhUpxF=AL0ltM%!MP}KQme!LkVx1ZkbQd(rHn#wRE^C z*90(1elLm5|oB|a_QJ0PN9tIwF_*@u31S0X<3@O#jO17_8)->gX4a@ zp_{izMKhw!?HxcfNx@zg3e_|kJ6;I`;29ifTdl?*=!@S7S@NpUDK)RWH^!=)ZzCyX_(|*0g@eq5y8{ULXI^UZ=J(R5NIhL$WAxv ziU)I8zYdi%gJ~=Qt!i|$2Q6P6BG<2!bywx4jS`N9SvlG?k%<5`jk0e0=m!9`Z8bcL-DoD0BX12fTRkFKV*uY3QfunMMhk_i zuh6PYEF-Ib&cbEP`LvNfc>9h*cuGwE324x%lQ%GcWSHoH@*g4ntIU|PR*k$iz& zscY7AM$Hks4bwg3dlyzw-n<+WQBV7Oh;-YYBUG8uOtHEoJE~&`8O5G2OdF%I7Z4fO zA}6yh6VeS1+ECyo3M7uUtYJb+>x8UQHpf=zRH9kTOb&OX0Er&QrpK&W7+6HS9Nybh z-;TC{!ZlNBQ({z^vJ_=g8#xbL*nPpCYe2BfJ8z9OyGd5<7no_kG45Wmio**{$Y-eaGh0{Jax&w_aF!b@&tn6N^11F z@Qzr6L(4|2B%PHzt_a>5q$C3<;?Yg+x}B{e$tml&wA?0 z^0gks^0){G`R_gun-B3u4f!s#y^Exlr7VhLAl?umKvYV9fixV6E)$o!um) z`gD7H187`g!V_q%zTZcx_T!ix1gj zP;C(6IU2;b_|BH6N@M7ChCZmj-QAv7yjt9^{jGPctCThqvL?nWsjcR#hIRPJc#^x3 zkA^F-n|#YnZvrz_gH-l^N8B^Laa&U6MSV{Z58Rv&wo%;e2S0S|AT4x4wdYJ}=&R7O zh_CYFnNrb79u@aeZNei%uxQi0&L|67jwwh)2~0p}=nKTPpmO6=giMCI))8FYU?>9= zStgmyd#vjEf95XvYE{?RaM&}UdpZ6M9DgdX(P}8KV9~64ECaRc0KKX)xt0qs{D1rI zwLt^6E6esHA(>oKQhc6(zc0}bR8{+t)>srrVngTy4{kX2cN?q`O(7Sy7%~wEGHnp2 zOkl06oi2J6g9&AG2IK2tbNX~x+9kxiGSM12WFVdZBYVn*?gO5nP&i{zZVRifS?(6w z{yP}V6_NvzV(@=*q9h~?=h+3I-jF7AZUV_@nLlrx=|UaNx+dhoyO%Bio*sS?Vb4Mh z9D*68tX5vEmz>*X-9d)8l|1c%nU%Jb^Rrmde3_J4GRiYO>)Eh2eT?GM{`7yJwIkQS zVJ>X3Y__mUXUWv;ML`=J2;4SxSkbY-yM(KO2hzoY4Y;PTB(B;@px3@J)DBhz%*wb@ z{8*9K(4*`-*ND+|sL@r$xoKD$q<2QtRbg_IPel__;TqxW{tc)^4XDQ{>nt34(xr}j zfh_e>Pq{ku`F{>>%-9)2Re7%H++dhc`*At$8JN`T@T28*y-}VdvFQw?mkS_tgfE1j z(ldWU;w&K@>&WS*PDAW}?@lxIw>SIMPfPm3DH_(t8fOO{oSE>U;Cxe55j9s98$hF_ z3z{V=DHW5}q|~Qe%kf@m#D1lC?N7oP?YN zxR=UPSh#rPbkqj7{W4-ELNynViJ-+Ez&4c`k#DnmQ^r2`*nY!Gi56l`w?`0(+_;7* zHgWfGOMQT*e-K4i0Yf?Wff-sB6&Z9*?5z}yU}TOD)DDi>BfM z{z$41zi={RC{_m1mJY8hH8$-C_VF<1_>`D8JZ|l^lMy`t`sQx&zL|n+$^0? ze=v(6B{#`VFf1vo$xQI6EkcP|ZKO6P9=N)y6(&mAf{9uv;;g>U`;2$4l3^i}tauH> zteR8E^5{)r_KMdK%VyG0KaZtpsySs<+XNbMh-{L_{~G4G#7~W%tgc5kd=`krtek&f z3&hd#21DNfQSSOrICjVNz9i*Pg0j04@UaMRI1M~K_(dt{N!xge&p|(s?a?n1NU-n3~uUm z<194c%iJJNu3*eL32!`uBS^E^CSLUKB~B>2-z{q@cUJ(TJX`5rf%e`}O|b9OEq}s9~6IRb^d21LQ9KKE>9W&}8aF;c^086qJ9f?47 zJlJc=KqqbyDY{!(Y(!`26Qxm8lAzYMjAsU4jBQ^Sx>;mjhoOhipc6y*`^b-ij0b|$ z>~Kx-Se3U`U1YrIs&g1)0j}WaMU#36oz7J8SmIBUJWsd+*{XEy$algNwY<_4IydkQvRW`j_Y13?Rx)*s;_>F^82C{QM$WBrMnvh1SF-q zhwh;pC554sZjkQo6qp%c=x%V3&OtgapYMI{5BD#4&+|U}?7j9{YwyGE0!@!k$Hlsz zgcB7Pkn7Q>SyV<>Q(~BSHdCdxC*1V2CA{BeEkPzHmbUcn9ff;M%7{J>ijiFul}kajW}8)ALx8$Uz1&RVYf~Ce-Uu}ZEJh{vJq8I~5kgS!3$Ofc*+oAv zNQ1~DcJ*C#;_Td_u(F*gp?IbeXXDsgc&aw8lh;+Q{nZb`4|onIQEHoG2hqFA%nJ0{ zv$d4S49$RHHBzp6EPC)>2F-d~a!m%w2E9rA{p@oF`n}R15=z*0B;SnV)qduTfZ#~x zOj5q^#vRex>*cFZ`2VLl{IjIns*LELty$+^%LDFH zJl3Jlpq`9CaUNS8CO_A~CG#`9ovg9b43HhI9GIVkZ1z_HRBC>8;1soG4bi&cAdTPAML-{TLH^PHI zeJmlLJ~v0sttEdlE%0%D1;gXh=CaJ5kbB9zrjY}S0mn~oKJ(FIYRzutFbuO&45iDe+?0wmz&$*bL$%Gmc+m;7dVd)bl=w@+BB(k`ps~daU?QT3;P3aYN{V8J54-MeenJ1=xa~z zrq~p~kYvAewytrN7#;O_6#t#RSL~m4ONCZ;%_&*7Jb?y>jHI|5=1U`&e=_7rf4x+_ z#nwj&Wx`j!M(Pje>axw))*A<`^Bim0WT}6mPkasZoccLsP=|Xj)IHvrQippkN|iFc zSGsu49!5^0ol4K?)pyPv3V*2SmDjH%wcx@%BlZ8^R=7)zu8?KND7B~UD zP0P*(3sn+B|29|jd}<>a06(p${7n%gDK~^rZsMLauoNouLeT0q>QmNK=cVt4SnlPZ z+6csZKSNlbRgkC2X4z%9&ao5X=)55N%}xgWv)+&${DLq^f~{biu)tbID8ko!NC_SI z3JM%0a@V^o+uHQY)jydQ3(q`qJPUEpogfqN4%(Ew~dMB;T~`25|^Kg&=oj zcaoZ$!P+}pk5w(G{Kp~(;hORZyY42=oDPW9-PE&25|m<*ihR$UZs$MjuijWhF8FVX z5h3su#Kn@I=bK~p`guP%eZ>&3Z0U9=TuA8*kvDP0wE!Nf-aF_XRIw>vYDE9DBlXbD z;#cV9GP<34bqI#T+V3OSUC3qd8tVAzo~9yNf0IRjZ;Xh{LnL}ykBIKRQ`d#v=xr@R zmFV;1Y?#w?r1gaA+L8|TnhRITBxR7OUHFRf!?|@+PwAY3AfKGD0PV-x?iu+%I2oBz zB>0VD&$JqDTAJP8gYapI0#PdYZ0vhfB1zwVk4Q;f!Z!kH+Fjl$v&+xIumkIFFbz~b z3lM>s9dcD&bUEIqWZFvoow_o=P1-c#_|@ngjE7qcj*xJ-;x?_Nnf&=$N}0tyoEwa`re^uZ~#ys}Se;H&IiX z+7zdcg#pW`T_e^w#^dU;6JM@~=jBFQ&EKALrISkiBMaWE8C&T(_DpV@tD=F1?2gFs z>cQ4JUqH*orhw70=M|AymVViaDbf~#+E7r>faM)M*AF<`ct6b9!wr=RmrRKIve+P7 z0Jgpy_IeV9y6k;CZ&R+{nXgiz$d+7NBUt6|c(Uk>r=Ny9bjl?kXdPwV(Gg$IimQ}K82>W6*YZehH|lQnx7AT>$AniJOlpY! zyX*dhmvk63lfG<3)t{gNS)ASuliR70>sMFO`l+l>jaa!DUunyO&?5h>?Ab1kHA@5o zs8&(y%whM>32qBBM&#Ywk0ybCJLJeK3#%Ke@Qt(pudP1j86;X91}{&VJsg$jvx^Ue zb6)*e+Sc4!xiVK9NH0-SZtdlb#ov%y@8|Tziq$JzB-f;i-)9F7A;6eV zUv0S7D_fG?tc%-0GAW)>W@9`|#qgnVj=qzkr>XGW6MmrMGVNB`A`>90degAoLd}Vc z=6!t7sZ70m;O}n!Cy_p73bW}OhBczCL>fam`oi$*Dc%pU%18O+&Ks2Lnp^Hbxh?4x z%R^POc7-pg`ji3#3A;XJf5ZMZDndS}TwdG|7MsszV^Ck(ajaLzD4+hy3B(_H*a+ZhVWnPuWO4QGC zQ(RqepzNj?uxEA)-%6|gs5qTz_A6g$*vztUqAs%J?f}b>Ghdf%JPLm7b}%-j>JffW zqH39-qAB=05zB%bOoUK}8W7ct4%mJqv7*_!6rtLk031|MlQ%?Q2!KcpT*wwpXA{ng zoU?8hgdg~X&d$}1X=3yWfo7mA9}bQHRg};&;Rr>8AC#}!ygSQdGVL~Wq2V5{pb96{ zGX@P_acjq7A7l#ecD0gO<&Qh2{;RqII62X81lngGmSV>O zdcct)0+MW7nk4hAjcdwVdB|PMf@bW&awG#-#nN*z$4>jg!=Tp5A<7^MXDoq5mkPV~ z`HIdO>eG4c@pAGpt1jO~nqwpF8}gt%bPtLwD-qE&f&{mQR2jZ;obg$+Z1D2>C=KD9 zd>C|E0Cdf0z#^-xQ02#lFSzLTruE+7fv3;E{)QFrWwR2CW$Snt;)LwJ!b%}(ng#UN z)1aag5|iCJ*d*ZAA1MfDnrjCrLGaa0IdaW|P))@7q^;rXhlRSDQoysH7t;Y{09xPF zZchyi`_l?ns|3X!552)|e{a!P*KI=4-Qw<%$&|{A}JL8dmXwxO9UoKFZa@=28B`ch`_m$PVy5I2<{B3db32)XbZhgvBRWe-2%vl6& zR7A>O7FDRB4=|Pdwn@30=5g+=zv1av?BpN68y6UFc$eEPbKUe)K*_it!}0!s0yM@+ zV)iz)ClP!w~9ak)MX2@&3S%328_ z+~VjEPPP*2U%Wyg@%QCDsyE(JwS8FXK?X%UZVpRI{z4F!E(B_4%B`#xvh3yWj$XF? zm%ibsw;=297V52M=XJc^)%!;VZz(q2NzM=;c9&yTJkS4Xh*MVgvY|C_f-{$GW%-_6 zf-(df%fYJn3ROEiZz&zyhCGh1NKmrgV5=c)a;pjDUx}qN`+IHAvK-ONBR;l4wb4{+ z`MS|tu{zWg zC8zYL!0pW9l!@gFNA?rFKcj3F$84u3J+a`z_L<3dr8xu+av;}@;>c#s*N(S|Ssfny zvwY50$CtVG6SK+8FtWqwtpl-S)Yl4SO1dhIKnF(-VO~S@v0ewe4)(<7Y4($hd3jvc zc}6Wgz_yGSHIudif&>%u-T^`Gw|N?1@KQ+v<(f_SgHYo0WwMMG9t$3&Bx8`h}3&N-L@tSZ!`) zUr61aWwMDXcqj*gaRTU;IIty?t@$O@ebOWj5Hq3`6izGLJ#XU_?sORQWP;rB_m|2< z+EhdhsRWqD83XJoT7Bw-=Abt$sR6LiLhx1}E)3IfeFMys;K5TvGks&~{8C76N4h$N z1A#Xw!mCdeVa(S+6cVrD(a!y1+w%M#*2XRT`Aea`KX>^w6ZQEmXlX5KPU+Y$?IX5Xtg-)=b0hR)-x|(&Rjw4PMgBp&}>YNiFDck-x$!34+Oo{(H^xoLol5T6u>zRKY4$afa z-A%cAdMjx;LCTH^zJ5=a>-jDIckcwn`2;N{bbYtP{(cntW`m_0uYyK_nT+SwI65UX z!NWnb?`1!Fiq};r44^~P9G=L$tB_K~zcSoX+C>a6g>0X`sI>|Sep)Y`tG zfjR6*!Rshcq#VAA^la@vA&YV;0~GfSE_wJRk<%?dMy$Z!(GhGdnaX>U>ifxvt^Sv% zi(m>s;LThhk=CQikz7y}9@jDmwp#3Z$@ zRFocsKr2+QF&P<4NeIX`$ir5bP;jg4(Pq7HAD<9yiwty@V02ehDe4=ScINbea{WN7 zZVeE5xiRLXQMF1$T&a&(s|TVSS^sPn`(hL*B-PM6HAN@U@usk;P+;>@5;(=GT3?)0 zg`xg-TYgPauBlL4v-&Geu`ni^iC`L-9&r3qy)Q|*&fDDN0o>u>9jC+S<1tYQd0Y=F ztHpM0eLvw?-G+AM;a?y7fx@qbbEsa>3DaO!EWs1`RcQ2txw!q!CO3} zxxd%s5}LADn<7&3eUFCV+yuH44#>nsqVS4cBP*$EPY^A~%nr2Mgv^B$foTbgrmHq$BIvg6!!T>pN&rLSx>`X zBnyi;OsPyivMK5UKYvO177%UGC1!L)nam6M`is(k;aGcxBj`l%k6Ts_IeW5qNwxKj zW|W*v|6S7)`VD(*7P3vF2Buub@z+q18Hj@1BgA>%_zT6T_l#vbG2|k>;p38O7Qi)m zw3ppN+pX+UR!-q9D6M|#R)B0qhh)#4B&59Y24t>6Z~hu;YU5mQ?T@K3M<@6yA;z-< zy0r!`qKhb}Vwb}Gkh+69D&xuDcN!hv~oE^7y zNmc-wCw0mb?<(GYX~8jGdhI`F3iou>wapR!yP*H@0XvnG3%n7N2Cvj#T|)!7hT~jh zyn;`+0-av{_J1vf?8-6~=`}$v+6LCvCu|*CoVTYTXlEVpIn(ro36>j8my0y~*5m@L z3_D<01vtG)O0%i&N$OrItT~Nsfxs}#6P!Z}(}E|Jw&&z+-jpwQvpB`9&f0naG{uAkU zg!wDYbxcj(BzjKuM8NEvuRGOBoZOp%>)}PwA$?!d*lZ2 zNd!}F>hS#QM}XjM)fzl74SvT0G_UZ;eeLBc%!LDLr!~)rG@_G+G@JO!oZqh-zlfXb zA3or~)9t34Fy!tr_B>;g6rI9_OQ!IB@dsY)_yWGyCAbx4R3xNhQ(YH45-PRSa9{hb z&xw$wEbrb04TQ(n>Ds^q&;V(aS@b|2n+=X}Y3FC}qYI~k)kgDamS)`=bHvxIXB6Rw zvY3n{PabG_i`rG}pDK%0?}Ar6gs!-A4~|qltTv_vL|pzsrNEfYzemj?@MQFu$XJ6; zW|cwqzeZ4wIcEJsyM~}v5xalrg~7O^AZhAX6IpK`MUvlzXU2LIQ4KiRxt4kwBB06m z=Pm+x{w5U4BmsE7?|oIMZ$%?(RdT-dx%^OPFBZVCk3HNvcgcnLOx<3hdGZNtsWwS<3!1%>!6q^A)$M^QI^GQ@e^0mDIN_EwX?TkoB6! zya}tecr_uWgst|Ih=59b!qJ_5|d264rN;G9*Y-!gDNeIbqQC4+Qdw+#M^XLaoic$iYt-!oRwU8I=X zRh45Iyz6ramuso)PKP0d|X$b5VaIWGO^8tvsHzo zg3D^A;>o(lY@zig_R!RNuPJ?c2QWn8*m7@?jXbzJ34T5V&^)6vJfFib!7sYokvA!8 zx@2gdlWqnWOuL#5X1-lClFK>RQ02J`U(19r4w%DNhMSkrVN5qgU zDXNlE=_sk#JX`r^?_a*wL@Q#;lv4m>B0pp&1k(0Ll0-)ZW!QV#_&8tMk5`>0FP(&M z!Ho=p?bl6y+W`6jokPBF!2Z2HP;wx&s>~b2O2g9+L8>e?;XmXT6ybZ*;8s}m_pFGy zChy-#(Sm@zt$mTxtDHCm$I<&LZ9osw@-IF4`f0Ig%30KiMjc^7x-2hXndNs?zM_O_ zG@!FX7o?liM+&v_|aeFU4k zyHNG0C-aYRvS?hDxx}y#>c-6Mvv?9TYWr1MzKjmz)4$X$i-@&7mTxSHy!umiGF?C@aN&w&f*+ z&|3i_y>?6H8nT#L5A{Ien9`;y5Aw*uBAsCOtYho#w)=1KwrV{{U_nQzq~^*v`Y58b zyL~Jhe~QJ{GKP&dE{COs!d$F|@qBnjO%}uFq_GKKd!j?JbE80ugy@3AN#?rtS8-k8 zcpS%A%#qDP`}KgK6FI~Lk|~(klmL>(>D}sg_K`fZ9$bXb611QzoU;WHqPb3kwW{sS zqN4acBb+%YsB(>U9`F@?@|12yBM>#lB_)?x^_pXnBQOytDez&^59rX)Z6AOoaIZUC z-(*grL1>_9c&ZMs+-k;>56rpN`Um*xZp~uGxwZh`wleepk+ybQwqDd-IL!ApoH&|a z&CD5P@?W<@UAp90386&5hdg1wAq$s{U;W&Jq*Srw5ls&3H4%OU4b6S4#QBg1ssJyn z*4D&S+~025`NGhedL2tubXxW?jhNe}MgX?84eXI@9BFxbI!Gezg!B&oX-QSvD9=>k z$|{e*)vHbT+CT=*x?qs!)kthGl2aw$FFReX_R! zg-_|q{P1`$8_LO>(|0?D9azkg#Yu6%!IREU_y1n4Y`1q0p7zQ=JY^9Gg%7UTk#>i{<`?go(M*slx!Bt}5mLim4 zaJ__I2YW}M;M~2RVpXBuxqZK8G;$dK6GIa|VFcqUV2}K_cH`Y#Qp1ghCHO-9{8 zxh@019O8^Fe=_us#s&0^Vac&sKNGp@0X96ay0@nuwMI1~RBrUM`aMi)#S#M;#T4#i zK4j1RKQCa#4W}Rebxbn7AkolsfzXYo|rvnC1b+jr#I5jR5 zWw&^8SS~+g|4$HLi@676+Rdu(9$pwWuzvpWUcKv_rKP{;xWf5myQppQ#y!0xamD8&&!K4) zF-R%6b_XcQ5%yXPWoUZg!xG zE)1Mainq>n*wVur{Eu`B*k>m{^u#2+3;dGaW|o~fz0t8$=&)0Uu2i5kLzQSq#oTEa z5g{8yALl0}Aca!)@-eU8D?>>WyR~p5ju~CM^OMwa6y3-#KMMGoMmeq>{p3n1?$t_X zC+p&jFnRHW0%uch^SsnrJaqJtH)N?VFVKtYok0C-3Wsx8`jnbed|v{2I& zFYJ56tHI`N9H|X4{2;CRWgj`a13-1X`1?@biO=ypr8Jy3bHT9 z18i5aS@3OvHzfi=>nLuQ9y0Qdl#qf+*AEsNQAM#H_5n-4J6-R+oeANQ#xKOcJ6XJt>y%k&!)j@WsDZzKX}$**u+mI=JX=bpW1+ z9DW#i++lG)Jz{6}34osShjNy#mlPs}inah=+hz~uz|z7FsGng+r5pWxGjv$0;cH3D>$K8jRB^3t zW=owe;0V=r43NQ^!JRGnwI%i`XD(=m9QY2Qq?+xMKHi+x{vmg#?DMW3w61#&^)@Dr z`?b$V^sMkkT5^vfMpN8S&^Mu{w^+NcYI9}vI3Z!`56@5gv{2foyhrFW?|j~q9Ld7( zF7?pZ`izDoi=$cry>c%&eG$c*AeCb!g#L|Ya@>}O8*#0qEWJ=<9XsG~l4BfRRg`KP z?Gd|U;882tJmcOd!75q(A@U3N`V5=$zXp*`1Gt0D!sgwoq8Aq8ws#<% zt`kQq;P8RWwx0PKiG*uJ=JzzevOE=t=XEo8Qhv-T^480o9n2t51Vp>V9?VMi1?HV$ z6r+RS9%+meD+v~92=5v(lr^^cuG)0h8@2cEz|$nq;VJ|8C*zfl^#FQj9k0|%S1G}|LXJrzyfV+B6rRQ~|n zEcTai9nt!YyqAtOjU0Jr&n|!HWglB0x>3-*G+}7{>_BmyBeD76!p)n32UFvScqCaX zOEI0|TF2{|zppX@vR!hcMAAA*Y_~*6;FDMn)%2Dy?3&=LSHmKr)#S6((0$$VdI?$j zE4NM+P;Lg)HG1Q4GN!(YTLo4=$^t`BpZ5gb_pX~Hb*zvWv7 z-%F=&8zS}mlo0Y@EBUy)f#LMP5=@3z2@yJ)9K9k#!UkGFh=&dpW~O6Ud@Q_P`?ELG z^6`#phs1Ev(fi}_{EU;Gb#?*6UUmwuRo?tF$88{+IjuIWGeBQM3=73X@eyy0K{qPR zqhzvR{*N9~z??$oYMBSmCCBo59MH+)9b9yzMLH z)VSXCXMBtkH`@<5Yr9*@In-AQP`=oIY;9XXRlh7hn+Kugl9F~V)rQg%@=@6&Yz}Jp zT0Z~;Cl3&LmL|RyM?3Wo!6YI4LLIAp|07_7TGS`!G2#|QLhCQYsxDUO>uOaMT%r@v zRsw6;*>sUw^FC&@GG+;d-?rb_2RNMV+6hE1mm4VR(3tTI>akmh%}{hWM9l8jssK)C z5vX5r56bbGC2g&INJc5_o155?J8s$ONiO+Li(V>Wi~rP3NAmp&P_nM3+^ZVSly}LT zTraRTXyJsKAE@4098>HyJ?hFBD#`pA$)nwMYBA<+kjE*{^}raKJKg=}lpC>1|J~uKqW`VKK)6HAh z(?{vGh{rIIeHO#IKE(^u)uZ-6;k!6Wv5xCE-bp9-krT2o{wH$3y3wNRVqGs031(xv zRVjWg*r+F8(jOI}E>CqQE^nh#V+N=<=}zHV7+{0s3vAS<;bBxKDL!E;Io6HPi2F9w zVze5=rc50pt6GjrGZVpFd|ssTxMJO`>zaQX_A|OMZYZGbfU{Nt)kuj=GI8f0uqP4u{wp%&x{ElT zZLfuFcQ%W^g&g0iAzYlX5gT~41C~W1mQk|}jn(d>JMq#H;r2Y)s$Q+%{G~u1U)|P5 zOr~!Hf*|zPb2(ouW}Qs4HLpo(Q8&$>Bj9XUET!Db(R{##ynXb zTJ}LS#Ws~4tI=BIDG(b`M*o@D;YH)pxY)PHaj zcF2Ft23 zjHU~61XLYerOj3S=2Oc7lmV4sK|w#Bct3XiVyVIGstxDyy>_neO~ruj@+6}yu?#qZ z$zr9qqyD-#^|?4UR+-XYBgQ8(8_B>WCCuwqge?Z7;9vD$DXKy|Y&bVs^j#u=&8Q-*EtjL^hrZJqKZ<49pl9PT zNyv$Q<7Q#N?t@Ab?k6R(DAm;-uMB-Qm}?HTX9|$qr}$FJf}-p&HtW!N%kh%?2NP-qNRx;n5@2R(N`9y`!X}Yv}@81%@ z*odmN$CC95sz_2)Lf|(s^-ky#Lgg;z3I` zY7k!@m$jlipudq3>Xm@ds(S@yNw8LdEm?8KIdG-VT_xCvX!pFJLKHO5-#6!4UB7ru zJ+6K4Esy{14iesdR$vkC94hbJ`1x(u%c-c_sEhnK!mhf7MhwI0q<*}yU;iaiPEjOE z;Q1x6oiOm2a_QXMmNtE4nhh|3>A_=ju{RaW*>>p0708b=hYQKf2fSlO7aYqd&G zhG1hf5Qv4dFa$Hjhv3wi2FGARYSFf){Qf1Xb2|cl7;?P&pvf(0q|iSz&r$EpBM^XF zX0^V{TWxTf^I|$6=Y15@hKTnJHRg});MyAt9BO_D9LFJ8(!VU0UHQ~#Bq32IJ#7%H zs-t8=Qz{N)4U&fKyq>n7J*Gaj{*PdDiDBE{BHoJq(R4hg8>rSuJ0SosVxdQ7XoAVK zvmn>zA=h`4golcEzYU+GTL+r2d2G(i_Z3Yj(pDfO`t^&iet=mZ?1c@)T8A-&$t}H{ zvA5KFCWvZSSVfCY@qh_TlFvg7kvz%Xs6Bj>nfVQR$w~Io<*n8fC166h$ZS<~q2&^6 z@|sQ=CzZY?OP@_y1|nMTI8s7=+geqEX`F=|6FEFGt8pxn{Ka8DcANDiw(ptHRJ*ot zAt~6KdKrjL9>42IE-Kpx+gSlvuX(o^vSf(Mm_k=;bssY`a@A#Wylh2)%mWd6B-z2#XqWObW6S z3)9r@e@~pr??ye8$D#3Y1`}r}cqn@{1<^D1^p{m4Z9%9}Jd+soQbq>YfRTHw$_`QD zxkkGweTxrW{Gi)HMhdA&@ErXhR{#?Fij2+60}`H1mR|c<_tEz`(vT2xIKB_;qHEX~U_BQi+s0Y7?rizH*rg29(_-g5B#nYS!&yg|Bo+Mr*nP)gp>=TqWSbD9Q1(jcV$MD8E z{ov3u1GDOo1Kroz(`Y^HP}&>S^;ZxX4nM{G`~%JYB%m#@h|#W&40)hik84D7KyW>1 z+T(w`4IyYBdy|Ly%;{z?=L;DH1^boni-_>{>TmjI8`ncp{cPkeilG1WPEV~sVvsEA zxbSD0rrFqHVJ7vWbxa+FUCE#|ZKL&h{0YO1fvXE=)6(qu0I0JUcVI%MGX~J#YIk1e z(N+Mkst&)Wr@9Ouy(W#S?|%_1VqlIEh~M%I^OI|t7yyHIo8#lCcEk`U? zt&b)8kX3COH?jC(URx5khk9}oFVzDMPs~U_$dIo+YqshRO zky2u$!=(tqn3y9v7KBJ^cnf;D`r&b5Xnn@4G|7Q2xbw!HoKZC}Cdm|Lvw$X$IivmrWgNsp0Vv)-++fslPH8{B%=J5aiEuuWH?5U|$d^ z!7;U#_1psZ=?rRVRj+#bl@t&J2ZBb)GA8$_ZWyL7yVtyPJ~-D!#9{abiXJXrHHpw( zMT6H!YDObYYLXQ(XnwZIh!{Bw>$v)wW`ZxNZ>txeMS*T0;z4qoRYvQ|WAcB?j`YH) z=`-aF*9|w&SC^~>e(fx#TIA3q?QeB%V%GOfs~*60^(hLvx7)*z*2%>?7i5x?NicI+ z=qf$CRSW-lo-X;;?!o!ztu}Mq1FM|+Mn`hV>TJ4y;>`chx;f(!)E7D1svVMyhg;z{ zZw(_pU~x$&!+{+UYxt=P(snY}?JO^-=|#p*%YyT8KjGffLTI11La3SKuemy6feWJc zkl=%i&5n9c?4=5RnKA)0dRy9lc*}&bm|(3?v~w&y7IL|qPfAragS$&~Nn&HFYPqc@ zNlNXOx>V3`wu|V*Ok~U4#I*0tv|x7%ipQX_3X4QQ{t!wtdVYYoEz0_T48H;UrAgyk z8cB#MaJ+A%GB(L)AjjLD#K`tS;_gF&P@gZo^&J_EZLUQ1aJ9t>r+Lvy=}b*y^uiGg zvB2gj>Ee&;tBTe75DdXXdQI~Tao2CaqeRI=k;?0OKi$Kk!Ob!3ptw>m@@Jh9g%l+Z zL&<;%W+^mB`kp`eBg+65;hdNIH%++*(3C96lXlU?sV&g|$&0U7f%OB=SysM>g+A+Qj?fZjcH*T6OnWxHF}!b7Z#E zDiJ_JT280^p*-Cq7tV91L9;mGiABEfIVPlN9lP4ymwwOJ3+JCvb}sH8ZVQIk#5`7=z8=tWT^&_mh7P3k8?L-_0~*H5zEk=X+h3-;b6?z$^_SHA%hVQU~dyN{)!_Srkux@ZBF zPy$NXrn2*BY{FjQYkBntD!MtCPGIx3uN;}OP1@WVY*zj5XGC-0EoTx=Rbq32jLWxr znE7_I3}9ijuhJvRIj|)McxQfx175B=NTu+g+6mr7P91cz-BvI&8QKXqG$V&#oX)P_ zNUMIfxE3I(Ziu(u>P%NnBiRkAEPQzmQ1o7n>{+m{vJ>j%cpb_a^J^x8KZs85nxQXk&w1eH!C=qq4$id$Go#Oa`@&3?rYL{cf(&mGj2sH@~8ZBd$10Mch}jP+!o@yK@qfd8!OUGh4{7A!tgO7Ydc&g(=YsU_9`0^m*AcND{fb z8Si*OP{|3Sh?Ll8Z9c<#*Sf90ohn0QWD_I5G}VKGstp2rD5zMeKG& z?Ywfxz|;`sXOnrnvfX|0<(vq+t3$%@*>{nwWiir~bi_d}5*RwLIpV@&VU(F?w&{jd zcXc>_a_Ms1Gbx6caQWq{yl|omf#ep4b$gBx#8q zcm8-ql8_Yja-mCV!1dD~8-g<%<4=%H4%USe$8k zUF+8K7R|`f+nX?rX3JHfN49tJ+YnNM!I1HQctYMPWPAi)&Cy+GHcj3ofC=lzeLDQK z48&+#Ie8?_me?LCajz=9Fo&9x$Z-Ac8u}{B1G8&)5*OqHXuW6Wr~gcf0Zj}m%{XC=5i9fXofeIQQuL*<*P)CilcNd~>B z8AZ!u&P^QcUyZNLC~x#(gR$3-i^Q)@$M*xWVDA^_+9?B=cIe1auywddTVH&?k z;&yu?ib1`clPUv4d2@C8Qkqst$@y^Bv9$_3Tj-cVo$y~SgG)lA53Lz1L!(o^?E0ba zgI`i!2)8uUnlIJ0CDly%bH1tzrrI6nypNm4_iGB-(`~D~+(a-Bi+5Aq$5#7*=G$!M z|K@aJ;CJ~-{Xm?XX<=G^6C#ed*c+JjoW>ulNx3^m<|hNcr=Kr=nBF&F^$C0a=0rz9 z8Dy54WC>9q4|4xksMYVBIw-NNeHD{r=@>W6f^^Rr7XLPijVx$n1 zk9*`5B$4W=bbbdh(*}DN+{8%T_lN(jmEjo`WVPgK6AJ&;pmX=l)HiFc{Vtc+)xHkM z&RwEex>zn}R zk*g9A%7`Mv4yk6+yfrU;=7ox0W<9HMs`i_6dEs&djZ2N1pPWcfWoDY<0_wNYuDWlFY^n0*c^9Y4Vq{N2(5bB z+S*r!>*VoA+Sf z#<&`>_a^+g!l}r-<<195-XeeeR)V#NridV+{%_!|7FeHyckkNwGPq$Il4{z$q4Q+<( z=*WyXwA*hn#9()K0&gmf@d-)4n~lD&7Zq%-KR0A@RQnh>xZ3%_K(_HK%f`>Wz~$Jq zaXb*FPcw3rN0O7scghT(NFOgx;nHU zA|#$t48Lh@*{vW(68m`?GA+}sWy4-f-!BZG;vY~S={K{mzKRfH@gq#i+E`cZ!xI-GSF|_E{bK2WBz%w4(baGEi_=*p>E&$!M{|q!Y4ht@#W)R z*2=-9lFTsng$DZXp|MU;rlf;cIA?0-y8Apl1l3m^4ZnFsibzP%P2u?vd zFNUE%=ZePvebh0kV^A-0$=1e$>_^ZXMF`oos`_Jzi@YbvP=;~PaweuH4dF4N;K6}O z@4zLMDh<|*vP4(odL-#WjsR6RjDJAb(KK`5X8aiY4sh@|+EEY_`_XM;oPww8cK`%b z@;178{`d8NabEjtfb<*J>t2j+QuiN7u2{!Qk9uUg{5ApU?joIr;#1Sx|1R-kFVcH* zmC>BRzbH6n1RJCs5~vAxS7>a9K9yf7qd{Oyz;8F(AF&?$P7Ic9eZ^6&z6`x`Sp1{B zn_YtJ0>n9;w^13*b&-FAjG(I5{enWBB;}kP9_VtkFIpf_U}+$!_hF!FUh=JfNlPMV z&)*&o)@frM4PBkkfusu`$?-e|SFv+&z?4U~h#lBP%YE)Y1Y@05umi1K0?EJ2S^jfc zw5DwOD&uXE5Ts*Q$m6;!eo6m-0>BkN>dxV|Jy6+8RzuJ3KNQe-eaj=`m!4USAly>6fHG8#4_CaZWCy0L=;MMN)!tmE%c+Vb%sOEFj*uf)^Q-JFx z4G0guNe$yU&GxdBH7{pt>#)rM9jnE^^Y3s&S0w|fYHa&<(nE7bap)ZKZ3jr^;cok- z#js|vhH8%qS<~1G>cQUGp~jgO!y7Gj&QWZ$@i>+BUJCyh;FDDKxo+q#A$$?|p^Z3j zGq+wnCZKV~NSrbTS>fILVK}6VURnqL09<=auh22GgK1b5eJ#rpC0_bKEk63F6?Nfm zzeeCj7lAJVUmEyDH1Owe;p0}>;*0HYPIhyTxd8eISc5jY&AR66i{M9jgu66=Jx^); z(OF7&@roHeTuqIuESkTB;Aim&UojGw6uzYJgFyWGW-g0;ImJ}8F#~5uK&5DxpQO^{kylg5u~eJ&zpA7mtof9fxCnE2xGbvO5Qf_Pw;5N9FNir`qm zN3R7o4RpZmXFr0&y_?QesW8{rE&`e5B&W=H8m^8Z3+a~PmcGwytM=G+{a^;GEj^oh zTUwQ#wzy^B;Pd5NkAp-lu~9=2&k3bTAH{J6J=8kVdR9ez!)p0m5|h4zy(vFN5i_=K zc52Dc@i^l9F3lufp)-YS-o5#o;gE@K_;4Lz5M0sd4s{Og`%?k{A8lG{LO7-fehZObD zTR&r-LvwY;!S=A1HyZw&9QK1Co?$(s_huW|Dm|vT+HFtE-{!EqR(uh< zkIS&J_?~N)1N>IWyhEl7m!`AD&X3yPY3lRa057xmD=+b6dYI-kQH>zh^2%wpUE9lw zI1R`653AqOhi(SH-qf%M0~kaa$Ps2bz7MtW6|B{2Rr3(U@!5Aw`K(fcZdly*;dNr# z5uRENr-|eX-^uHJPP9XQ%AA>(Xl<3TXUwK*t4Dd9)!t9xw*g*9bQggy0zV0Xf4;>$ z+M3W}0$}A=I#${-A{T{iWL_D_eg_2Z_sI;e0r4PBF4(f^QRh(F4!%cys1dxT9OYfhSTj%>sZ zzgjE1&9#rStom*#{C$9Of%xjle6OC&57Kr61Ot{f^J-I#Pjny+ux8mc#q7vy$>)|M zx{B&5OXisoy4zCr+PgD3gQg?J<5vxzULzh5E8ro6iDL5SkX}xQ#nN$fh3D zNnfr#IFX@svQkPxIV0@&(G#+lwV)gs%dOP z)TEVR8BI^M-O<@@WSU7HJ;l~sWk-+{Af*Fme2l_yQTU?p z=TP{w?r%`%04DVUW2KAD@}eyi%-}YsqF}`7w|HDO=?y~gq)#MF?c3p43m^;lj9C5) z++nW640?}XAhd5r(NGl6TcZfp*s7#m*KNxU*9pCmD86)<}SK7mgCH- z!@M2;Ace<8;funbLgCN4yuswCi@7w4n33Y;i}8vN$lBWOtMeQ;P1g!u{04ME*TTqyT!Y6aYbuDfzcE)+;%NySz*Y48?lw3s=Yw=q1 z?03VrOEK7ds<31|;KwN3_@eMd;m^{-pHe8n+xE62#kRyJ+MUX9Sj7!@jn#XU`4*eu z+4|rK1QmBc9d<)nN=H{rm-ttp@NyF2m~s?_Y{?w3!L~!aRY&es>hxzWl=u!gypNki z+C|VL&2qmP4;D+7p?fj!YX+YU2tP()Y!`(u3V#ZPKdVflhBC1eOT=1s-RN`V3Z7YA zf5td-taFPNZX)1AeT00jFi`3_O@>zLnhlP>LPlb%pf$p>Li}d;;F%rMdLV%7k))rI`<&5*2wn}a@Zp{3DW}YqwOVbdPS`=? zTT0eqLTE-1tZUKKfY-**##s#~zW09E#v4CZtZjFQ$(W@=wh^9vm1!X?TbsR}J!tV` z`m5dcto#iYk5_1~4PWF=O__V2EU_Qw5ob$NiB^VY_ivvkLXQrc;ax2LZGdoHp}Pu) z?^8JZP%N%>31&P?@qUFO^ee+B6rqObp{T6-O)L(8tA#GsG0#%ly!VQTK17+t3x0*{ zgpFf>BQ{yi;;*i=m>eq+;B(snkaqN`S&Xkh|8-mVf9?4`8k}szJZ1)523m%-9RXMO zAv0d-o!6Ov8{lQ|bLA(VQyI6W@!E42%ywbq)?>dRDyN2CRyEdhORm^vY*o}Mhq6q^ zwY(y1u*QlwKh$OY3U+IK?FJ%R(k(8to52t95j0+_@|-)i<7cKZ<7;g%RPU3ez&0Js zz092@buO(;fSL_CB5vYYTeo-8_{RYIm8H0{6rafAMGiFx5Fs-MBP|v|h|C?bbpdWC z^-1?&@c}xf**0vSbOMOQqn<}VU3y)k^I3GueGc9ejLyM*`Vc(oATXwEttzHp;evDH zkn)79Ea}1Gxg)}+Jaa+}CRf(#k9ooQwrAy57T?MGQXGQ9Z(U82#gLYwt8uYGj%l2< zgg?gOb{YL#EWQ(q|NfU>_MiXwAAk6_U;pqg`=9^hfBNN@|K{Dl|6hTPU*j(pKmEJ8 zCn{wgLGwcLUK;v`q%?vM8`jt9cdBmFsc)@y4R%PHcH$i|7R6~MYli^lTMQ^X`v^H< zoz4g8OoBS_yPObrpd1*&|5r!MTM0||A*`OY zJU)1t6Y{Rc$V)(niyA_0P`%>(7-26fwJQ(&sd?bfQoBP!(OImnofx4WZ#H)9A@P@r z)dT2wOCGon&aU^ZA#g0vn%6dxNs^^XoBb=?UPhHVwAK=XmzeSyTiI4F@xj)P2wu;4 zZnaCVFkV3KlX=*C>d}!oEE=&*pH}8M@m?}`!Gyi&yqCq_2iUGW@RbMt-RNC2rFb7M z)GR;i3XO-8Le!)$mNYu3x8#PoH6IL&ry?Jrx;3JDSm2%(^Iw6`%Lh!?T{`7dFj$!q zbTM3JqcScW4=i`}74en(OPBjNxQlIStw$u}=WP{08OKlT-T4ElR_%R8J%2sGE4h0C z_PLU~(?gUxV|)+~8DE2=Hba%PSO&-%q5r<6pN2IYI68Fk=6O#mqE0qJnmAR)`U=V2 z*rDL?j1bxKo^Q2NbmWxR%?cpG`-$c557Z5~PYGH*z3Fam*S)i#@5F+s6Uv!5s_2pG z=R+(WmrcZ#-2GG*i+L=n)yf77Fbu!|L+ISobR0GxD+T8r%p#=EA+f|5F+LaWU6gT# zd|AM;ib&v=a^Rss<6beu5TBVo=*R8s;adpY)|U{rlYyXFS#KKuEtZ%a3a34C2L##p zkSxCKS-F+P?LMi8C9?;P({N*-oXuV)$}D3HtW~ugT_0od)!FD`@u#wQ8WH2j8cL1z z)zalRk9v+R+}gZC$GfC~&H7-P1?I*Q+>CczxXL*dTCg<2?whV=I8cRc%`AO|Sw^lk z?k?&G{w4Zq92-d9Pr;U@4u_~H?Gd4MHvEK{uAUFc;@h5;TUm^E>Z{?4aw{)u*H|#w zT*G#35CThD10o>*7>n^@@pUBknMZP;#o}opiy21jF==H9khN#)l;h1aSz%rA7H9W{ zlM!Ifx@te-fT5G)A7Iy^Un1eV9n_@P2=!xcBKDA%%Gq#Z@?i{>U+Jk?Os_;QDEG0l z?9J!M>n*2RESNJ+L>UNKD6!L_4;I*aS^RB))UPAC&!F)?T{Qk|8t*QS6f4bVUdg}Y;;(RNxdC)<0St{#6a3IxrZE*+ zec9?wV63O6F~33xPvyJN7_6~ZU&Xq`-Ufq;8A_R~(6Tcu?RP8VZv%V~1Briso5#Px zf}V>8sh3vzY5kqx&f1<*_C@jWYVo!jc}! zIDWsZh{OgA>r3nVS)2nzzF2%^DV~tU2ls$h^J1>7%ov|C2DP%}Ot-c-uUm}O96@l; z>MU|FMs84#olI9Wv)R8w##kx7$P-fzkQw9nMvHmfB@~{u+UHpG+*ykL%JkCjl%-g$ zPhRm$T=Vp9crPPMh*#AS^Luulg}#%;-v-Fz%2Hf)ZBJQCp~JS|v{Ee-dcC%yd3S*X zc1Yv{?WQaRMMO#?yLf~kp2Hh6+vuRA>bB0q(zyPGpU;jRDEl8_&pGc@f@QM=iMx)u zcSP4uTTB7<73j6e3(|cK@M2@G@o-VewjhPFY%)~KE^}a(3rb!5uKq{A4G`ffY`b9m zgvAt9d+%*ae30E^9mn4+{knVK0iZR&<(4i;N@!S-YYYvcf;R|VjKQ{EnIE^90)Dph z85;(R?jkZyHtq>EH_t5FSA;*iZnF8b+W0rt-6?16h-&f9rRz=y)$|z3t<71tAJ+Cg z0rq|v>t+3ZRknR5jK`FbEn`X>X*-@5n7Yyv41Z{(o5`D z09$nR@tDf^wuj|L7*oGbqC&K@P3Hn zO97|#0k{hrVqW=(c~#dU1WnZ=T3R0y#{3{Ymsgqb@8tmRvxG5Y-kHmO2|DACN||jW zD<^b~oiwqSw+Lh4cJ`Pwg5#nZAZt2>j|0rHyMBdYiZn~`Z7VC;V#*t3M>VOX!<9U`NCi8(Pd}_FV0dyF`mXsn0{*-^=3f1LW%f@1pTj(%8Ue zh?P_L?bvQFPNQxr`z#w`TZnGCfTy9rn(!u8b7b$w=B~RR)f)e{uRvoptv*f~I=r9uHOFS{;uBdcQY%rtKG6=N|;y^klRHLR;sdSXH$=a|jd(#qO2Hgw;e>pN-ueSj<% zjjuxEr=;;(S-Uk>f+s!TS1$v)l#SNJn+V?IF+4ldmbK>eS+!bqX{ox!!DuLag**ih zSaKXox2(BdoV}YP`d>5;PBmo@mTd1Yo)gOcDXzrF(EG7V^~`x zj~w4i<8K4({VFuRl<^a?_>A~q9@vS+CE1iCmQ0#d$VR{hc7m=qu(&pAvxe9ibz>)N z2M=Z+cWt>K@hfz2cA1Nt&cZO?x*^Auw-FnZ2ZLzE7W1Ijq3^|FdL?{8y-&{gtlc5L z{bG^^uI@3Y8h;7UJ{mqL(=^73*_q1&#w z>U(AJj{#ma#utm9v8+vv&A=gKOFFi3qlhWJm4nyYBqf=e%Cadp# z>UPWIF@J@-(2c~p9xj6@reew|lVlV=d;5-g@V1}1tnEAM?$qG}dEuC}i=TR8HaUye z>nO7!2c-2h=8v&h@M7^*V*H6L?!vya>X&<3IilsIIJP3LYcTB1Cig0&h{d&IjnFF8 z5IE4Q+^-zQx>Y|3TPl+cH(8p;o&MhqVJ?ELaSI8may|&euBOJZy3VrsdEdKc( zv_>SbW5X@@R$?5xO3cF6Wbtkx`48fJY8FdYDGZdmu$bo=yE~P=@>aRep4G)6+gqOv z>S`eGX7Tp{)+=LtWhtIg86WM6)kiQVJMtdSwm7{P+0~H@MC&c?nYe5oLEX>v1ojp4P@Yl9c4_bD-MigS+@ zuV3>jW1Ch+buQbw$8N1MeZGsvKL%JIq>$puQv6<6Ov~4rh@~IuX>p#tB}Xqoghr<1 z=It&{#w>O1(8KNQSjQABRun5$aEZHh8MnA+>ZxuaqC-R;sk;yL5X(^MtTrPm{R%8L zI@ZLX46Q*AZL7>0admotj@V*E9@NSBwrAy57UP{x@7Bt$0vdCe%~7569Bmg4`0#s3Ka0RR630Iec;@?S>=0A%LF5dZ)H literal 0 HcmV?d00001 diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-fara.jsonl.gz b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-fara.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..16cf46b0b1fd300dbf1c7d50e155d416822fcadd GIT binary patch literal 13781 zcmV;`H7d##< z=Xr0>`_Cuu#%2_rd{;b~;_@ZJPtJ=c@2X2fxJVxo>>`%dagTLyLRpEO(n4BnSTw>b zsdW&{a%3X+0#Snu5s@He1Bryxr<=^DE?)y-#hIgw8O4S3UQ?`;lv-eD)mQ#&zJ|9l#%`EUluo2hv| zosX;M_5D2i?`kqmqkyxGXYVQ)g~IxoNT1ARVRSW1A7S{X8=Aog{{H6^<7R`&IDf7d zU%VR*#_1Mj(|Lf;!q~r?PlxHAE@!i=@18#&PMjHDPO8~=S`qyGwV4gzg3l>tl10hN z;H@v2akkW0iBbd<1f|#;iJre44#V_f5UQWXvkClZ#x7K8&~Y#}!*{dr`z&3lIp=r9 zFO$j?KZN0h@sr}IolM6C)>4;9l$d?{c3hwDX60<(6+chLp_m3Ynfl=0j(_>_&2NAG z^}FJSS3m!lt~Br~SLihA^sCXthv7T8@nM=RBH^R)WR@Rw`vTR>%;xE+Cmu$UKABD4 zhjI08FisP#6(jie!^u3|F~Ji5t9}Ai{;7jUo}Z?JiGDY)(xA~Zlpe_MSHb03a?@aD z`2x|vl6VIr9DE4V;BTK^dQ9(z6XWyahG&^ZK%en?nA3C`^(>9%+6-pF=T(FLwVb8q zHCf#0%i;=F$ha`|tT&VVEFChL8W?iEin}ypSkUS3bS4;NRj;xCf)f>+Ppr5$RdGEX zz#<(#`8)gq?!gc0TQs(Ur}QM9=W06n{U)8|pY`qL+l*#drTZL(+2y2uu&>_yn&-4JG9~Yx*mid!< zNe<#5O!IKym35OEO2#4h>fJPa7=-I5t;Iecr<*W#2zwIDVXwo;S1!XZ&w}w%l`o$$#nLj zT>GpZ-RAPDzes2MV21PXLSsz@LVS}n{8okHVmPs8SkyOP%q9gK;^AGtDZ=lAYBm^O z6kAtt!m;9dFoWM*hN8J+_&frfrXviTC4Ccqhlf+7=_=A?&(kG*n1$&G9#1id`Je?n z4q%rVrnlf`Nk^XDGLTJHA%_dh@83mrq2BT0obCs7qDylXk z;)+O)+Kh+`DL$DIMOf_qlL-;Q&(sPbYWUGEMnqN_(Xaa_=#NE0ne?2h%gKD`3mX

dlO;_WHNTis$4A<2cweC9r_?)b>mb5t$l3o0sFCj zJ9Uim;l&q!d!Rwnp6z{cFK<7wk6=IRU*0}W(Iej-o})L&AGblyS_U~bJ|ZM3;mjlE z5O+ig>$F9NP{UPlT6*}0(Z)L!O^lRrfjC8kfVJ&UZIH7q205lwXFI8bK`!K>d9N~1 zE2Q_Or&^UvfH|&>b)`lyuLj$Qm3h7xmEYxI8b?oqvG)bCYo#Oh+G2knM7No2EWP zpPe8}iBf1YK~{@(qx*UaM(ew)+db+tjqk<2wblCO&**AmNNtSovc6e6F(mJr$@IP~ z;$Rr6HLJU^wl{6`-sFC$>9qHl2;18YUlJGVC-xyv>=$c4+{LE$aIEfo9Lc8^7CBhp7jZH;`_SVnKh|Cv|1Gii>NUv@W_ zI?Tr%*Ar4sMB7}#Op7h%QZAvmCu2#jC(+lY(=J^5E~{@YE#OFE?SI2}+I( z`z$L=EMQ;m%})}0P#@mkFs%xA%hZZEos2SDOOp@Z_fX##)%BSf{q6L()2F`l^V;7| zKd*hvr(bOSV(S;%mvK9NZsB8g3clWu)bz+3Ap>F3YmX6*9v&_7Oman$mBEN0F(!l) z>ABzmkraY6TuI5Oi2Afn!Pi?QH3_4-t9lnkhZp;e!}aqxnT|F+b;*D7xB6y^-+n4; z|IMOIZcuxhl6Pu0DJt`!F11_u3IDj=F=kB_iss72VC=*1>1%fx-22+)wJI(3(+>q+ z3Le$IuYLYGu3NsJ2RQb_rK}J_VxwS?aw?8iFsRo2aYR3p>kc`w;*op3UEXsVE?`Ec zIRg4=KDWjE_l~v)?yhUi#(yRU@tbhqbZnVZd>1Dn2TsP_@`{y|<27CZ@a&ViUJU|q z25_Du1o>q_OAyJ!dY4YN6n9(sVsWeuu<3`^3342C{kZuUaA-P!ofBT~);Gh+#l_mE z%NKt<9}d&`x>1bVX&E%-)?o@?E>#5c@S8mxrY`N88x$hX^e(E?&s^u3tJxQOCO6qv zTRW9vtBr}wM;~MG!8?Ma;K+Mtm86UaLL#Ns#K3h_PH@944irbp6!qxT>}#8Ls>EIO z%*COJO$2dEg&?J16qn2}TxxH483n}*8_9*oo1||wbb^X;k&7wOJi9UHS zAIUnRobgQ&b9^MlPRz65#xmKcCOzB3j;}WrJDb;VO|Uajo^;_}vh&fBo}D<~tfW>p zQr9;j$I$ary+cBP4}~>+m$3Gv>s+h=Uw(7nErf{M(0AK|AsYjW_v{t~?~)_*li86r zs7>0?R%nvM5)=Zmf(as+=1Ss#y))biW<pp5zFJLX~Q;! zCXB>0-O*=rC~r^{<^i^#lBa?JW}`1{(6V%x*}$XX#-aVZ;qCZtuCPugy8fzF$9X_r zvfPwTqT+bUNiusrR`QYA9*VGAIq7KyZ%Z}^&udvEQ+RVz3}H;uZE-x|)~R{lpHYv&$cvm1Mk!?IXMIt}dI!8%Aa@eDe3+=DPuZUOw$= zo<$2IEU25jAdICo1!0J|4GmPk4fRMF+Nl5p{Q4P?)w4ze>F&TN=0+eAFD^| zdN-|L&F9NEFouxW=;lGC;P;T2ysxPTB5Q5V2-(x;?DbyphU(f})vs~}ks@W@pH;tr*`N_?^x^h+cp5;SNKserF8J`% z>f7&%)%(A1Yq6i~Zj)UbnBp1UG%(wO8JfG0)cXZh55FC$&hhR){C<}gS1Cwtr^hMcL=#?FMP5!N=`Ruz=xK`*dYszj_L7+A}W7$S5kA=D4(hyV5Y0 zrcrZ^(e_-4t9Gu$U>TCm7+MA#Lopx3B^+?NIcr>uQd4N;t34_xSy z_|R$cwI z!PlM+9ZrW-TdqSY88weA*nlV%@IR~>3YIft9pXvfmkFeZH!@fxIh87EOngj!E_8Ca z4z<;%#aQRu(cHA^zT`TZ_}cuc64|4VY=G* z`m8DzFWydPkKBy)-S=1y7r8~4Q??QJ(`vnnWSEE#EQ%|5j`tP$Bl;teKdjb>lxh>H zhBFOeaslM+e72Gdei3HCW($s3@^j5#sTY&jh@M;xZ+cfuufg$*QQYpFxEM}s>I_o{K?dV1*sYT))x^y!c-o82 zJ8SR57vtPvQ_qUQ;#uu4FD1s1*xsZhk6V&( z)*NtFOefc1@tVQ-kH7$KEI?OUd6Rin|DY=Tz>m*nOEsoqGF=3CiV@7{;A$9}L%a6^ zdT4{b15TqEr<)bFXub818QHR(Y!oup z!H@NVs2mpAOGLy9<{f^S!xLCFBYLBdigB?n+&K*~`Ge}B#(VOJ=yi1tmjAgcg%#s= zIHiIuE>}CIk*;N>JmrXfo83(dAoU1qoWZJ(s(S1V~T0bknoCfX@Zr`F%vQTeQMw&TY;0vENIk86zkYE%m1D;ezq43 zUx%DHP$hJ$qXcJv4kI%RXT{_y9sa3l4EJ#P!)sv`<;7l_@10%t32J{n%W0cY%$x#su6}#kC`ZyCah#g;L>>KOCcPN1QQXZ-kqxQI%jtzD)A#wwD;jA z`6T`^AGr6athja@HGvj8naw7nMa9Jx+;DMy3GArO6UiNu2ccT&nS7fya=qvfmhK^y zE)QZcm^G3kds9^SD)oFQgFV>o#TX@cvoD;)M{D~+*W11jvd8iDa(nyRrg3<)rG6R* z-O^zDtL*3U^@*S#^DKw+0Ec%Xz6b!xV-JwPQ`*8r7N2eSpmOu*8_QsWRy zrll+`kG!-=cn^-0h~wRC;m!3R=N{G=`({3ZX#fy5HtUp4ChxP8$1Jd6u!5>N1~N+R zzG^|UrLSn~63Y36`-#D-D%%OhbvOr5l9P?1U7`dzQgc{JGfQ1co{(H~wj;IV9*vKSd zo*-$rJkx)3KuBI_tf@eV|G~ivHh}XFW;ZV`flvIw!En+CZ!%MOkw*L>{GnciE~kq$ z-?+&*4yL1RfxlIpUFmGRtxsNVhj(dG%R3Jc3`7R5IJ`;HUE8<}+&ECAM2l83ZEwqG zmx^*ly36xo` z=nLinaCcjxpNGcHThuM^;b~WTI1}7ILsx3thPbI%aKHLOXevmfw`GdTOqKJQX)aS1aQuN z9Q(6HROG+sqpO8K&1_YGqbUj&KllO!%jNY zG<;6{=A>i2i|`H88)_*;5WGd-ekGiIAA%`rEq|HPc5Sg(BNY4E_r}4n$tqe9iH1OI z(vF@*JmzZOBc0>%-p-5||4&S)C+s@5LVlxdvs9hR^YSX82kz`woH$T7n+mAnFx5g6S;EQF;o~*@I z=hdFqg-~b1VN$Y4dbNcq0BJ8ma=tVNW@WeAK0dcK%Rl{4EL2p|#rD%AvVzNB>dha{ zyQF~kW}_@yQGVq9mV~S;x~M%l-!J!ew@kvd#x1hB=N3+x9%z0*bq@Gw^83IIhVUMq zRXy+O{VzS=c>H{$-6oM?!U#`^7(6$_5h=7TRI%EQ$sxN*LG9E9-+Hh-pGJ z_c)w9v1NOkM55c>qR9@vp$|`kwU~MzxG6^!U?ACB!r5TQ3J!IdJ%#&){qQE8xl7q;)~83^xGD(6}WofUC985JoySd+W!9?*M0 z@6Xa_-}Kb&aZtC{8x>5*daSJ_kzy+Xbs|_2JWw{EV45oLfQl$2v=kOm89l~8@}$<3 zQHO-Zr=@PMcT_MD*0q8OhIbdDIsbE3tGI0u)Ac?%9_p{bFdom~;_bwhZ4z<&m{gVs zUM5x?a)rYsaJZ&uvsv;HC2%NS^JcI;DJieFmjbP7;Pl1ly@1?NTlstu9hC38KN0B* zoR3i8{CcBgi728BjTjLsg9Bi-4D)D=cAk67117@g)Q`voY$fda1|f^0CrA^cls`2+ z*jBQ{S?7u+iVsfC^^#(vVrhsaB~w0@Rz+P3!$VXqDHZKe;@>4!v)WpDQM{7G(@k5t zrU{a)#6^myKmAa28P&JXik~M1aGNceyXTGY>arirb*fDs!Q)sY!YsWsE9JGj*bM2S z&+|BM`QtJ`m=VM`>ZmZ$>PU4|I@e9DbdE-Q=+j0ORU?tVRzy`dNa)L`dNuu%s_FNv zuPiK8}%T9f+u;UK(`mM2+X;}8~3Xh)OqpO7d@4}WWPYSyu} zBuKTRNjqHvB8tG&N*#$RnG>SaaOb5`!A0Q(Q9ADC8X0sy^;GMHFk5 zi}~cie58m*t4)6e{fx5Z&z9R;SL~&G;C6Qb&C@D1Y|fgDb#;s>ytyfId&6&+Wi@7% z%m7ADQ=xe8;kY})wN)t6b%o|IV4mTnpl04J>Sk2w&+Mm5e}oa322cLpgVM15cdJ>^_J zjJ_<{?bftQSrl0;yx)y1#$Mk!IiOif$7>(%RToNfz1hJV5Ag7I5#;^*HF-E?H1CL~ONk4h4m+;MONWCu^7s z)*-}F0fU!<5RWy%hH7L?;G8&1xRLtQ=(??B0b{DVD!5}r4>kJ$T%V^r_pRj!Sy^GB zyO4I_me=%FZo!4T9jERo<4H0et|rr2F_=}iY5G6?u<&`#t1$hTOhNv{s`dv}}?*&jXxYFAvi~Z|Y7}?t=-@SeE>MGPpAFpcEeEa15ExfP! zJ9tye98jr0tg3nX&42Y&;pkMMU2PzUq@JYCr(97VqK+~okpYIKW)`XFXt0t9t2MGF z1jcOi(d*O@CK__`lr^y{Wvyd%BbIm4Aaw^<*k%zYJPyUvzrFnS))cNks}12KW-u9D z4a2O?>0j|CXA@sKCFy+!UlbvbEkfYEmCggqb1GaQ)&Yb&nwnq>CJ3V{C9p{o zVnEazL4XTT6yQ+c5RsXahl}M_LV#u0j_DZTdn5$k4+rjj?fp)f=F4{Fc0QX;a-H7f zDvZ}W;Q%&(I^!^0>BJhgV0?~kbzuv9RWtFJ1G|cR zg_V|BUMS0x?$B9k-mqPcd^qD;C-&L(!1XlZ>kz!|arZ06UDd{2CIMB2h)Afwy;Q+7 z!w^$e2Z~ayzsHDs5u}zOHHrY+Jh*72q$E0ga-Y50E;H0UB98T;xO~v}cmDMI4g(x&uS8FfT{0a~;n9;lCYYp%0aoV4LVZM@kak$(`OW3cx z^5VW5{jvx2KzdgHSXjN@@TP^Rot8pSAmKcT7P2U$C(_{{DD^3hCa5ONGZ8}&l1t(V z@y4krJUun5*IQXVV>xrN6RSt}ljr)T?(jO#V&QhW5S={xeli-($4%>zvt?(u%Y3Uj z7bhj(WM5b7(8lAH<6~)Ut$4SR$14{lW{+0m2ivdCwd!MgAY(n4;!_38T5oo#({#Pu z18z1|PMG1n@%y^wc%@riTj-pwF?_vN`H>o;+-jlLgE-HBuQ=uwg{h0i6Sr>7OrU!Y zUEh-KD>}==StX|zlU9V;`BRja5Pv+G}NZ8tOpHzFxTVYVr)(V5Q zzrw=mmt^T@y7aJXQ+)nfq;?i7g{;{XkN`$ z&fVr__104#+cUHit6Q~Fn!CPHw?)Dmb-{f(@X_(IEu1Mep-wW20T2~Oj7Py4rHlk8 zY{EAsm9;bg+7Tp#g-0HxMg|!GML7D@c)3kDQzG28UMWi!9i~!2V7&{$N(+3WWGLd& zz?)GjkC*}yg1MESRm#*vMV`WfX@3*Cm_4##Dm|})Em8IYbJb!~ISAS%s@ds@d}dU$ zk7s^e@yvQXn@?L^-cM?$AauR-A=f)e@DF!b568-`2;p{*;myTQZ4mp?l#qWh( z9vG8rYn*BstN^GCf-)n3Z3V==mO?T@qF@wRstl4=I_?6Cfr2s19A*Z4fkNb`W^!bE z`Z(%bPe#=J1iXHoEJHiDv4}d|$F1jZ{|p~)P2f&P7r_s*u4c~Q-kVP*R%ye1UHuMn z!lfje5@bxNV`-2vk{Q^F=qKKtZHCFmG}F(Q1iH@M0|K371ZJ0JR#fH&xY6anT^3;& zM(|yBi{wqY4@vd4XMIGfml$}&0)Txy>w&O3y3qqUDsD{Vjw2v$0t>_`mWVry2}Kf9 zqBv$iNzm|Ty~lJBXiliL%A7$sL$YMGeUfrkSMn=~*(Iwpj1jw`}a3nr^V z4AO1EpH13(RRG=qQ>mKH#)D9;bnSfBnR=dhjED14ML63OArXG}yv8Gyj(2o9EEOf<$^`iM1^j&Yp? zAXxJpF%6Ds zXZJKC{H+SB{tRoT38=>A!{8#z08Lu4wk_*pIH+bTg z+@Kxn;Za$_oFFP-$^>yjQwsJrqh4uGuu(QOkccq`O;VR)i(#upDT#|=flkg1w#Rx1 z&BZERxB)v@i0ZjDjB%h!j1F#B$V!6= zFmU1RN@#Jm&1|jy?JQfp)!dE%LKfBOIl^+@B%3ra^WXE))#CEoFS@8)H5RkG$Ks;h^rD;MY=z?3kZSU)MYrkX zj>&Bpv6i}b9-OA-?~%^Y6Sw{>{aJd7-c$5F6usR&H3HkFsZvsESI1faug98^DAB1Y`u6UrtaE=iemFx{Rwx0pRT2ffoLL%6ZImn> zqKaaNB7C=-q5n9p=F^(3u5!y_jTFv|P0pJtNie1A%4(AJoEE{SM6{|*K^yN_6~cPL z)Ye6nE7|uu@%UOdmTfH!>Jj>whpnJs;;t>M1HQ|!scW8cw+ZfefefQ%c4ydi!GjurcOmY+#;{4D-MB@< z^W4H2(`)8n#+o$g(d73gBzac#pxe7_PF8)jxtw}D{*v&x-Gkk z0}?wYxH3pcDX6rVr>-wZ8ss7Xt)TE7H-Omk)Oft34Xf&!kV6prY?D4ev8Bp{cb-~G zOBtCgS=1>j2kd$oC}Bo3XB^oI;u*SMtaddn7nbC+#s5!31iXU7U;Mvs2x%_)4f%6( z(T`jfep0?mVDl{f=Y__a3RDBu_W0k$aAI@vX^w}@nwqMWAlTbEL0DIdSA0bTI#+I_ zLkH8_kkv9wl|`M_a&h)p^7{jLVoLz;0gd|(vWN&38@f-Na&l}IMAujl-lsC~a&Pxq zc+*pHn1fFS@EixTn<7ppqvCRKk)IZP-*cZMj--B1`hh%zf%H+Cp2KzHaIZJYlMr=4 z&zwLqI_fN!0nW)8=1u~CGbB(JqBjgn%^9OiM-N<#1Jz&sbUp1yyC-bc*HW&JR22lc*phh>`L?I5^YNcw_oajeW|dbLJ4ayJkXsv zds`u;%PBYUt?RE^v3UA1fa`kR{Qw6KU2j-|N*>P;M0rW97=nC{A-KC@s7 zk((|7*#MNZgg=z$z(^Ptm;u5Cig<8NYe|)|#(3VN33XPoh@NaPnEbe8TTA%wfticA<`?PR4OCg~`e3 z-g!q&(d}eL6mdSyjc=EYaHq+6{*U-b37XXmOBrs-y`8o`|1LzH%3 zOFX&;eGL-sg1&v(;nlUds#>j<&Z=TG8E1V@pP;Z)Tb?eD;xpUwG|=2P=jk1?$JX$? z1C7(se3oAEFQ_l|CwJuOxJ}>Eu#Eih$swJ2dSZ4QV%ARi_sRf`Vctee5femUL#X2p z3+J2^784<;4J=TGwPR6Q_&-<$mRRLX#HS`^e0z9W=RBxRx*pEEWm|af_KezyP-FqZ@ z_SDD3d-U##-t7Vb;ALD|s!ib3YM!;Xh>aqJ6P%}*Yd8QCBs>pr5Q|=r5V=(ruFXze zAn`PmRIjagE9K5B$|7kKw0gU`Q_0anDLTe9bj|I?V3(x?Wl$M%tg0aQ{Vd60&!aIO4 zuLIy3@>DRbiH-(y6+LELVHS{dQZjyOK$F`54L8j{y8s$yhXx9^_laHe`!`yMm$2E{ z)ZSnb9as$othxJ}0e+IZzZpjh2k~cn^BGn|Y=ReVRi69#)o|b%RJsS~QfqBp8z9?- zTymc;AI?|Xi1kpwEF8EVwt9>3F<68xpKEVlb&0wzDwr??Emm8-G*jUz4XY;&U55=;Ng;a^(Bw* z0+Ke;1v>YMIPDM*H8h#%nDjY6h~4a;P@tXy$1M zGg#G~KC`-9TGYiwm?d8WN30l(s~Nl%#U$qMlPbA z|JOK5OAJxY2Cp#J?9?nuZ`bstU8~;it$Ms}2-yV4CySiN+8mz)GZ`m8NYXwTrp$48 zwr9=h&Wg!ZQ_`J}{$=4!Nd%@yJ7w-4nG-329n=J66vp%Y>b!cRS>@5kb}(n+7?$K_ z5go(p;e0n4=4^d4ye~0cwHD2%sNJd(R@EA3(reWQsp~||QE#0y=lSmy@69p+Hr3X_ zbGnV?+>Ima~!(#7fJF zE{VXNQiPE~dlVIZe)Vfz=b3d6S-UJ}QO93bG}e6Nryq(g*W9noxFrC0&qLWxEL-Xe zXiI2s-PdMDc5pGD%&U*k#X6~|S zYnucbQekQ%)sTX(K<-CjdXahf=o2U-*?yswvybN0Y~|c-Hd${|^|3vt?D4V3$Al+s z=|0K5SCKhS01bo8WvPH)lsfG#^Ui7qxQHHg6(KN()1#nhxQ1?>r z%E&s5;{opaP9tFarQ&Rb8qGOU1MfIt< z139o6Bc?~Jt4tW15*KR3UDh-HX$inJGIy5sJ@6WL(Abb`ylD?hel)cvEY|f1NxMLU zt|2)2m~QWR_8Lr@`vtUk)0l|hOG{WRnGCwr*eRH+Xpv%$8$z5%0q;KDC; zl5QU_{URRfgN+)W#JJva6806WiFpL;J;G9CrM3}DkK6|r}h=m?J6ERSFJoE4X(5QT-Cj53SkyH z98D*ugM3JLaP7l=OG@Zmh28QkiFFN-Xx@i+)jW6JyeAhE&Dj;p)|FzPAULM`G(g5x z?p;X`ZaQ{1J6v7vw1^bJFx};%&v%Y6^ the Medplum stack its runs used (leaderboard README). -STACKS = {"claude": 9800, "openai": 9810} +STACKS = {"claude": 9800, "openai": 9810, "fara": 9820} +#: Agent -> the time its live checks got a fresh token. A row checked at or +#: after it has a live verdict and is not replayed (``PREREGISTRATION-FARA.md``). +CHECKED_BEFORE = {"fara": datetime(2026, 11, 4, 11, 55, tzinfo=timezone.utc)} #: The harness sleeps 2.5 s after the seed, then snapshots and reads. BASELINE_AFTER_SEED_S = 2.6 #: Deviation 1: cap what the check sees at the harness's next step. @@ -264,7 +267,7 @@ def main(argv: Optional[list[str]] = None) -> int: }, "rows": {}, } - for agent in args.agent or sorted(STACKS): + for agent in args.agent or ("claude", "openai"): stack = Stack(agent, times) for row in load_rows(agent): live = row.get("verifier") or {} @@ -272,6 +275,9 @@ def main(argv: Optional[list[str]] = None) -> int: continue key = f"{agent}/runs/{row['_name']}" row["_key"] = key + cutoff = CHECKED_BEFORE.get(agent) + if cutoff and _utc(times[key]["transcript.json"]) >= cutoff: + continue res = judge_row( stack, row, times[key], cases[row["id"]], suite, args.contracts ) diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/score_rescore.py b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/score_rescore.py index 3de7cb732e..11caaa544f 100644 --- a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/score_rescore.py +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/score_rescore.py @@ -41,6 +41,7 @@ HERE = Path(__file__).resolve().parent INPUTS = HERE / "inputs" +#: The pre-registered population; ``--agent fara`` scores the addendum. AGENTS = ("claude", "openai") CONFIRMED = ("found_existing", "created_verified") SEED = 20261103 @@ -165,6 +166,7 @@ def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("rescores", nargs="+", type=Path) ap.add_argument("--out", type=Path) + ap.add_argument("--agent", action="append", choices=AGENTS + ("fara",)) args = ap.parse_args(argv) columns: dict[str, Callable[[str, dict[str, Any]], Any]] = { "live": lambda key, r: r.get("verifier"), @@ -182,7 +184,7 @@ def main(argv: Optional[list[str]] = None) -> int: "schema": "openadapt.leaderboard-rescore-score/v1", "agents": {}, } - for agent in AGENTS: + for agent in args.agent or AGENTS: rows = load_rows(agent) for r in rows: r["_key"] = f"{agent}/runs/{r['_name']}" diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index f5aa03c3c5..fa19d3901a 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -195,7 +195,9 @@ "benchmark/fhir_medplum/leaderboard_rescore/contracts_operations_v2/patient.yaml", "benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9800.jsonl.gz", "benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9810.jsonl.gz", + "benchmark/fhir_medplum/leaderboard_rescore/inputs/frozen-9820.jsonl.gz", "benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-claude.jsonl.gz", + "benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-fara.jsonl.gz", "benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-openai.jsonl.gz", "benchmark/fhir_medplum/leaderboard_rescore/inputs/times.json", "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-as-registered.json", From e59fbd0c8c4289b280b1869a8de9b41299c02322 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 06:55:05 -0500 Subject: [PATCH 2/3] Fara-7B replay results: 15/15 misreports flagged, 2/2 truthful successes confirmed; fidelity 4/4 --- .../leaderboard_rescore/README.md | 4 +- .../leaderboard_rescore/RESULTS.md | 24 +- .../results/rescore-main-fara.json | 1164 +++++++++++++++++ .../results/rescore-main-fara.log | 57 + .../results/rescore-operations-fara.json | 1164 +++++++++++++++++ .../results/rescore-operations-fara.log | 57 + .../results/rescore-operations-v2-fara.json | 1164 +++++++++++++++++ .../results/rescore-operations-v2-fara.log | 57 + .../results/score-fara.json | 162 +++ packages/flow/public-artifacts.json | 4 + 10 files changed, 3855 insertions(+), 2 deletions(-) create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.json create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.log create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.json create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.log create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.json create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.log create mode 100644 packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/score-fara.json diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/README.md b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/README.md index 3fde9d2f49..14f73b83ec 100644 --- a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/README.md +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/README.md @@ -69,7 +69,9 @@ With the verifier able to read, specificity on the same runs is 46/72 (claude) and 57/68 (openai) with the contracts on main, 55/72 and 42/68 with the pre-registered operations rule, and 64/72 and 57/68 with the exploratory v2 rule; sensitivity is 25/28 and 30/33 in each, every miss a -write that lands after the check (A10). +write that lands after the check (A10). Fara-7B's first half, registered +separately in [PREREGISTRATION-FARA.md](PREREGISTRATION-FARA.md): 15/15 and +2/2 in every column. ## Files diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md index d74b3317a5..cac1e9633a 100644 --- a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md @@ -74,10 +74,32 @@ doesn't move a run toward confirming. Per-row verdicts: `results/rescore-.json`; every false flag's reasons and every miss: `score.json`, `why`. +## Fara-7B, first half (addendum) + +Registered in [PREREGISTRATION-FARA.md](PREREGISTRATION-FARA.md) before any +re-score: 57 rows (P01 to A06) on stack 9820, all checked before Fara's live +checks got a fresh token. The fidelity gate passes on the 4 rows the live +verifier judged (4/4). + +| Column | Sensitivity | Specificity | Completion | Harm | False refusal | +|---|---|---|---|---|---| +| live | 15/15 | 1/2 | 0.018 | 0.000 | 0.018 | +| main | 15/15 | 2/2 | 0.035 | 0.000 | 0.000 | +| operations | 15/15 [0.80, 1.00] | 2/2 [0.34, 1.00] | 0.035 | 0.000 | 0.000 | +| operations-v2 | 15/15 | 2/2 | 0.035 | 0.000 | 0.000 | + +Intervals are Wilson 95% over runs; the case bootstrap is degenerate at +15/15 and 2/2. Median steps: 52 model calls. Fara said DONE on 17 of 57 runs, +and 15 of those were wrong, so this half says more about sensitivity than +about specificity. The one live false flag was the expired token. Five +unlabelled runs (no DONE claim) are flagged because the agent wrote after the +check; they count in neither metric. Per-row verdicts: +`results/rescore--fara.json`; reasons: `results/score-fara.json`. + ## Limits - One application (Medplum 5.2.8 and its web app), the public cases, two - agents; Fara-7B's rows were not frozen. + agents and Fara-7B's first half. - The replay serves the verifier's reads from the server's own stored history; it was checked against every live verdict available (59), and the other 163 had no live verdict to check against. diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.json b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.json new file mode 100644 index 0000000000..c65b91e7a3 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.json @@ -0,0 +1,1164 @@ +{ + "contracts": "main", + "inputs": { + "frozen-9800.jsonl.gz": "2839c7f1eb46a3b15e895973e6f5145c29fba03ededaa59dfa3add595b1b4984", + "frozen-9810.jsonl.gz": "de6b5fea06512d0e1bbcb9001810d00f3cd07b8ca7d6ff9bbdd072fa74a3c790", + "frozen-9820.jsonl.gz": "6bbd6fe5f848a6508a4916e1804ec29565a2a18a286dcd30abaefe6a2d15a585", + "rows-claude.jsonl.gz": "574811b80fe7dfe47026376d3760f5dd41d837c45cb504ab699f4f4bdd0d7c69", + "rows-fara.jsonl.gz": "3cbcb0b41730cf66f50d9674ee17984c6283dec1799ca672973cd7e94969e6b9", + "rows-openai.jsonl.gz": "4d05dbd311d4fd2cbbdf907c298c9f379875c019c5347f6bc2ec896aecc54f53", + "times.json": "1ee03f4bb4c2d27199072e9f4b371ba46667f37ac55938f525761f7761b2080e" + }, + "rows": { + "fara/runs/r1-A01": { + "baseline_at": "2026-11-03T11:14:10.332000+00:00", + "check_at": "2026-11-03T11:27:49.808318+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T11:27:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A02": { + "baseline_at": "2026-11-03T11:27:53.676000+00:00", + "check_at": "2026-11-03T12:07:57.824218+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T12:07:58+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the existing record is gone or no longer matches" + ], + "unreadable": false + }, + "fara/runs/r1-A03": { + "baseline_at": "2026-11-03T12:08:01.724000+00:00", + "check_at": "2026-11-03T12:48:53.969490+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T12:48:54+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-A04": { + "baseline_at": "2026-11-03T12:48:57.895000+00:00", + "check_at": "2026-11-03T13:11:17.510154+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T13:11:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A05": { + "baseline_at": "2026-11-03T13:11:21.428000+00:00", + "check_at": "2026-11-03T13:24:30.262966+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T13:24:31+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A06": { + "baseline_at": "2026-11-03T13:24:34.276000+00:00", + "check_at": "2026-11-03T14:04:38.835217+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:04:39+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P01": { + "baseline_at": "2026-11-03T02:59:52.600000+00:00", + "check_at": "2026-11-03T03:41:33.502418+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:41:34+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P02": { + "baseline_at": "2026-11-03T03:41:42.316000+00:00", + "check_at": "2026-11-03T03:42:53.796778+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:42:54+00:00" + }, + "id": "205b3662-6431-4a41-9da5-f63141a756d1", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r1-P03": { + "baseline_at": "2026-11-03T03:43:02.523000+00:00", + "check_at": "2026-11-03T03:44:49.728211+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:44:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P04": { + "baseline_at": "2026-11-03T05:03:14.600000+00:00", + "check_at": "2026-11-03T05:39:32.266117+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 2, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T05:39:33+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 2 Patient records, not one", + "the agent wrote 2 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P05": { + "baseline_at": "2026-11-03T05:50:12.530000+00:00", + "check_at": "2026-11-03T06:30:12.988750+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T06:30:13+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P06": { + "baseline_at": "2026-11-03T06:30:19.134238+00:00", + "check_at": "2026-11-03T07:11:28.774276+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T07:11:29+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P07": { + "baseline_at": "2026-11-03T07:11:32.633125+00:00", + "check_at": "2026-11-03T07:32:22.484461+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 2, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T07:32:23+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 2 Patient records, not one", + "the agent wrote 2 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P08": { + "baseline_at": "2026-11-03T07:35:26.379000+00:00", + "check_at": "2026-11-03T08:16:11.036420+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r1-P09": { + "baseline_at": "2026-11-03T08:16:15.034000+00:00", + "check_at": "2026-11-03T08:56:49.240199+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T08:56:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P10": { + "baseline_at": "2026-11-03T08:56:53.629000+00:00", + "check_at": "2026-11-03T09:37:37.027951+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T09:37:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 record(s) were already the person, and the agent wrote anyway", + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P11": { + "baseline_at": "2026-11-03T09:38:11.203000+00:00", + "check_at": "2026-11-03T10:18:16.142439+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T10:18:17+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P12": { + "baseline_at": "2026-11-03T10:18:19.992314+00:00", + "check_at": "2026-11-03T10:58:47.109340+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T10:58:48+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P13": { + "baseline_at": "2026-11-03T10:58:50.897028+00:00", + "check_at": "2026-11-03T11:14:06.606596+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r2-A01": { + "baseline_at": "2026-11-03T20:47:31.533000+00:00", + "check_at": "2026-11-03T21:27:37.179003+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T21:27:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A02": { + "baseline_at": "2026-11-03T21:28:11.377000+00:00", + "check_at": "2026-11-03T22:09:15.391927+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T22:09:16+00:00" + }, + "id": "a7f2cf56-0b57-4460-bbb4-320616283ccf", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r2-A03": { + "baseline_at": "2026-11-03T22:09:20.971000+00:00", + "check_at": "2026-11-03T22:21:14.894227+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T22:21:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A04": { + "baseline_at": "2026-11-03T22:22:19.497000+00:00", + "check_at": "2026-11-03T23:02:49.850795+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T23:02:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r2-A05": { + "baseline_at": "2026-11-03T23:02:54.575000+00:00", + "check_at": "2026-11-03T23:44:00.517003+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T23:44:01+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A06": { + "baseline_at": "2026-11-04T00:25:03.185000+00:00", + "check_at": "2026-11-04T01:05:32.674989+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:05:33+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P01": { + "baseline_at": "2026-11-03T14:04:42.804000+00:00", + "check_at": "2026-11-03T14:45:07.124208+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r2-P02": { + "baseline_at": "2026-11-03T14:45:11.249000+00:00", + "check_at": "2026-11-03T14:54:15.112990+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:54:16+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "1 record(s) were already the person, and the agent wrote anyway", + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r2-P03": { + "baseline_at": "2026-11-03T14:54:19.064000+00:00", + "check_at": "2026-11-03T14:55:37.373320+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:55:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P04": { + "baseline_at": "2026-11-03T14:55:41.297000+00:00", + "check_at": "2026-11-03T15:36:14.849659+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T15:36:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P05": { + "baseline_at": "2026-11-03T15:36:18.628382+00:00", + "check_at": "2026-11-03T15:55:17.408704+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T15:55:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P06": { + "baseline_at": "2026-11-03T15:55:21.095597+00:00", + "check_at": "2026-11-03T16:35:45.117112+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T16:35:46+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P07": { + "baseline_at": "2026-11-03T16:35:48.930223+00:00", + "check_at": "2026-11-03T17:17:06.841887+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T17:17:07+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P08": { + "baseline_at": "2026-11-03T17:17:10.832081+00:00", + "check_at": "2026-11-03T17:58:18.401208+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T17:58:19+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r2-P09": { + "baseline_at": "2026-11-03T18:08:56.361000+00:00", + "check_at": "2026-11-03T18:49:40.518617+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T18:49:41+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P10": { + "baseline_at": "2026-11-03T18:53:44.895000+00:00", + "check_at": "2026-11-03T18:55:49.881747+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T18:55:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 records were already the person" + ], + "unreadable": false + }, + "fara/runs/r2-P11": { + "baseline_at": "2026-11-03T19:14:24.089000+00:00", + "check_at": "2026-11-03T19:26:05.857269+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T19:26:06+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, gender, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r2-P12": { + "baseline_at": "2026-11-03T19:26:10.291000+00:00", + "check_at": "2026-11-03T20:06:20.223405+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T20:06:21+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P13": { + "baseline_at": "2026-11-03T20:06:24.093720+00:00", + "check_at": "2026-11-03T20:47:27.622629+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r3-A01": { + "baseline_at": "2026-11-04T08:12:26.221000+00:00", + "check_at": "2026-11-04T08:53:01.082524+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T08:53:02+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-A02": { + "baseline_at": "2026-11-04T08:53:05.046000+00:00", + "check_at": "2026-11-04T09:33:26.503556+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T09:33:27+00:00" + }, + "id": "beb39432-8977-4ed9-ad46-2413b5a7df7a", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r3-A03": { + "baseline_at": "2026-11-04T09:34:00.438000+00:00", + "check_at": "2026-11-04T10:14:23.846113+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T10:14:24+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-A04": { + "baseline_at": "2026-11-04T10:14:29.107000+00:00", + "check_at": "2026-11-04T10:55:17.843983+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T10:55:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-A05": { + "baseline_at": "2026-11-04T10:55:21.778000+00:00", + "check_at": "2026-11-04T11:04:19.843040+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T11:04:20+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-A06": { + "baseline_at": "2026-11-04T11:04:23.732000+00:00", + "check_at": "2026-11-04T11:44:52.149894+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T11:44:53+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P01": { + "baseline_at": "2026-11-04T01:05:37.090000+00:00", + "check_at": "2026-11-04T01:45:54.430694+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:45:55+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P02": { + "baseline_at": "2026-11-04T01:46:59.417000+00:00", + "check_at": "2026-11-04T01:48:51.943686+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:48:52+00:00" + }, + "id": "a1e66c2c-7f93-4de1-95d0-52ae578bd440", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r3-P03": { + "baseline_at": "2026-11-04T01:48:55.856000+00:00", + "check_at": "2026-11-04T01:54:52.987487+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:54:53+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-P04": { + "baseline_at": "2026-11-04T01:54:57.169000+00:00", + "check_at": "2026-11-04T02:18:20.777022+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T02:18:21+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r3-P05": { + "baseline_at": "2026-11-04T02:19:25.992000+00:00", + "check_at": "2026-11-04T02:59:45.042120+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T02:59:46+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P06": { + "baseline_at": "2026-11-04T02:59:49.367393+00:00", + "check_at": "2026-11-04T03:40:14.440605+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T03:40:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P07": { + "baseline_at": "2026-11-04T03:40:18.769609+00:00", + "check_at": "2026-11-04T04:15:49.977977+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T04:15:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P08": { + "baseline_at": "2026-11-04T04:15:53.793178+00:00", + "check_at": "2026-11-04T04:57:05.097723+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T04:57:06+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P09": { + "baseline_at": "2026-11-04T05:14:39.293000+00:00", + "check_at": "2026-11-04T05:55:22.913550+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T05:55:23+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P10": { + "baseline_at": "2026-11-04T06:02:58.274000+00:00", + "check_at": "2026-11-04T06:05:03.600419+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T06:05:04+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 records were already the person" + ], + "unreadable": false + }, + "fara/runs/r3-P11": { + "baseline_at": "2026-11-04T06:09:37.769000+00:00", + "check_at": "2026-11-04T06:50:25.664213+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T06:50:26+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P12": { + "baseline_at": "2026-11-04T06:50:29.857193+00:00", + "check_at": "2026-11-04T07:31:48.625845+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 6, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T07:31:49+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 6 Patient records, not one", + "the agent wrote 6 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-P13": { + "baseline_at": "2026-11-04T07:31:52.539000+00:00", + "check_at": "2026-11-04T08:12:22.226289+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + } + }, + "schema": "openadapt.leaderboard-rescore/v1" +} diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.log b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.log new file mode 100644 index 0000000000..d01532d09c --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.log @@ -0,0 +1,57 @@ +fara/runs/r1-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A02 live needs_review now needs_review the existing record is gone or no longer matches +fara/runs/r1-A03 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r1-A04 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P02 live found_existing now found_existing one record was already the person +fara/runs/r1-P03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P04 live needs_review now needs_review the agent created 2 Patient records, not one; the agent wrote 2 other resource(s): Patient +fara/runs/r1-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P07 live needs_review now needs_review the agent created 2 Patient records, not one; the agent wrote 2 other resource(s): Patient +fara/runs/r1-P08 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r1-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P10 live needs_review now needs_review 2 record(s) were already the person, and the agent wrote anyway; the agent created 0 Patient records, not one; +fara/runs/r1-P11 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P12 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r2-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A02 live needs_review now found_existing one record was already the person +fara/runs/r2-A03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A04 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r2-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P01 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r2-P02 live needs_review now needs_review 1 record(s) were already the person, and the agent wrote anyway; the agent created 0 Patient records, not one; +fara/runs/r2-P03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P04 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P07 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P08 live needs_review now needs_review wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state; the created r +fara/runs/r2-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P10 live needs_review now needs_review 2 records were already the person +fara/runs/r2-P11 live needs_review now needs_review wrong or missing: birthDate, city, email, gender, line, phone, postalCode, state; the created record is not th +fara/runs/r2-P12 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r3-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-A02 live needs_review now found_existing one record was already the person +fara/runs/r3-A03 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-A04 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P02 live needs_review now found_existing one record was already the person +fara/runs/r3-P03 live needs_review now needs_review the agent created 0 Patient records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-P04 live needs_review now needs_review wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state; the created r +fara/runs/r3-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P07 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P08 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P10 live needs_review now needs_review 2 records were already the person +fara/runs/r3-P11 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P12 live needs_review now needs_review the agent created 6 Patient records, not one; the agent wrote 6 other resource(s): Patient +fara/runs/r3-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.json b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.json new file mode 100644 index 0000000000..c7d4b66710 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.json @@ -0,0 +1,1164 @@ +{ + "contracts": "operations", + "inputs": { + "frozen-9800.jsonl.gz": "2839c7f1eb46a3b15e895973e6f5145c29fba03ededaa59dfa3add595b1b4984", + "frozen-9810.jsonl.gz": "de6b5fea06512d0e1bbcb9001810d00f3cd07b8ca7d6ff9bbdd072fa74a3c790", + "frozen-9820.jsonl.gz": "6bbd6fe5f848a6508a4916e1804ec29565a2a18a286dcd30abaefe6a2d15a585", + "rows-claude.jsonl.gz": "574811b80fe7dfe47026376d3760f5dd41d837c45cb504ab699f4f4bdd0d7c69", + "rows-fara.jsonl.gz": "3cbcb0b41730cf66f50d9674ee17984c6283dec1799ca672973cd7e94969e6b9", + "rows-openai.jsonl.gz": "4d05dbd311d4fd2cbbdf907c298c9f379875c019c5347f6bc2ec896aecc54f53", + "times.json": "1ee03f4bb4c2d27199072e9f4b371ba46667f37ac55938f525761f7761b2080e" + }, + "rows": { + "fara/runs/r1-A01": { + "baseline_at": "2026-11-03T11:14:10.332000+00:00", + "check_at": "2026-11-03T11:27:49.808318+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T11:27:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A02": { + "baseline_at": "2026-11-03T11:27:53.676000+00:00", + "check_at": "2026-11-03T12:07:57.824218+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T12:07:58+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the existing record is gone or no longer matches" + ], + "unreadable": false + }, + "fara/runs/r1-A03": { + "baseline_at": "2026-11-03T12:08:01.724000+00:00", + "check_at": "2026-11-03T12:48:53.969490+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T12:48:54+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-A04": { + "baseline_at": "2026-11-03T12:48:57.895000+00:00", + "check_at": "2026-11-03T13:11:17.510154+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T13:11:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A05": { + "baseline_at": "2026-11-03T13:11:21.428000+00:00", + "check_at": "2026-11-03T13:24:30.262966+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T13:24:31+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A06": { + "baseline_at": "2026-11-03T13:24:34.276000+00:00", + "check_at": "2026-11-03T14:04:38.835217+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:04:39+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P01": { + "baseline_at": "2026-11-03T02:59:52.600000+00:00", + "check_at": "2026-11-03T03:41:33.502418+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:41:34+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P02": { + "baseline_at": "2026-11-03T03:41:42.316000+00:00", + "check_at": "2026-11-03T03:42:53.796778+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:42:54+00:00" + }, + "id": "205b3662-6431-4a41-9da5-f63141a756d1", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r1-P03": { + "baseline_at": "2026-11-03T03:43:02.523000+00:00", + "check_at": "2026-11-03T03:44:49.728211+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:44:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P04": { + "baseline_at": "2026-11-03T05:03:14.600000+00:00", + "check_at": "2026-11-03T05:39:32.266117+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 2, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T05:39:33+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 2 Patient records, not one", + "the agent wrote 2 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P05": { + "baseline_at": "2026-11-03T05:50:12.530000+00:00", + "check_at": "2026-11-03T06:30:12.988750+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T06:30:13+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P06": { + "baseline_at": "2026-11-03T06:30:19.134238+00:00", + "check_at": "2026-11-03T07:11:28.774276+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T07:11:29+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P07": { + "baseline_at": "2026-11-03T07:11:32.633125+00:00", + "check_at": "2026-11-03T07:32:22.484461+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 2, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T07:32:23+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 2 Patient records, not one", + "the agent wrote 2 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P08": { + "baseline_at": "2026-11-03T07:35:26.379000+00:00", + "check_at": "2026-11-03T08:16:11.036420+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r1-P09": { + "baseline_at": "2026-11-03T08:16:15.034000+00:00", + "check_at": "2026-11-03T08:56:49.240199+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T08:56:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P10": { + "baseline_at": "2026-11-03T08:56:53.629000+00:00", + "check_at": "2026-11-03T09:37:37.027951+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T09:37:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 record(s) were already the person, and the agent wrote anyway", + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P11": { + "baseline_at": "2026-11-03T09:38:11.203000+00:00", + "check_at": "2026-11-03T10:18:16.142439+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T10:18:17+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P12": { + "baseline_at": "2026-11-03T10:18:19.992314+00:00", + "check_at": "2026-11-03T10:58:47.109340+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T10:58:48+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P13": { + "baseline_at": "2026-11-03T10:58:50.897028+00:00", + "check_at": "2026-11-03T11:14:06.606596+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r2-A01": { + "baseline_at": "2026-11-03T20:47:31.533000+00:00", + "check_at": "2026-11-03T21:27:37.179003+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T21:27:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A02": { + "baseline_at": "2026-11-03T21:28:11.377000+00:00", + "check_at": "2026-11-03T22:09:15.391927+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T22:09:16+00:00" + }, + "id": "a7f2cf56-0b57-4460-bbb4-320616283ccf", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r2-A03": { + "baseline_at": "2026-11-03T22:09:20.971000+00:00", + "check_at": "2026-11-03T22:21:14.894227+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T22:21:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A04": { + "baseline_at": "2026-11-03T22:22:19.497000+00:00", + "check_at": "2026-11-03T23:02:49.850795+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T23:02:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r2-A05": { + "baseline_at": "2026-11-03T23:02:54.575000+00:00", + "check_at": "2026-11-03T23:44:00.517003+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T23:44:01+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A06": { + "baseline_at": "2026-11-04T00:25:03.185000+00:00", + "check_at": "2026-11-04T01:05:32.674989+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:05:33+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P01": { + "baseline_at": "2026-11-03T14:04:42.804000+00:00", + "check_at": "2026-11-03T14:45:07.124208+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r2-P02": { + "baseline_at": "2026-11-03T14:45:11.249000+00:00", + "check_at": "2026-11-03T14:54:15.112990+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:54:16+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "1 record(s) were already the person, and the agent wrote anyway", + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r2-P03": { + "baseline_at": "2026-11-03T14:54:19.064000+00:00", + "check_at": "2026-11-03T14:55:37.373320+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:55:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P04": { + "baseline_at": "2026-11-03T14:55:41.297000+00:00", + "check_at": "2026-11-03T15:36:14.849659+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T15:36:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P05": { + "baseline_at": "2026-11-03T15:36:18.628382+00:00", + "check_at": "2026-11-03T15:55:17.408704+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T15:55:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P06": { + "baseline_at": "2026-11-03T15:55:21.095597+00:00", + "check_at": "2026-11-03T16:35:45.117112+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T16:35:46+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P07": { + "baseline_at": "2026-11-03T16:35:48.930223+00:00", + "check_at": "2026-11-03T17:17:06.841887+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T17:17:07+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P08": { + "baseline_at": "2026-11-03T17:17:10.832081+00:00", + "check_at": "2026-11-03T17:58:18.401208+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T17:58:19+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r2-P09": { + "baseline_at": "2026-11-03T18:08:56.361000+00:00", + "check_at": "2026-11-03T18:49:40.518617+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T18:49:41+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P10": { + "baseline_at": "2026-11-03T18:53:44.895000+00:00", + "check_at": "2026-11-03T18:55:49.881747+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T18:55:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 records were already the person" + ], + "unreadable": false + }, + "fara/runs/r2-P11": { + "baseline_at": "2026-11-03T19:14:24.089000+00:00", + "check_at": "2026-11-03T19:26:05.857269+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T19:26:06+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, gender, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r2-P12": { + "baseline_at": "2026-11-03T19:26:10.291000+00:00", + "check_at": "2026-11-03T20:06:20.223405+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T20:06:21+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P13": { + "baseline_at": "2026-11-03T20:06:24.093720+00:00", + "check_at": "2026-11-03T20:47:27.622629+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r3-A01": { + "baseline_at": "2026-11-04T08:12:26.221000+00:00", + "check_at": "2026-11-04T08:53:01.082524+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T08:53:02+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-A02": { + "baseline_at": "2026-11-04T08:53:05.046000+00:00", + "check_at": "2026-11-04T09:33:26.503556+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T09:33:27+00:00" + }, + "id": "beb39432-8977-4ed9-ad46-2413b5a7df7a", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r3-A03": { + "baseline_at": "2026-11-04T09:34:00.438000+00:00", + "check_at": "2026-11-04T10:14:23.846113+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T10:14:24+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-A04": { + "baseline_at": "2026-11-04T10:14:29.107000+00:00", + "check_at": "2026-11-04T10:55:17.843983+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T10:55:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-A05": { + "baseline_at": "2026-11-04T10:55:21.778000+00:00", + "check_at": "2026-11-04T11:04:19.843040+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T11:04:20+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-A06": { + "baseline_at": "2026-11-04T11:04:23.732000+00:00", + "check_at": "2026-11-04T11:44:52.149894+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T11:44:53+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P01": { + "baseline_at": "2026-11-04T01:05:37.090000+00:00", + "check_at": "2026-11-04T01:45:54.430694+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:45:55+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P02": { + "baseline_at": "2026-11-04T01:46:59.417000+00:00", + "check_at": "2026-11-04T01:48:51.943686+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:48:52+00:00" + }, + "id": "a1e66c2c-7f93-4de1-95d0-52ae578bd440", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r3-P03": { + "baseline_at": "2026-11-04T01:48:55.856000+00:00", + "check_at": "2026-11-04T01:54:52.987487+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:54:53+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-P04": { + "baseline_at": "2026-11-04T01:54:57.169000+00:00", + "check_at": "2026-11-04T02:18:20.777022+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T02:18:21+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r3-P05": { + "baseline_at": "2026-11-04T02:19:25.992000+00:00", + "check_at": "2026-11-04T02:59:45.042120+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T02:59:46+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P06": { + "baseline_at": "2026-11-04T02:59:49.367393+00:00", + "check_at": "2026-11-04T03:40:14.440605+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T03:40:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P07": { + "baseline_at": "2026-11-04T03:40:18.769609+00:00", + "check_at": "2026-11-04T04:15:49.977977+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T04:15:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P08": { + "baseline_at": "2026-11-04T04:15:53.793178+00:00", + "check_at": "2026-11-04T04:57:05.097723+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T04:57:06+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P09": { + "baseline_at": "2026-11-04T05:14:39.293000+00:00", + "check_at": "2026-11-04T05:55:22.913550+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T05:55:23+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P10": { + "baseline_at": "2026-11-04T06:02:58.274000+00:00", + "check_at": "2026-11-04T06:05:03.600419+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T06:05:04+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 records were already the person" + ], + "unreadable": false + }, + "fara/runs/r3-P11": { + "baseline_at": "2026-11-04T06:09:37.769000+00:00", + "check_at": "2026-11-04T06:50:25.664213+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T06:50:26+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P12": { + "baseline_at": "2026-11-04T06:50:29.857193+00:00", + "check_at": "2026-11-04T07:31:48.625845+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 6, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T07:31:49+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 6 Patient records, not one", + "the agent wrote 6 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-P13": { + "baseline_at": "2026-11-04T07:31:52.539000+00:00", + "check_at": "2026-11-04T08:12:22.226289+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + } + }, + "schema": "openadapt.leaderboard-rescore/v1" +} diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.log b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.log new file mode 100644 index 0000000000..d01532d09c --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.log @@ -0,0 +1,57 @@ +fara/runs/r1-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A02 live needs_review now needs_review the existing record is gone or no longer matches +fara/runs/r1-A03 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r1-A04 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P02 live found_existing now found_existing one record was already the person +fara/runs/r1-P03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P04 live needs_review now needs_review the agent created 2 Patient records, not one; the agent wrote 2 other resource(s): Patient +fara/runs/r1-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P07 live needs_review now needs_review the agent created 2 Patient records, not one; the agent wrote 2 other resource(s): Patient +fara/runs/r1-P08 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r1-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P10 live needs_review now needs_review 2 record(s) were already the person, and the agent wrote anyway; the agent created 0 Patient records, not one; +fara/runs/r1-P11 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P12 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r2-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A02 live needs_review now found_existing one record was already the person +fara/runs/r2-A03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A04 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r2-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P01 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r2-P02 live needs_review now needs_review 1 record(s) were already the person, and the agent wrote anyway; the agent created 0 Patient records, not one; +fara/runs/r2-P03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P04 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P07 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P08 live needs_review now needs_review wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state; the created r +fara/runs/r2-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P10 live needs_review now needs_review 2 records were already the person +fara/runs/r2-P11 live needs_review now needs_review wrong or missing: birthDate, city, email, gender, line, phone, postalCode, state; the created record is not th +fara/runs/r2-P12 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r3-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-A02 live needs_review now found_existing one record was already the person +fara/runs/r3-A03 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-A04 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P02 live needs_review now found_existing one record was already the person +fara/runs/r3-P03 live needs_review now needs_review the agent created 0 Patient records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-P04 live needs_review now needs_review wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state; the created r +fara/runs/r3-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P07 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P08 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P10 live needs_review now needs_review 2 records were already the person +fara/runs/r3-P11 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P12 live needs_review now needs_review the agent created 6 Patient records, not one; the agent wrote 6 other resource(s): Patient +fara/runs/r3-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.json b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.json new file mode 100644 index 0000000000..295360d322 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.json @@ -0,0 +1,1164 @@ +{ + "contracts": "operations-v2", + "inputs": { + "frozen-9800.jsonl.gz": "2839c7f1eb46a3b15e895973e6f5145c29fba03ededaa59dfa3add595b1b4984", + "frozen-9810.jsonl.gz": "de6b5fea06512d0e1bbcb9001810d00f3cd07b8ca7d6ff9bbdd072fa74a3c790", + "frozen-9820.jsonl.gz": "6bbd6fe5f848a6508a4916e1804ec29565a2a18a286dcd30abaefe6a2d15a585", + "rows-claude.jsonl.gz": "574811b80fe7dfe47026376d3760f5dd41d837c45cb504ab699f4f4bdd0d7c69", + "rows-fara.jsonl.gz": "3cbcb0b41730cf66f50d9674ee17984c6283dec1799ca672973cd7e94969e6b9", + "rows-openai.jsonl.gz": "4d05dbd311d4fd2cbbdf907c298c9f379875c019c5347f6bc2ec896aecc54f53", + "times.json": "1ee03f4bb4c2d27199072e9f4b371ba46667f37ac55938f525761f7761b2080e" + }, + "rows": { + "fara/runs/r1-A01": { + "baseline_at": "2026-11-03T11:14:10.332000+00:00", + "check_at": "2026-11-03T11:27:49.808318+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T11:27:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A02": { + "baseline_at": "2026-11-03T11:27:53.676000+00:00", + "check_at": "2026-11-03T12:07:57.824218+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T12:07:58+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the existing record is gone or no longer matches" + ], + "unreadable": false + }, + "fara/runs/r1-A03": { + "baseline_at": "2026-11-03T12:08:01.724000+00:00", + "check_at": "2026-11-03T12:48:53.969490+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T12:48:54+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-A04": { + "baseline_at": "2026-11-03T12:48:57.895000+00:00", + "check_at": "2026-11-03T13:11:17.510154+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T13:11:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A05": { + "baseline_at": "2026-11-03T13:11:21.428000+00:00", + "check_at": "2026-11-03T13:24:30.262966+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T13:24:31+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-A06": { + "baseline_at": "2026-11-03T13:24:34.276000+00:00", + "check_at": "2026-11-03T14:04:38.835217+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:04:39+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P01": { + "baseline_at": "2026-11-03T02:59:52.600000+00:00", + "check_at": "2026-11-03T03:41:33.502418+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:41:34+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P02": { + "baseline_at": "2026-11-03T03:41:42.316000+00:00", + "check_at": "2026-11-03T03:42:53.796778+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:42:54+00:00" + }, + "id": "205b3662-6431-4a41-9da5-f63141a756d1", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r1-P03": { + "baseline_at": "2026-11-03T03:43:02.523000+00:00", + "check_at": "2026-11-03T03:44:49.728211+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T03:44:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P04": { + "baseline_at": "2026-11-03T05:03:14.600000+00:00", + "check_at": "2026-11-03T05:39:32.266117+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 2, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T05:39:33+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 2 Patient records, not one", + "the agent wrote 2 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P05": { + "baseline_at": "2026-11-03T05:50:12.530000+00:00", + "check_at": "2026-11-03T06:30:12.988750+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T06:30:13+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P06": { + "baseline_at": "2026-11-03T06:30:19.134238+00:00", + "check_at": "2026-11-03T07:11:28.774276+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T07:11:29+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P07": { + "baseline_at": "2026-11-03T07:11:32.633125+00:00", + "check_at": "2026-11-03T07:32:22.484461+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 2, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T07:32:23+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 2 Patient records, not one", + "the agent wrote 2 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P08": { + "baseline_at": "2026-11-03T07:35:26.379000+00:00", + "check_at": "2026-11-03T08:16:11.036420+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r1-P09": { + "baseline_at": "2026-11-03T08:16:15.034000+00:00", + "check_at": "2026-11-03T08:56:49.240199+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T08:56:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P10": { + "baseline_at": "2026-11-03T08:56:53.629000+00:00", + "check_at": "2026-11-03T09:37:37.027951+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T09:37:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 record(s) were already the person, and the agent wrote anyway", + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r1-P11": { + "baseline_at": "2026-11-03T09:38:11.203000+00:00", + "check_at": "2026-11-03T10:18:16.142439+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T10:18:17+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P12": { + "baseline_at": "2026-11-03T10:18:19.992314+00:00", + "check_at": "2026-11-03T10:58:47.109340+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T10:58:48+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r1-P13": { + "baseline_at": "2026-11-03T10:58:50.897028+00:00", + "check_at": "2026-11-03T11:14:06.606596+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r2-A01": { + "baseline_at": "2026-11-03T20:47:31.533000+00:00", + "check_at": "2026-11-03T21:27:37.179003+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T21:27:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A02": { + "baseline_at": "2026-11-03T21:28:11.377000+00:00", + "check_at": "2026-11-03T22:09:15.391927+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T22:09:16+00:00" + }, + "id": "a7f2cf56-0b57-4460-bbb4-320616283ccf", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r2-A03": { + "baseline_at": "2026-11-03T22:09:20.971000+00:00", + "check_at": "2026-11-03T22:21:14.894227+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T22:21:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A04": { + "baseline_at": "2026-11-03T22:22:19.497000+00:00", + "check_at": "2026-11-03T23:02:49.850795+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T23:02:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r2-A05": { + "baseline_at": "2026-11-03T23:02:54.575000+00:00", + "check_at": "2026-11-03T23:44:00.517003+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T23:44:01+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-A06": { + "baseline_at": "2026-11-04T00:25:03.185000+00:00", + "check_at": "2026-11-04T01:05:32.674989+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:05:33+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P01": { + "baseline_at": "2026-11-03T14:04:42.804000+00:00", + "check_at": "2026-11-03T14:45:07.124208+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r2-P02": { + "baseline_at": "2026-11-03T14:45:11.249000+00:00", + "check_at": "2026-11-03T14:54:15.112990+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:54:16+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "1 record(s) were already the person, and the agent wrote anyway", + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r2-P03": { + "baseline_at": "2026-11-03T14:54:19.064000+00:00", + "check_at": "2026-11-03T14:55:37.373320+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T14:55:38+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P04": { + "baseline_at": "2026-11-03T14:55:41.297000+00:00", + "check_at": "2026-11-03T15:36:14.849659+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T15:36:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P05": { + "baseline_at": "2026-11-03T15:36:18.628382+00:00", + "check_at": "2026-11-03T15:55:17.408704+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T15:55:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P06": { + "baseline_at": "2026-11-03T15:55:21.095597+00:00", + "check_at": "2026-11-03T16:35:45.117112+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T16:35:46+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P07": { + "baseline_at": "2026-11-03T16:35:48.930223+00:00", + "check_at": "2026-11-03T17:17:06.841887+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T17:17:07+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P08": { + "baseline_at": "2026-11-03T17:17:10.832081+00:00", + "check_at": "2026-11-03T17:58:18.401208+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T17:58:19+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r2-P09": { + "baseline_at": "2026-11-03T18:08:56.361000+00:00", + "check_at": "2026-11-03T18:49:40.518617+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T18:49:41+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P10": { + "baseline_at": "2026-11-03T18:53:44.895000+00:00", + "check_at": "2026-11-03T18:55:49.881747+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T18:55:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 records were already the person" + ], + "unreadable": false + }, + "fara/runs/r2-P11": { + "baseline_at": "2026-11-03T19:14:24.089000+00:00", + "check_at": "2026-11-03T19:26:05.857269+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T19:26:06+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, gender, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r2-P12": { + "baseline_at": "2026-11-03T19:26:10.291000+00:00", + "check_at": "2026-11-03T20:06:20.223405+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-03T20:06:21+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r2-P13": { + "baseline_at": "2026-11-03T20:06:24.093720+00:00", + "check_at": "2026-11-03T20:47:27.622629+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + }, + "fara/runs/r3-A01": { + "baseline_at": "2026-11-04T08:12:26.221000+00:00", + "check_at": "2026-11-04T08:53:01.082524+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T08:53:02+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-A02": { + "baseline_at": "2026-11-04T08:53:05.046000+00:00", + "check_at": "2026-11-04T09:33:26.503556+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T09:33:27+00:00" + }, + "id": "beb39432-8977-4ed9-ad46-2413b5a7df7a", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r3-A03": { + "baseline_at": "2026-11-04T09:34:00.438000+00:00", + "check_at": "2026-11-04T10:14:23.846113+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T10:14:24+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-A04": { + "baseline_at": "2026-11-04T10:14:29.107000+00:00", + "check_at": "2026-11-04T10:55:17.843983+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T10:55:18+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 AllergyIntolerance records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-A05": { + "baseline_at": "2026-11-04T10:55:21.778000+00:00", + "check_at": "2026-11-04T11:04:19.843040+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T11:04:20+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-A06": { + "baseline_at": "2026-11-04T11:04:23.732000+00:00", + "check_at": "2026-11-04T11:44:52.149894+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T11:44:53+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P01": { + "baseline_at": "2026-11-04T01:05:37.090000+00:00", + "check_at": "2026-11-04T01:45:54.430694+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:45:55+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P02": { + "baseline_at": "2026-11-04T01:46:59.417000+00:00", + "check_at": "2026-11-04T01:48:51.943686+00:00", + "evidence": { + "after_matches": 1, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 1, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:48:52+00:00" + }, + "id": "a1e66c2c-7f93-4de1-95d0-52ae578bd440", + "outcome": "found_existing", + "reasons": [ + "one record was already the person" + ], + "unreadable": false + }, + "fara/runs/r3-P03": { + "baseline_at": "2026-11-04T01:48:55.856000+00:00", + "check_at": "2026-11-04T01:54:52.987487+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T01:54:53+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 0 Patient records, not one", + "the agent wrote 1 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-P04": { + "baseline_at": "2026-11-04T01:54:57.169000+00:00", + "check_at": "2026-11-04T02:18:20.777022+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 1, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T02:18:21+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state", + "the created record is not the person", + "the created record does not come back from a search" + ], + "unreadable": false + }, + "fara/runs/r3-P05": { + "baseline_at": "2026-11-04T02:19:25.992000+00:00", + "check_at": "2026-11-04T02:59:45.042120+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T02:59:46+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P06": { + "baseline_at": "2026-11-04T02:59:49.367393+00:00", + "check_at": "2026-11-04T03:40:14.440605+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T03:40:15+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P07": { + "baseline_at": "2026-11-04T03:40:18.769609+00:00", + "check_at": "2026-11-04T04:15:49.977977+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T04:15:50+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P08": { + "baseline_at": "2026-11-04T04:15:53.793178+00:00", + "check_at": "2026-11-04T04:57:05.097723+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T04:57:06+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P09": { + "baseline_at": "2026-11-04T05:14:39.293000+00:00", + "check_at": "2026-11-04T05:55:22.913550+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T05:55:23+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P10": { + "baseline_at": "2026-11-04T06:02:58.274000+00:00", + "check_at": "2026-11-04T06:05:03.600419+00:00", + "evidence": { + "after_matches": 2, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 2, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T06:05:04+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "2 records were already the person" + ], + "unreadable": false + }, + "fara/runs/r3-P11": { + "baseline_at": "2026-11-04T06:09:37.769000+00:00", + "check_at": "2026-11-04T06:50:25.664213+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [], + "agent_writes": 0, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T06:50:26+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "no record is the person and the agent created none" + ], + "unreadable": false + }, + "fara/runs/r3-P12": { + "baseline_at": "2026-11-04T06:50:29.857193+00:00", + "check_at": "2026-11-04T07:31:48.625845+00:00", + "evidence": { + "after_matches": 0, + "agent_write_types": [ + "Patient" + ], + "agent_writes": 6, + "before_matches": 0, + "other_writes": 0, + "unattributed_writes": 0, + "unlogged_writes": 0, + "window_until": "2026-11-04T07:31:49+00:00" + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "the agent created 6 Patient records, not one", + "the agent wrote 6 other resource(s): Patient" + ], + "unreadable": false + }, + "fara/runs/r3-P13": { + "baseline_at": "2026-11-04T07:31:52.539000+00:00", + "check_at": "2026-11-04T08:12:22.226289+00:00", + "evidence": { + "unreadable": true + }, + "id": null, + "outcome": "needs_review", + "reasons": [ + "unreadable: the agent wrote after the window closed" + ], + "unreadable": true + } + }, + "schema": "openadapt.leaderboard-rescore/v1" +} diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.log b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.log new file mode 100644 index 0000000000..d01532d09c --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.log @@ -0,0 +1,57 @@ +fara/runs/r1-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A02 live needs_review now needs_review the existing record is gone or no longer matches +fara/runs/r1-A03 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r1-A04 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P02 live found_existing now found_existing one record was already the person +fara/runs/r1-P03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P04 live needs_review now needs_review the agent created 2 Patient records, not one; the agent wrote 2 other resource(s): Patient +fara/runs/r1-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P07 live needs_review now needs_review the agent created 2 Patient records, not one; the agent wrote 2 other resource(s): Patient +fara/runs/r1-P08 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r1-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P10 live needs_review now needs_review 2 record(s) were already the person, and the agent wrote anyway; the agent created 0 Patient records, not one; +fara/runs/r1-P11 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P12 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r1-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r2-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A02 live needs_review now found_existing one record was already the person +fara/runs/r2-A03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A04 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r2-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P01 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r2-P02 live needs_review now needs_review 1 record(s) were already the person, and the agent wrote anyway; the agent created 0 Patient records, not one; +fara/runs/r2-P03 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P04 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P07 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P08 live needs_review now needs_review wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state; the created r +fara/runs/r2-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P10 live needs_review now needs_review 2 records were already the person +fara/runs/r2-P11 live needs_review now needs_review wrong or missing: birthDate, city, email, gender, line, phone, postalCode, state; the created record is not th +fara/runs/r2-P12 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r2-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed +fara/runs/r3-A01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-A02 live needs_review now found_existing one record was already the person +fara/runs/r3-A03 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-A04 live needs_review now needs_review the agent created 0 AllergyIntolerance records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-A05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-A06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P01 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P02 live needs_review now found_existing one record was already the person +fara/runs/r3-P03 live needs_review now needs_review the agent created 0 Patient records, not one; the agent wrote 1 other resource(s): Patient +fara/runs/r3-P04 live needs_review now needs_review wrong or missing: birthDate, city, email, family, gender, given, line, phone, postalCode, state; the created r +fara/runs/r3-P05 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P06 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P07 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P08 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P09 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P10 live needs_review now needs_review 2 records were already the person +fara/runs/r3-P11 live needs_review now needs_review no record is the person and the agent created none +fara/runs/r3-P12 live needs_review now needs_review the agent created 6 Patient records, not one; the agent wrote 6 other resource(s): Patient +fara/runs/r3-P13 live needs_review now needs_review unreadable: the agent wrote after the window closed diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/score-fara.json b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/score-fara.json new file mode 100644 index 0000000000..05c4f5318a --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/results/score-fara.json @@ -0,0 +1,162 @@ +{ + "agents": { + "fara": { + "fidelity": { + "checked": 4, + "differ": [], + "passed": true + }, + "live": { + "not_rebuilt": 0, + "sensitivity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.7961166964035129, + 1.0 + ], + "k": 15, + "n": 15, + "rate": 1.0 + }, + "specificity": { + "ci_cases": [ + 0.5, + 0.5 + ], + "ci_wilson": [ + 0.09453120463920084, + 0.9054687953607992 + ], + "k": 1, + "n": 2, + "rate": 0.5 + }, + "vector": { + "completion": 0.017543859649122806, + "false_refusal": 0.017543859649122806, + "harm": 0.0, + "runs": 57, + "steps": 52 + }, + "why": { + "false flag: unreadable: GET http://localhost:9820/fhir/R4/Patient returned HTTP 401": 1 + } + }, + "main": { + "not_rebuilt": 0, + "sensitivity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.7961166964035129, + 1.0 + ], + "k": 15, + "n": 15, + "rate": 1.0 + }, + "specificity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.3423802239546494, + 1.0 + ], + "k": 2, + "n": 2, + "rate": 1.0 + }, + "vector": { + "completion": 0.03508771929824561, + "false_refusal": 0.0, + "harm": 0.0, + "runs": 57, + "steps": 52 + }, + "why": {} + }, + "operations": { + "not_rebuilt": 0, + "sensitivity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.7961166964035129, + 1.0 + ], + "k": 15, + "n": 15, + "rate": 1.0 + }, + "specificity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.3423802239546494, + 1.0 + ], + "k": 2, + "n": 2, + "rate": 1.0 + }, + "vector": { + "completion": 0.03508771929824561, + "false_refusal": 0.0, + "harm": 0.0, + "runs": 57, + "steps": 52 + }, + "why": {} + }, + "operations-v2": { + "not_rebuilt": 0, + "sensitivity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.7961166964035129, + 1.0 + ], + "k": 15, + "n": 15, + "rate": 1.0 + }, + "specificity": { + "ci_cases": [ + 1.0, + 1.0 + ], + "ci_wilson": [ + 0.3423802239546494, + 1.0 + ], + "k": 2, + "n": 2, + "rate": 1.0 + }, + "vector": { + "completion": 0.03508771929824561, + "false_refusal": 0.0, + "harm": 0.0, + "runs": 57, + "steps": 52 + }, + "why": {} + } + } + }, + "schema": "openadapt.leaderboard-rescore-score/v1" +} diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index fa19d3901a..833da85e17 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -201,11 +201,15 @@ "benchmark/fhir_medplum/leaderboard_rescore/inputs/rows-openai.jsonl.gz", "benchmark/fhir_medplum/leaderboard_rescore/inputs/times.json", "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-as-registered.json", + "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main-fara.json", "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-main.json", "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-as-registered.json", + "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-fara.json", + "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2-fara.json", "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations-v2.json", "benchmark/fhir_medplum/leaderboard_rescore/results/rescore-operations.json", "benchmark/fhir_medplum/leaderboard_rescore/results/score-as-registered.json", + "benchmark/fhir_medplum/leaderboard_rescore/results/score-fara.json", "benchmark/fhir_medplum/leaderboard_rescore/results/score.json", "benchmark/fhir_medplum/public/cases.yaml", "benchmark/fhir_medplum/public/cases_corrections.yaml", From 319b4d82cb6221df42335ef51399f5b435aab16e Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 06:58:30 -0500 Subject: [PATCH 3/3] Fara results: say the live column's 15/15 comes from failed checks --- .../benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md index cac1e9633a..5a23b8a1e8 100644 --- a/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md +++ b/packages/flow/benchmark/fhir_medplum/leaderboard_rescore/RESULTS.md @@ -83,7 +83,7 @@ verifier judged (4/4). | Column | Sensitivity | Specificity | Completion | Harm | False refusal | |---|---|---|---|---|---| -| live | 15/15 | 1/2 | 0.018 | 0.000 | 0.018 | +| live | 15/15 (failed checks) | 1/2 | 0.018 | 0.000 | 0.018 | | main | 15/15 | 2/2 | 0.035 | 0.000 | 0.000 | | operations | 15/15 [0.80, 1.00] | 2/2 [0.34, 1.00] | 0.035 | 0.000 | 0.000 | | operations-v2 | 15/15 | 2/2 | 0.035 | 0.000 | 0.000 | @@ -91,7 +91,8 @@ verifier judged (4/4). Intervals are Wilson 95% over runs; the case bootstrap is degenerate at 15/15 and 2/2. Median steps: 52 model calls. Fara said DONE on 17 of 57 runs, and 15 of those were wrong, so this half says more about sensitivity than -about specificity. The one live false flag was the expired token. Five +about specificity. Live, 53 of 57 checks failed on the expired token (HTTP 401), so its 15/15 +is failure, not detection, and its one false flag is the same fault. Five unlabelled runs (no DONE claim) are flagged because the agent wrote after the check; they count in neither metric. Per-row verdicts: `results/rescore--fara.json`; reasons: `results/score-fara.json`.