From 17bf35df062986b2a42acf593edb0060dbf27cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henning=20P=C3=B6ttker?= <25299532+hpoettker@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:58:18 +0200 Subject: [PATCH] Support moved rows in SAS data files --- Makefile.am | 7 +- resources/compression_type_0x02.sas7bdat | Bin 0 -> 20480 bytes resources/compression_type_0x06.sas7bdat | Bin 0 -> 20480 bytes resources/compression_type_0x09.sas7bdat | Bin 0 -> 20480 bytes resources/compression_type_0x0d.sas7bdat | Bin 0 -> 20480 bytes src/sas/readstat_sas.h | 13 +- src/sas/readstat_sas7bdat_read.c | 142 ++++++++++++++++++++- src/test/test_list.h | 154 +++++++++++++++++++++++ src/test/test_read.c | 18 +-- src/test/test_types.h | 2 + 10 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 resources/compression_type_0x02.sas7bdat create mode 100644 resources/compression_type_0x06.sas7bdat create mode 100644 resources/compression_type_0x09.sas7bdat create mode 100644 resources/compression_type_0x0d.sas7bdat diff --git a/Makefile.am b/Makefile.am index faf22521..1694e474 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,7 +82,12 @@ endif dist_man1_MANS = man/readstat.man man/extract_metadata.man -EXTRA_DIST = resources/datetime.sas7bdat +EXTRA_DIST = \ + resources/datetime.sas7bdat \ + resources/compression_type_0x02.sas7bdat \ + resources/compression_type_0x06.sas7bdat \ + resources/compression_type_0x09.sas7bdat \ + resources/compression_type_0x0d.sas7bdat include_HEADERS = src/readstat.h diff --git a/resources/compression_type_0x02.sas7bdat b/resources/compression_type_0x02.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..77e2bbca3644559715d02f0d706e69766e5b4d01 GIT binary patch literal 20480 zcmeI4O{iU0701u}h<&MnCK|9{quZK+gY2-sKb)9QV-X^TCX_~mWPjQ~+85HKO`y;U z4jecTM-Cl2a3DBw5Jep5PzMFYNktrKs*YOxXrKRc@7edhcazr?61Ui9EpcCi>I)P506X*mwfli@LcfB`d?1n7)Ka6? z*5h}A7{oMVXvNOFg~`rW zha-z~^PRV+i>@u@$MB5i;#Zz{N~}Jj8K1eFzi{T<_zb6Bxmw6eS8hB@p4)!0WODQQ zwEf(*>*a+PaBzM5r7L+DQn*pRwtfBfwQ&3DcF31*Y+t&0Ekn3$YrZ!@Xg(a6&OZcl z_gnT7yjSDpNA8vFmK>U~8-~WdwbH)Ut{yi=O}_elt{pf2O|SK1_4u7hvB@WQ7QJo9 zQ%daY_WgeQ|CsDGF3s-h@!u!C``P$^l)Z^%?AI!Xx3G-z_TX}!Tcc#YeD>Povz@!s zK(uje=Udh_yG`D5^W#l0PL6ghZCqNMwCygHzWXi&_M4Bs%O&?c)yAQXV>_OvrHefL z@N{*{@21zbTMNzmHuGNyw`R|(PYlOl$BgNUz>gbIMKQ=nRsS8ZuKuBD2`4qskQfI)>r2LO~23o-}5ka#ZI6T=ma`}PT&JbU~Bn4zg)g0 zF0GQB>r7=5n#Q6Mjoe#WtxLL;5PY^oIVFY5fo7W(EmCnPWlec4(nu}?XUTX$JP6ttmg)dcET%Wk4Hleu? zQl(M4AVN@_TTX&0Lc}7dgxX*y0Q+tO^m(u~e9+DJTXGHo|5O z+$zU|wuD>FY_N{X2+lK~VyKLQG=`5kg`N|}S#k-|>Vl>KCaB|)Q|^oRF=hsDNMK%N z#nCYigg~IN$`#%rb}}pIc_22s2x1s^$rE5HNxi_Ts_0T+3KR2;+Pe#6!bX&feL6YNG3qMNCdDohH=8ZHWfonW5|dC4@-f= zl2ElW7o@B#5T0t*h&-wkXXqlLAUdR>& zO2OfMM&3E+Fv?0v6>8MqTuc=4fSSsUhvOQR*h;OO zBMF6|YQ&RvUeIdBc=6UyuDDIQRu2xu2$W9b7)=S+SBwj~$Pr!)Wyl~%g+|a7bk)iFpz|(?!VpP`NE-@I+Ee8b Vxj~GaN<`k}f~qQgN!F0c{|(3}JYE0* literal 0 HcmV?d00001 diff --git a/resources/compression_type_0x06.sas7bdat b/resources/compression_type_0x06.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..15ed94a2bdf41e29d8a678c169b189452fab6ebe GIT binary patch literal 20480 zcmeI4ON<;x8OLj8$Fs9>);mr%j)9nz*H(x(F00;;+1c4$JB}T)iSuw!j3S(R5{m?{ z@{WkaMoLf)9HLxuLkS$+9-{lXWYer{U^{20;4FcN|tGdXb6TF4d|Msn z|JLh&JW+qPQ^uih@U`>9P<@`0=_OtW6as}nAy5bu0);>!PzV$Pg+L)t2z-{^QD*gJ(PpdkgQ)sLVk9G=%Tq=imYwE>GDP0);>!PzV$P|8WAFttl|SdSdPD$x~TX-{p;e~^ zwiQ|(_lK1&S0qdC{q53d`&Uo4I&EQd~J^1z}R3V`{M-UPMZpH2^jdH^*_1eL2xa2(Cw_f{5XD?bl|y<2JHz`NB! z8{7JU{Ep__XHVQuCtuMF-?=<+_T;_8cR1<7V-Y@k;i*UA;}@Ta0s4!NZ#)s7d@={0 zxcDpp7Xkp@J{3QI@rma*?DLOZwBh_y7tcSv5wf_Q*Em!`sLx?nl|G%xU1_UnUMX#; zy)d^^%2>a4#14R(eVt1^teuS4jB0&yJXgnS{-6Ym0_({r?XYsI$kx2OCMYs z0RCF}tGU#EC*yyq?5^hT|0VrTc^j5xlEa_#Hq`4Em*ciIRK{CxV__6{r@9dJb*$6t zR#*G2^}20-q85geqxWs~b*bw~y?>uc34Q>9o#vw-aL5&t>g!Nn$2wk3OSjtL?&|2e zeb;vNzO!X{#cci!u`_yCeH0Aq;Z38y``Xd>{OJ2|)tg@}gH}Gj#y|D=+eb~->zq;- zZDjQ34)gylqgRJHeE%xF?RI`8R0tFTg+L)t2owT^Kp{{F6as}nAy5bu0{R z1PX!62tWYfVE}O6ll<8{(0T4NU(TzV;xiX7vn!DgO`u*YRL}Cq2M$-P@5=A;FIT_0 z!~Fl(^#Y|l`yZ~M5Y-~JA6I&NV*X#-m-+vPzD&ii5GVu+fkL1V_}?Nh{7pb-d$~WG zhMoXGu@ROcY$!)Aa0URfR4Wt|6Us}MWFF>e=zK$EC8BfDKO;_M~#)qYwHt=)M-RKkH|zy!6ph^D{LfS z&Ot(?2OZ%pMovhDxT8($;wHFjnh2_lius>{pjSLQXWlE6s3(X}Zjl9>OdEtrWMsDu zf>CGy#@OKy6!K0t@Ki)LmyZOLgr-SJrm*!8bZsPz(i9L0w3!Sydpk&T+T=;%wo`K)3qzenSVoItA~SwGN&i%3%{QSe+MlYO~_tascpN00QIg5wF{21THu6L*FRBQ)`l%Jn9Inj&MAMJdvz*3uyek{q;yS2zj* zRWuQUH*6A!YOn8_Wwb>F@}>i{U=bKi7&I&-mlQt8(O_OR6<5fpK44hz;Q%!7Kq*W~ z#w3_VdVeEIe0pJNh6j&0cXQz*(XSd5P0q#GS5 zoKINcE>4+1E0bCPCh_Z&z)K5O2_H!=-mNd@R-mY!QWTMCiL7>XWghHJio|4<4;c2^ ziV(89v56hyC=uLc*GI&ama;L!45P}W-aIv)_K`NrQ-`~W(WWu|CJBymanzZRYXFqk zI7bpkTE&Ry2FL{=`<6dREfWs=y{T2H7M8HBW`As7>Q0D;(RdI-wzYR zEV9#^fp!9svz)u$JY&d+%mHvOl+p%6r ze-DyU^Fg5ZSiXE`vfM`SvSlhIm7u;&85;GJ7#uj+aQDqnZ&$kHR9hr01I+(fw$R05 zc~KburCG~^Ad5O+4Q{X@SO45`5oMMZxO~u>+t=3kU9&216#LU*aKL#Z;hDLEuSDK& z2ba6^B{HKOBX>Kgz%^D_sl1~wdb4yApV=SZ=-`JjUAN9##D(F#GmiWA&rE{Os_8)& zt=ynmy5F{fiTpe9s3plAM>0u5MHz-aBDUY5G3J`LHW(yL4>5{COSr&t#UYSeK?k>E zw6S282j=+kbt^RrVo6IX@v?8Uq02o3*10E%`QnM$K9VoJW_k{@pgxzP8bG1$Au?B8 zys6ajBlJ@S%On@2k%k!j@u|Sw%2IFI@(aXxiBjmdRPF^$?oA}Kj4*=}gQ4{bC!X@9 zIfwh#xX5$LaS^;JRUI8C|w_S-Ir>zEVyev)q)-U()*l*D>U zeh%(?uzxnvbt8H#VQ!)pxh{06z%b+nXPg?GCQ{@+H2Afdz$9WUc&wr(t6!lj`>f2( zL-wCl#`*O&nTe}F>Qj{En+3t#lL)m1*avH)&o`*pdwA(qWQlkM^h^i_-|SrOSzwa? E0GUYmXaE2J literal 0 HcmV?d00001 diff --git a/resources/compression_type_0x09.sas7bdat b/resources/compression_type_0x09.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..18a1c28c31adedaac6dd98f794105561995c8590 GIT binary patch literal 20480 zcmeHP&8uBk6+bU|XZWEOm>u@_hm$7MC_=>8gpx=x?oS)2NlX)yLZLeE zpg8gm5X{_(N?OH>2WqV`*}a#%E7oINeON)3a9E5;z^z4?j)mH%ab1MlRHi=ns=PJY}B^2&F^nNez?E3 z&uq2sef1}#wYr~M((yhJ7zhjm1_A?tfxtjuATSUZ2n+-U0t10Z1A({Zee0?>-+Zh~ zjF`e z+Y&L4`}4C)Qyijq|FO9^{#}3n=i|5B-#k8DTwKkco)@xz+CQz~^}XMJ{M-w#Z*J~G zd*|`bdED|l%6zgJIfO41{VT(SiLpV?zNo1aOPaQcT=ytT*&P!H(w&RuHGoM+$dL5$>^kO z-?)CGyz&YwYa(IE^)TiC-v-GdEtNo5um-qWftNk8_F7NAD?Y}!KcKPhia<#ob zr^e<^-yhWe`)se{((SJH|2FGA$j1Mp>@PUlx>j-cD~=}bH!h!SogK$#u0ONr{b>%w zY8-p^b-R6N&(V)|LE9YdIC@+RUp6YSvaqRthTe{4{56q)i z^S$eDEHxiE&3_==Slp{VF?|{~o?g`O!;AN0i}&MmRe#t7>+o;wzRJHnYL(YG)0ehT zdV7!k|5EAAHpANwskYnmjn+V5ATSUZ2n+-U0t118z(8OiFc26B3`tJUT z>8E?_|Chh7uJ2p?&iw277l0Fe3A2fdbHD44`_&zw@i`C}2n+-U0t118z(8OiFc26B z3-Q^wst^NP8`C+yGyv(7M{r|oBIhwKmf8SvmiUWaxz(8OiFc5eI5!l#%u3v6H zE$+BUK?IHeoZ}f4PH7ry4z&1cSm7enDhRL9n_vqqS{ll&lGOPkspTS-#GGd}kibl+ zT4Wjni&S$Z(6Wg}VFC{#Ce13bQL0)(p;jAC4gQx8b=G9+WRRBe5^J=irlMT5fvOxZ zUMow3U zkyfn?^{Oh$YfjR`?`S2-=okfelc|wP1Mh)KMu4mqta`5;!cvUm2|nq}Jk=HIo)u^| zxP8{n2~Cn<)(1?NY;aVFroQW>h7IC1wRMjD<~;vQI%do-GNc znCJrRdxSMf_~axUR~!R`&B|v+tkGP1X(^VZs7|ot5;TpIie-_kV4N^hgU;Y%R18#7 zjdUEAoN&^)z;y|hxNN-TQ1WnrCf~r6DN-9OMkHgk%Ota+xN`{v0t-x19_E@)&V&-2 zkij_;vx!M@MGKdbs@NRYMNnT^Ra)U^bs<_}0 z5SiSVT|i&QN})9iTe&bfL_?5mfFc4MaNASVY#RTgz6C{HX3luHl*bGE7xQ3gO2V`D=0AwI;3tdI)uMCXthA#$G-Q8l}uWGfS(iV3g;rRI22GF235sKQ&@rkPQ!M2nl zkdU~LknljFF+Q3|j4?c#_$r3@;-ftH;Dd>n@TA80gz*oobMKyWx4p8I5EInQN$x!} z-}%k&%zVzwoHJ+60bqOk#UEb&`cL=o`_)_DZG+A)$?FFWclWVbz*wV68bhR1PXydpb+>h2`pFh*3mLN zGf^wu{AZXSvd+M)ZS~K0It?e{=^wwf(67H>-R# z|3($B{g!a=s`M@KOJ#k*#@eTX-{zE!&gwiR0C z`^)ms70J?v|JWRCpSHZ8dH${9Z_a^pgRA+Wsv=`x3^ej@Yunsx4#Qi@KdXGhdz8wu z@1&D&DC%2={R6!TYQOJH_J#V!X^;piSDSDvn*YyL7Ee}w0@yc9+X{S8Wx8(NVuWFV z-6>^D-#uUm>ceifCTCdlUM&tE)sP(!YqwxE4 z*J!)WDRtEbM!g;8|3jni^pEiKn~b*G$19;ipb#ho3V}kP5GVu+fkL1VCR1PXyd;06K^05}H#&U=!78#C<}zLdYDel&jb@(p$+@`(x5?+Vqk{H=`rH`d4V zTKSeXJs`q8~jLiT4Sm|Z{|A|jiF)RcMfkL1VCV@MDj5Vo8GfV5#0N$CXRS|!&8L_xj)bVv+w6DTqOIF6AOPNI;4 z^FDtL7%7Po1`$(|I-%GF%qH6630cc53Q_it#wqcfN9Y5!E;7&zi1-kYO`ISt1szIXS|h@F~8D|6^T)}1ex$6vM%rv&36O}gbSp#2xtQU zZn*P~V8$TUGKS`cuoMM$s-sR%SZa_|41hodrS>6`#%z?|WJy%A0-WHauNjZC7(s*g@s z9srlPn~SQWYByh@)-o!ij_VcAYt-Jxkns=R&T117a7r)su)iXOQ;`djPPM3B+_TQ4ztU z=)OXs?%f_dNn)BZ@hOLswSmJ+(H|1^||GHL>KVr zRLgr2cwh({q2FDl_bkn*y&lQz;mnvfB*NuAT^Nnemef;ZPaM?VP?XyV_cBD~L^1|S zLdSuUBqQsjYX_as5owtKWKL_!V~Egnti?**(n6BClhd3YQP`oR%vjQ$RZLtuGiRka z#23f0L`snkXj-KiQdkB{j87oKE(pxCqmm;=r9&!l{u&UsdW{x8pHc`h2_~RxHmsFN zC%MO@hyu+mFuw_H9g;>w8KjjoSlDPfWdingaf~U8v|XNdyMZ+Z^$=??B{9Je6Y>m@ z%T2j!F}FjkT2^R-g32F%5N2a?Mmrdi!kV>S)*M;hyU*f4;^<%ywH$`Weal&Wu#yw zGA|NRBrMH%9+A=auEfOW!~B6f+$HsJB2+8)y#|{l=5$MuW0vI3g@ZIpXHL8(2vtz^ zawl7w!7Sk1cjWF_WId(p;#_t_avDr@AM18GQ|tjuP$V2LFH(*Y&Yj$RK^#;jD2Yl? z;UkF=dY?5vbj#CmA$JZp-;K6+C!4~GEzppAdOqikq+CFL?WER$pmvw$ivh@;pK${5 z2^vItb-j0gWE$yPB3onZf|X(|as@;b`Lba{5`O+p-E;F(T(H&OdoszBXB0oPaCmuo z{;9wWo1gxE8=FML6#uH3w^YYGj^qvx8xK<=70*N4e!?DfxhpMPB$K+;&_w6U#)R~c zQ_GysBily4LMg=7XU!r&LOYenO3P7V7}O~_PYGR~)_4k_U>@2p{jw0AN)page) free(ctx->page); + if (ctx->moved_page) + free(ctx->moved_page); + if (ctx->row) free(ctx->row); @@ -935,7 +941,7 @@ static readstat_error_t sas7bdat_validate_subheader_pointer(subheader_pointer_t return READSTAT_ERROR_PARSE; if (shp_info->offset < ctx->page_header_size + subheader_count*ctx->subheader_pointer_size) return READSTAT_ERROR_PARSE; - if (shp_info->compression == SAS_COMPRESSION_NONE) { + if (shp_info->compression == SAS_COMPRESSION_NONE || shp_info->compression == SAS_COMPRESSION_NONE_MOVED) { if (shp_info->len < ctx->subheader_signature_size) return READSTAT_ERROR_PARSE; if (shp_info->offset + ctx->subheader_signature_size > page_size) @@ -945,6 +951,13 @@ static readstat_error_t sas7bdat_validate_subheader_pointer(subheader_pointer_t return READSTAT_OK; } +static int sas7bdat_is_moved_row(unsigned char compression) { + return compression == SAS_COMPRESSION_ROW_MOVED + || compression == SAS_COMPRESSION_NONE_MOVED + || compression == SAS_COMPRESSION_ROW_UNREFERENCED + || compression == SAS_COMPRESSION_NONE_UNREFERENCED; +} + /* First, extract column text */ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { readstat_error_t retval = READSTAT_OK; @@ -965,7 +978,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC && shp_info.compression != SAS_COMPRESSION_REFERENCE) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -977,7 +990,8 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ goto cleanup; } } - } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) { + } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW + || sas7bdat_is_moved_row(shp_info.compression)) { /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; @@ -1012,6 +1026,117 @@ static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, cons return READSTAT_OK; } +static readstat_error_t sas7bdat_parse_moved_row(uint64_t page_index, uint64_t subheader_index, sas7bdat_ctx_t *ctx) { + readstat_error_t retval = READSTAT_OK; + readstat_io_t *io = ctx->io; + + const uint64_t page_size = ctx->page_size; + int64_t saved_pos = -1; + + if (page_index >= ctx->page_count) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + + if (ctx->moved_page == NULL) { + if ((ctx->moved_page = readstat_malloc(page_size)) == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + ctx->moved_page_index = -1; + } + char *page = ctx->moved_page; + + if (ctx->moved_page_index != (int64_t)page_index) { + saved_pos = io->seek(0, READSTAT_SEEK_CUR, io->io_ctx); + if (saved_pos == -1) { + retval = READSTAT_ERROR_SEEK; + goto cleanup; + } + if (io->seek(ctx->header_size + page_index * page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64 + " (= %" PRId64 " + %" PRId64 "*%" PRId64 ")", + ctx->header_size + page_index * page_size, ctx->header_size, page_index, page_size); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + goto cleanup; + } + ssize_t bytes_read = io->read(page, page_size, io->io_ctx); + if (bytes_read < 0 || (uint64_t)bytes_read < page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + ctx->moved_page_index = (int64_t)page_index; + } + + uint16_t page_type = sas_read2(&page[ctx->page_header_size - 8], ctx->bswap); + if ((page_type & SAS_PAGE_TYPE_MASK) == SAS_PAGE_TYPE_DATA || page_type & SAS_PAGE_TYPE_COMP) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint16_t subheader_count = sas_read2(&page[ctx->page_header_size - 4], ctx->bswap); + if (subheader_index >= subheader_count) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint64_t shp_offset = ctx->page_header_size + subheader_index * ctx->subheader_pointer_size; + if (shp_offset + ctx->subheader_pointer_size > page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + const char *shp = &page[shp_offset]; + subheader_pointer_t shp_info = { 0 }; + if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_submit_columns_if_needed(ctx, 1)) != READSTAT_OK) { + goto cleanup; + } + + if (shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { + sas_subheader_type_t subheader_type = sas7bdat_parse_subheader_type(page + shp_info.offset, ctx); + if (!shp_info.is_compressed_data || subheader_type != SAS_SUBHEADER_TYPE_DATA) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + if (shp_info.len != ctx->row_length) { + retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH; + goto cleanup; + } + if ((retval = sas7bdat_parse_single_row(page + shp_info.offset, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED) { + if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else { + retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; + goto cleanup; + } + +cleanup: + + if (saved_pos != -1) { + if (io->seek(saved_pos, READSTAT_SEEK_SET, io->io_ctx) == -1 && retval == READSTAT_OK) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), + "ReadStat: Failed to seek to position %" PRId64, saved_pos); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + } + } + + return retval; +} + static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { uint16_t page_type; @@ -1041,7 +1166,13 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression == SAS_COMPRESSION_REFERENCE) { + uint64_t page_index = shp_info.offset - 1; + uint64_t subheader_index = shp_info.len - 1; + if ((retval = sas7bdat_parse_moved_row(page_index, subheader_index, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -1076,6 +1207,8 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) { goto cleanup; } + } else if (sas7bdat_is_moved_row(shp_info.compression)) { + /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; goto cleanup; @@ -1298,6 +1431,7 @@ readstat_error_t readstat_parse_sas7bdat(readstat_parser_t *parser, const char * ctx->user_ctx = user_ctx; ctx->io = parser->io; ctx->row_limit = parser->row_limit; + ctx->moved_page_index = -1; if (parser->row_offset > 0) ctx->row_offset = parser->row_offset; diff --git a/src/test/test_list.h b/src/test/test_list.h index 9b78c98d..a6e8526d 100644 --- a/src/test/test_list.h +++ b/src/test/test_list.h @@ -2383,6 +2383,160 @@ static rt_test_group_t _test_groups[] = { } } } + }, + + { + .resource_name = "compression_type_0x02.sas7bdat", + .label = "SAS file with moved uncompressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } + }, + + { + .resource_name = "compression_type_0x06.sas7bdat", + .label = "SAS file with moved compressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } + }, + + { + .resource_name = "compression_type_0x09.sas7bdat", + .label = "SAS file with unreferenced uncompressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE, + .rows = 4, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } } + } + } + } + }, + + { + .resource_name = "compression_type_0x0d.sas7bdat", + .label = "SAS file with unreferenced compressed row", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS, + .rows = 4, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "" } } + } + } + } } } } diff --git a/src/test/test_read.c b/src/test/test_read.c index 1bd93895..d3ef0bb3 100644 --- a/src/test/test_read.c +++ b/src/test/test_read.c @@ -265,14 +265,16 @@ static int handle_value(int obs_index, readstat_variable_t *variable, readstat_v rt_column_t *column = &rt_ctx->file->columns[rt_ctx->var_index]; - if (column->type == READSTAT_TYPE_STRING_REF) { - push_error_if_strings_differ(rt_ctx, - rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], - readstat_string_value(value), "String ref values"); - } else { - push_error_if_values_differ(rt_ctx, - column->values[file_obs_index], - value, "Data values"); + if (!column->skip_value_comparison) { + if (column->type == READSTAT_TYPE_STRING_REF) { + push_error_if_strings_differ(rt_ctx, + rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], + readstat_string_value(value), "String ref values"); + } else { + push_error_if_values_differ(rt_ctx, + column->values[file_obs_index], + value, "Data values"); + } } return READSTAT_HANDLER_OK; diff --git a/src/test/test_types.h b/src/test/test_types.h index 57839333..d0862844 100644 --- a/src/test/test_types.h +++ b/src/test/test_types.h @@ -42,6 +42,8 @@ typedef struct rt_column_s { long missing_ranges_count; char label_set[RT_MAX_STRING]; + + int skip_value_comparison; } rt_column_t; typedef struct rt_test_file_s {