From da58bc0791dd4cad6f243f5830d9e9e742b343f2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 18:36:49 +0000 Subject: [PATCH 01/14] README: what it does, how to install it and where it came from (#26) The README now opens with what the code does in the words developers search for, then the install command, try-it links and, where there are any, the repositories it was created from. It ends with a Find out more section, and the logo is a file in the repository. Everything else the README said is kept. --- .github/images/logo.png | Bin 0 -> 17167 bytes README.md | 55 ++++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 .github/images/logo.png diff --git a/.github/images/logo.png b/.github/images/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..4be7430064391b6e021bb33b74ecb4a2d4fcbaad GIT binary patch literal 17167 zcmeHv_g@piw{}1}f+$T<1XM(VpmeE0iYUGJj)3&uOHe@+6hw-&(0dELMTnw+bRqN> zkX}L{bdr#F{oZ^3i2Hf(FPq)TIWuQx=FFUVW}f|^qov9~$3X`Iff!yqSJnf8$bnmO z5DgXZa~L#s4gxWOUMN2`2+G=-r_M4MpYPsx3J7d64#BD@P*5oE2nMKp67P-`=P+MT z{#4uZ#&W0KUiDLV>Z={@H=~6o3#l0QfPi(mbs_GWXm?Q=(UTKmSbSiBBinDsz+qqQ zvz#Zo4OyDStZAv9X4TI98kw#$w?dDCWaC(h zW$BoU>#;UMp`Me^GEyIcKzEE2o%J;{U(UQlMLf>4Dd<#)qXMBeOEUVMi!F1@wYWf$ zubkd~M-{i*b}D=@dmcvtLhWA>|JrXjepUj!`bn=S#t@Y){?w*T@nIp^cRvC?z2Co> z^DYQfGmg|m%u>9@vfSaJ1r;whb9P`(PnbcV(oYPazzpua4{sip{?iDcB#9Pd01X{$ zNLoN1Ex9ktJz+X>)P6~3Ktk{nCBF+0NGus6XxBfI%$`3kVx3Uw`rU9Cvp?<*c zdj$g_>m4-+b;ji?#P#60I0azx+fz+^J1rT>S$Zg*{1oPDhMS3t@hp^CS5o3kxNKk_dlyBeA8{s zp#L%zA8we}q$!;`i=HV44KcwWAW))*BxOaYm&iDeh&u(KrK4Of)p?5$#tlKhm!~OWZc{!AY)-bu3G7zRP=wOn8<*8_B+63 zQ4uHrWbq!jMABacu3kL<@)S}AF8?R|zpggAXd?v#EgF8e9%E3dpt+cTE&9K!aXG5& zdav<_{4@L$+XDgpCAkxt#2Db|RYMEeB?H{9^4a<@`yG0q2miU}Ef-R{$R$aa82wM+ z%am1n*40QHR|!qx+y5k_9r~@C;np%0SGa-aj9>MB@uCIsUa~lj^zai%#{Ign9uhwypkqgU2!_<@p_>$NM=-L3MJmbnL?NY+O5*l!u zs0v7W+Jh(v@OSP=AM~PV##cDqRPhv;pg(-X+@ zg|>dDuIhiMe~tCuHt_1N67NX$>%O7*k_Rw)t7!2YDWGFvL5wcE$D8Ay%4GV_Dob$4 zp>B$HrqQx1zkN>zKZuQ_!tu`vts*QE{7?Tn6 zCp78WwW#0!2CcE*w~sQRtA=_G4xiwms+giEH0OQg;_GCf8fJ6?1bosNS8j}-?HoEQ znfH!iYQ0XTAf7TUg?nU?ui#ylac3z~cTMJ_r!e>EX$M{_*w}*qj3|!$v8;Rk^9p|x zXV1m6x|&nWtp_I+VQaVY)aE?GS(CmL>Ktgo^8QXPq+n-z%O47ezTnO+eg!$R4<6O4 zkb^kwTCh$wq?FQL`^zjAw6{zCM~oN#t-KSDh5JvT)mm|MU01k-l$&a3Ko=_N4QB)D z(cmg{u$C|aTWlkcBd6VwQ(O@?b2()=qaya`CM(^f)k)wWOj7&DXr5ANQE_3pX6A&; z{7i#U)flw-Vr#sB`DWvMJr%Zx)nLg7G+WR?W21SO)>tPZL$re3RDXpTVh|E0I)EPpYUNiKV(pksLIy?6T?Fg4_ zapfxoQAFMslK2@AwhFfXHg;q5L^GpUL^Ox%!NY6i6{V$r&W~jA@y9M$Du@Z>$eIQlTibhd1`uWL{1iIh{`2&QL9JVQUx}7)!1Z5PalPyT z4W6LuCm*k+P)38pPH&_{9C8`1tWKVHGV_ac z_wTBfOHtC%8%lfUt1SfSvsM`K7i-=un%~#%O1Lc;r4{rMl$K{QHhA&JsB)g0bT|zr zp3*z@TYX<=ep69b=QCUSJxLo<ZDT%JDTCUR9`y!GPqPLKzG;Z*E zFw+osJPQt9w=}G_Mq|jB0Lxm>-Ek{Nv9Wu36*v7!W>DqEdEpk*9`~=^%($m$ihR6GS z<5ueD%js%osGOB0rxF!!Hihd4yTZ?DwB<#KxD|bE*Qve43^C_AQA1GFO#~9MK8D(w zKkM<^$gP~T_S9RqzF1~iwem~!(Sj71t$V7nn6*oem~N=9qj8<~R~G@#Z=1@VkJ=0T z(e?ESJ%uNshv6d?x7%1Clw?#(#LN7=5Uw&S6W&Sf?2NfB5_sWTflNy)9YYdSUX;MV zQ3*BI$!Jz;>2vRU&m+9jv?yv&k^oEaYWVTS1u;I4u1XQgBa26`Lo&J(@JA2 z4}*29t_{=8|Q15~fOAmO+D?&!@spOXsS-1Hn>L+9+w5hstGoa9MacGCXW zI<-yatc)s6AB{acOI}{ZhkRkWK}S#TKQ0a($!K&HJN|Y+5%mT086$xR5$25OZt;$V zoN|jBY*Lf-mfBCipL492w}mk?{L2@37Ri_InL0WPSiC+UTv3y}`!$zVUOe{-Ctf_a zQ@@P%KatBo+T>jXZ&$@8lms!o>1UhhZ9mk;4X5eCPiDI!(i0%XQ}W?sOeqz{0;J+h z!wZylV+Y^G_e&`eJrI~r2aV@aEzKX*bt87f^*<2LJSNixDCtrP%h!#IP%IAx_bafk zZ;H$@xsy{qxiPSZ5n*OL_ltT=h4#*KcNA6AP{IlDx3sVx}9Nb_|hnd5n#5 z+g*8LiN8G<2rDRU|N0x04<0>Y|1;;72%GaAt_WPuImIU>pZ$$v*d?f=q1LVJ7ACIi z*esS;Yv-GUyYz9OyODt#yNW9zcvztS2(M$%1eX1e1^L<#WPQ%pu=eGP_Jy3o7{~+6 zTIRS;bYO!1>l??$f{j9MoTSQ0z6bKpV<>ro%w$1RxTaEbLaAKzZ^z+l*Gaz%$TQx` z-MrT`muURZ+zNiemmSfj2S*q&Tz&jn9J^IvC}OKvh5fKO_aYtMUOwe9_V9zU;B63C z!tL7D8U}6dJugk4*sn!RX>RWOKUje5#fpB&>i8rJ4F!dmnQ1yssMUa#o^5O!^P(iD zxB+Pme@By?3dYv;uhz#LkKJi9e&mpjU7w2QjQ;(LMIq|>lUH0kIJvz)KQm+kvm?8~ zEtumF`0>fEs2TsHWy!lb81$@grkQJY{*vjV_R_S8PZb>R_x}7vfWSu8Mvs4CTVu@H z&(l6U8AEt@!c4RAqw$jnb zg=G6y-|rx%h8)^>^KlgKKuCt0FR;{#%xnx*qU>& zh)C4$iN9prHfwcn?g&Y^NJ9mh)cJzHBn?bfm?ZALsa+RH>+2_G2>ZTU2oB7bC+_&3 zX*}7B9sv^wgs^$<$@!4;-k#O{Ad0aVIko0RE>bBv(>*b#hf5SG`emGn_xP|wTfU6Q zP+O}8J-oX9z*@P34a;-ML7q}cBXffzBi0%HOqibmZ(IF=k6)LT`lHb~gRwxhIlIbT zxTMzIcULCo0~hkjAX=lL*p33|LPX}XcO&3cIP&aS0%X51@*I4+&Pg{`|IIUlmaf(k zQO-jUbt>QMz;B0X_wlm#%h*IRahioud`AXFzu}p==Axk%`URJX4byM_^K>uh1f$cDUYMW;k zYsqp-ho0k4Z=osqKx$XIc`YdXuepn7}pCFY%`MOlsR4(CbJZ64&`T)HMA;mq~uEcus-tx<3 z3eRt1RB$`l*tsssg)nzt2z^=xl};E!-kA3y`iESkJB^harYgD*&9#OcpRohh3Jp~x z6edC8-4BQx1(qE>GHz*1;bvkWiasJ8%UAGq*$=d_&jMH>89AH`Q-SD_kgX3rN*xb| zg}&E0yqT;ty=&uiVL349nG$61GiC$B;&!&uiwMi9I!V|-)!hZ9uxhn$+PrUakEHoz zs}!?^k+d@IVfrd5$j?90b98Gf8e!lu*V2FT3kXQGE8dZY4u;FoGj9~US0=fh`U~C! zx#=Plir(-v2}XneJV{Sa4>>rR9BXuawSDi~#raJzy`JS|Wl8KpXoWDie!H2fLL^3M zHea`2i|jOvj)f}k8n^kvA7$78&G+z=xq_&uKbpLP*0qW`;TNrun63uTW}xURT-DvB za~4QTlZ&n+8K3KkykK^+XRgsQQ4`re-)$wvaESzq_`Sxh8e1_`)OEcIKH_&V%K5C5 zbWRo7HbkYc;xY}=(RF@C_GYhNDE=x)3^tz{_~$kClNk2szWlDZ#I?}~Mk#(N<_8Op^I0UfjAH8xcALD`6`ot>TE_MoKHHo0P6O^pue z+-q{SZt?a$Fr?w+vY*`!oA~_mjT$;c{k@Q`6eUwq^1a;Fd+FE`>Rp8K0#nMwmxY%K_GYwum^`c8aoVUyhv|Dppe1)CzW1%6sjknK z5u18aqE&F3Fd=kd88}BuHN{o($u=W-Kwioj;u4|-H51~K&9|_9E9&ZOLEN{u)ETy| zaur$=1&y!v>R7A{5XzL*yjZO;kPpo!sHbM;#CJ~yqC@tyZ2EgxwwNK7!Lyc(;d&g~&55Re9#fOP}VxJw(Dg^lFH1(CLKL4MT$ukW-tk z0Bm*aYQJk(d)T=juUP)ITt2B6-l03XSf8IpKaQZr6SJPe92}Hp4JF_lgu;(SQL@~X z_JmdImckb>lF%6uH*8+|B~aerLhyti-%_6(z7-8V+kli0^froHMQ>ay*T*{!W((Sc z@5egwRcuNHD`tn|&EHA1{_%%(Fli{}t^O$u zkN7d+Kv9o#509k!Eo~VAmBQmWU+2lS8o??4wQtH!3ZqLVWY+05jG~N%J#SHR#ek<#K zkC`+5@ToM&^mx+=Oq`5SeCaZ+5_!Ph@7@Q6!L$0Sec2d2=4~T?+g(!VdA8H)QCe3g zvbw%bKb376p5u*`18l;LsvqA}{gmi)G|o_dxy5j)#k+h3FLtMKheqMW^1F&VFu`ZZ z(1Z3qr0X}P;J<(WirFaqJh6~8NT+>INf}KZz$W?XvmWLaL2GBfqw7&eVfTCIaW#Hu zhXsR!>g`yraD(@tf1PNJyxvkaXbZ67x>)@8-eC+RW%EO=o@KX zSU~x;?vG{1VIv2_>D)R<{fly9x>Gj4=iPWVp71sai5Iez7MPl-wiL(}65ibG3p+Li z#$x2p14ivPkojMd1nmNyXj;U)1kE z#hR{UOLv-_hRc~gCpTbP>$K$)vD@%*ld@*Ck^9R$t+A^24+JI2KYxX%jj`CO{zFeT z&S=Ra-^Jd1qO+MG|2NB^&G6!rr1<%e?0sI9O-FFv0v0=^`~DX2uHP=Dp!TX!YSDr|u|U6*^#u|3-S3i71}>DvKHCM?#xMno zY*`&<1f4D^zND>CiLAQKRq}Bbo>XF(%AL}2*1^;v;ck~fwKRBxc1oJl#^V8w6yRQP z<<0>khrzaM09if^tniW+ZVB4oSCd|S(n#E31@Hvh3;)4`;e(*|*j|mm;v0Y4{B_-$ zhh)U?78dYsxtWxg(Io zNWy@9mJtzxVxgmzE!u zA0EJ61XH#@oPa%kb9Y^tx1qW>=iB@}R^L~=jd!2uWSusLy zlh3+Oa+A++?OoHu3WmEX@Q#9HNcO|1um0PU6|%oWvoU3FKdh~TUNKsUCghxL3H03z zeI0u{Sj7 z(4-{86Gp%tJfds*^-I~Iu$~GuC{V0K?a)rBQ?m-y>^;gtoD64`DGbH0&WdISSu9+= z%iVdv|Dl(RiZN+1S<1289@Qj_{@5Ysb41QLi}%52mh-?iAJ08N7V;bXWg zy{botQUWK`joOB@V{5HTwUS~l*0b-Xq|Ow!_Md!>I9>AOO4BX-7{leWcA>P2_N_nq z9Vl=#UuVI3wk1iPd5x+H{=OuLIm?)%rWU5_ybK)m0xy)UV zn4X4OoyaZ$A(0ar?fYK)u@o7|*8XSrrx}(v`oqU)h+m)y4TFpyvG` z{C;2$4GK!H4Qw0y*SUaRm}q_h<3`G6gO`q1zhJG0v-)th|? z;&*8otUsl^oLq3@?el;lAGH8N#tgSZ4m5C$ig@zZc570?v{VuF>donrQgXl`FZlXN z>6>1NQwNj$f^eYHD$kRnoyE6|Y^b)RrD;dtchVJ_YVk?f@+Uw<NqV4d z=aUS!;2o}uL~F@S)Y-Ij9n`h;23hWnUje(NX^a>R7sk=PJtnr>ikP(_SV@_F37F*_ z_C}lG#H(+3q*WfG!@|g^$Fu!7=IQy*n&ViOCL@K?0klb_a|K=U&L`cr8>x^&?SorX zPO2tCvn6r6l0Pc%FC#5gKuj4b7Q@xJD*B6w#|p82Iql6k+DnU=L4+Q%1=MjOs%&e(lMUZQs>BoA`H&9~%Bab) zu-we5sguS1aDN@_;w>`BZ;IHVP1?rl-cwuC?D*S@3%Ve31>X-vD=!<&T3u2++E&>_ znVAV?*Ud?!eEM#>rR`{bSB|kW^Cq3a4b_rz?d^LdrNdxJAC9W5F7%+L&wobl0Owg=V-7tLM8+y> zu8d7sH2@PCW33cVrRN7$Atclc2m4n{Z2QfQgZ@>mda5RMBW-57nc`nEG88*reibi0 z4Mh_~HTuPPctm^UjSt|i<_}AzjGYxwkI;=qqJF;rqdP}L5m{XiqpEu<^@>^(Yw%#}67@Jb|c94k_v9_JM z!a#gp#vtD4jkkTebQ2V|$|JK0moo4eFshk)E5j6G4LzvxJ>dceph>~&S%OhWvb7;% zJ$;OH+^l?hb1(n7eRFDSN|ntN(!K7TpPH{Uk;l;k2~A6$YjkCHLEY`2XC?b70!j`$ zO%Joy*6R*$exP^C{?MrASCa>C)I=<+z|mLag=U`Lud6W?p(g;yTXVGOPHR2;&>T(r zDo%OsZ<*-phJkO>zZFTo4L)XU4R`rAQxq~jq@g?2!--%^ zXUKMJ1zgv|qgt`cgNXmRSwf%a4)@s9nqBDb^l_@4CNLP9qa!js;bt~Y&H2-|TRN1{ z@x^yNGtI`+XPK!m=WmaXTUk)OGO+ORgX4@;rscW_J2U)OAVv3Xo2%bG^?{ zo13|l+3`G!yZn)&Lpv_*Z(D$#2_1#O;T#2}zQ+0=)>SVAyyM~-yMH+k&&31hnxYS+$P)i{VJ1(9@&1Xk(JHqtv#`1zOjWFA z;?xJ>>@47uN0I_Shy&|cDh%rUm5T+vch*kKvoULVl?d!*skI;C1i)ahUMY-#Pa11$ zM5uouA1wR+v2gBi`iq%0ay?DnC8^Wa-1aIc)+*E(Si$b#!n@$wW5vrudR-a*bkrTa zCPAs4c0tsp|fC}Piv;r7b! zGUzpr4(rsCezdvH_O@qEBccrvh9w=FJp`S86*3-bHGVQnErSvKzpOSv0d zrDGQCdyoQMmK$p9B>;Qc=A~Km)w*+wWNPwfPr9(8ir$u)f$=>j2O6x%i|ju63q>+t z$r}LrF5QbTx6bpBl4=DD3XHe;Z#M+{#sdrDiZ1okC-}4V0a%6C;aAscqpv~(N^Oaj zxsLrbvj-GSZRe35eixcGcS;PX<%BhZi^idnv+(U-z5$aw;TwcNHXOD^Azi!G82tb} zw&~(=Vd|R#sn}xg2wc%AvI>j;_A}sX&vr^lo%UZ$evLHtV z=HQfEQyzq|&s8NY_3aMJ|K0oPzs1i-ziXe+#d*+qx$5`4_iM9ejc0E-w!e;3N^iB^ z5YK-8s)FU>BtLI!q;H-Q>cWfz>Fg~g;fSa8Z>k6-n|;E9#hL=XeDk{A^KB2Bj01y1 zPaY39rt%epgol&9tZJ!R*Lt)AP(_*Dh@PR_tJd>{7d;}7Tkh|$ttH*_{w0g@0zT^g zrzqqrd)=e>t>~PIjlv&^?emlIZUa`vugntHsvr4iv4(kRcuk7Qo)f{uGh)U%A%Sb^ z89Yn)H~-|vjOd4;DOOSuGpXt?`&*>DM`%i>giKY~IXw2Bx8nw630SaijMUnz%6XyX z)LzggDCseoX%lMNx`)u`uRNNbeaa~}PuM8Ci;oG^qzR4M{jzgy5N)#KcZ`~3c{MT9 z=~BQkF{$ozc-VW%PnopF8Sw~MUld`7Q0l01Pg-n7V(vcQA&M5E4{JATiZDFst@HFyXTFTBp!&-97~}XA+%G>C?EWHpqcwjn(fhLW(pDJ z<*W+J4M+2})E@uZbp|&PSm<%;r4Ktg;Z4q&vHh$MwKdGDPMkc&?%rzmTPp32Uc|*l zdJx<`?Be@0#;GDg0qhupomGKK)ga7x>p$1=ytx>!gGKD3J@g#{;&rY7dt>+>L)>GH zTdZy{=-K;GAeq$(C`5?CzL9gz-1kGdjj_UUsDqYukVYx8eI8hUN(JM^A1! z3m}4U3Ct1W@NKBTxV!cv0!rS}uPefG^YkWvsdb~miw{0R#7zA?>Z7EfUq95QoC=xF zL_|am2lTaFhHaH7P1dpFx}2zG|JqF&R{_JDpLPu1F(^VhMB_v( zbf2!cJ5pJ%gx0FQ|U@;qk<+W`Mp89`SUZq)l7bRnG)#S zcZs&S%ST)`iWC{DKp}*h`4pc`sedO?Y$rvyTGhRYq4bGW?z2r9dou!Ce>&PDXDme8 z>QOtg1gC@gqt*OdM5iB)PPLYW&p#vx2d?zaR9VJ4_imvjOn*w3lv&SqYuiTDK2~K% z%4i3#ZTpNa2AmTYo1_D{*BvO7{_tt(J(4r7#ms`SsrY+2+NGv{_f^yKNLLkK%GeI< zw=JD-r=)aQ(rxurZ$ZLeg_fAQ;`$GXqq5=64~IK zpAR^^G+R;%-#kS)N?>n145043(tmK+cjJms zpqR^g-Iz%!deGH$V87RYgwo#s%mI4s4@&?7&95{g5!@3C|gqK zLJ9S5tq*^MY*38uICmeZH(whC`mSrmOL(2eww-kF?8vOEF={%nVU|&oI_DZ!8(KX; z82u^ch)MD8OLe~XOZc*JSM3K_cyoD}QP6~wCvDRL6aZy>g@3D@57@?zol!?Vzls}75;nb|hFpcm&f0&*ks@N|`jRs8D|>pRoLISW ze%J|3ZOBnn$jKHj=)JO_(Ierr!h-R0>qi0+V_vPAgx`5C2iQrL+0?vm&rwKg$b8`N zlkh|M$tHnJxq(eEX9Y89US-7r&u3rI8S#Zn#T+X)PMPrLc;}Cjyc?NJr2}^>nYFJM z(D!7+0py=dp^!fA`~&|wth$gn-eBqHQ;aIp|6l>b<)@!cSU&B{M|7XcupoOBczj+u zI7s!J#9!|y8AYKeqdHes&w!QtGt0;2_4+DTXx`QE=iW+6DLXX3(EiXv)Z+Dw$Q>Q- z(93;Dr{-|~m%!>Fb4u88SuAG24*C3ZMn?HhMmc5G+Yv2{GCMkCc_Q8u&*o1z?FOfH zDnG1WEu_Kwm{o3rIOWS`-li z;gvFVUSlN9)elch9SpNFsX?j+CLNg9w@VBuXR~Dz4|cA;v<;2Yb2|}oA)Y7U1~V$^ zv^xy;qdr^}y~Bh=KPa;DOVwYDArhDR88@+VF3IuII2xx%P`H)cszWGPaUh%4t89}H z=y>IF+rh?y3+&bcI>~o}U8;v^mi*cKhfCq_@>@mgeb<_btb+4m(dWP^s)!Pt^}2wL zPN~m)Maz*D*ZrhJ&T z|JIc5CF;w6a-Uujf0av{=%{vILP*OB^G<;$0%jbdZzS*CxR3m{dNog(0a4Ej)SS>gfo2BRe;LzFq zDaNU_QWBoCKLc-DO6qFc43zU9dLOTNdO&em<#A|JBy{Q8+1H1GYM1fsX+3eNR1R(2 z)y9Ec3^SSwWZ{F~A|3@(9|f{KdIrWf(kD}6KVJ={Np{(AtFNzLbzMVB4$ZC3>YwUB z>DdK+s>AVzSIxRJQIp$#C~t{oiC!zGhTN)F(S*3N3sDp22n<}5vkkKs7nb7U5k7lw zbLK6=b;k-8wa2nb`=uvL_+!~?FP28K+-Iz;bSy0VxosrA zo|itCdGoo*xMf)YTySdLaU-*+HP*;d^xEOb6wge7eL0i-96%~?Qv%UEY7Xl#IEiSq z=vp0w+(YXz?xd8_Kl2~OQa9^=Y3>3K7qC%)z8BGemYnQuTW_C56x}A-rqsm|W0k2- zhDK=-%_A?0QP?j(Iybxk7Jx-8h-tx!)Xzp{P_PIuojrGiA z{|+>_u#j;-0R~y;MrYntRyFnas68QP7dnI)$r|Y~TDTf~?)YM;sr^K-2tX+~XPV1@ zHV=AeD!&9U2O$;Et8asjK^KUk*7IQh*N&C^fAmoNaKQ(tcb%Ng&5YQr;t75$Clf*} zu?;NMv3al;CJ zEtj8{SxNcS_TnI&E68er(J1Q)0H?}?#Ut&f8*PS1QHPwH? zsR{P;kBczV?D!Lf$X`oN6b$@&;=mgUbB#F+Dk(@88?G{yWeqGb2;zsl6-6=Vb3u5nuf18IHfM4)B7G5a5Qs{sF&U#1A&^-_WDES zk5)wnC4LgcYwnU%A5(hoowjw~vaPiczhSz!yR@IZE=X(x|4-!DfW54RHa+YO>o_`#_ z%U45{D+GYN)#4Y3ls4SkXvLTHDoGw^VISDi5^q(w#~NE($ZEWmE~IG$j+A>9!N0Cd zAP$Fp^hnwI3* zX0mJ?0EUXX7I3(rZ{Lu14Wfq0H?-IH-TGm?Qt@ZZkn-Hms%as+^#^}QgQ&UN=DAYa zb|BG=_ULEeBam_Ihy7CF4#~jmCQICI7-n4DvH;=q1VMg`O)qOIv&wUJ`%!duGfq5Q z^L72rpGIbtRRnKU->=Z)_wj9B27Dzxwo237BWc5NY#)K{xO#jQWZn@tcAC>rTWT>_ z1fvCAnkg#v%zSy|oK!e3Q&?V-E^>V;yY*H7Lhtar&Q7v$6AWHukOnMRB-2S(+>Sb{ zd`}TRI3QN?e96Ztm)pqUnty|6YQ2FfDv;@~q5(y!>(c6&C0Q4vtkSEn1DxSC)8((j z&G=+q=a#f?H_(sI6m`o~eF~nF;FZ6l$`l)((#Ov)IfymTvPd*HjGfBXuaPY=xT(qK zI;>TLEpshVu{7iD+WgZ|<@b^_A#Hoo6h!7W?@S|4F@oA6lY>qN1ytNqVNTFR;~?OIQ2czDElj9x5U=M4?~R{X1{57jb# zS=H7w8mCqv?HPJxxHH8&rO$wIZ8cx8&%_L18l_aVo1CuPof+0Bn<`6LlQmvaJs=&Z zUhJuu zP_NtTvo57j1!@@jp8M#=-#&GWzL1lgJq!Ar{^e}5FHtMfN0g0>{JR!_!THY&?R;)x zxYXpcI1%g?$^vJg<-O=;@I*w*b=8CGea#i>`?+%{J};U0-z<~wWT4Z>j$a5*JOe7I(ZwyVbV8Xns!=KxB=gw#>_nSW^2c~HHx;NY&?Z?!tX>a7e5 z|3+8VkTk}H-GCazZL{*dp@*V5By_aYkQ5_H6c&*O&z}k1xzo9>#ku&9`8R4WQ$1=-@CQ=_Pm`*dt9t+xZBzRB&)u5f`cs8QX^;+STJFJ z%)>x$Irmw~A|E@-*v+=J?orsu_}j6+oIwzrE9kTABV5bLTk4O?_S?eHLTx^z>YjW8rW*}CTX5~-zWy8XBsGS54P3+xdF8 z`iA7V585WgmuXc+hKJ7Fd1|Vacu$OR7DV>d?xxMglnmPQ>)O_*&24eifb*l_st&I~ zArp0U-Sa;}CiSc!M4HVXDY+jO%wxS4f%3=)6S9e^OzQ)_#*HmzC(YQFAg`XTH^(mQ zEHkd@q~_5MsV?*CS@~n^p4sC-{;7HDCav!t?)qZ6;dKW=2F;h(4l}Qk0W%7@LO(x% zBm4sv=0|1ME%02%D=PkFj;WHW0Y;^erSLFZ8!}jcA2Hk~u z5Zpsnk{+!yQU{d4*yp6Ql5%IarAR_Crh%eS(ip*(pP7DK6aD5qO$#aNde@A_6(CSv zp5lSd7S1FvHu-~^7=OwnV94an8TWgeW6s@zcf!6EIV$+p`BS=eAq_;w7cG`6in-jp zxe-Uz1?j~#zPMxWd)aOq62Oe{p2?h^_DE+%h(q}_ms{k26Vor4{2f3ks(4HeT>p=% z`2PWMQv#Gs`;XvgQjk{-I=p?t$plb^jhxNQMJa2hH}?kLy7`_N0gjmw z?kWSt+u&s`*a2d7{&buh;2*(wSyB-75yUcmrRftWjgM|{)$(8J z;qJr`dM0=T9S0aXvXO^@O!0uyUhbu|h6w`GhGvIEIDt_d>N0a2_>X@sRyNTb8W3@Z ziDqiOI86e0FYZyI+YF5;In&fdFF=AeU+_ZylWPfIA422C4z)z-m@3NkUVj9#XKF1@ zm*}HNQa=Pf9+3R+djS-Yu`uhsvEv)}DW-JjZjJX)*&}g)P)Y-8R-3n-P)!*VAk`w=VJ7cBXvb%z!;0*? z2+#NGOQ$~?Isl889P^yB)q}g0SFIz$k-04Ve+Yi=5KL3>MHsX7Zhh?;naCfSRB% z&vv%gSSK}R9X!AE(J0%H@Y1!f>8nL(M4;jkBC`z#8nbuP zU`W=nt3YvAU*8>A^(^MR1AHbSWnMQhXY&wZ^gc=q+`kZG)4&n|G6vkMh7DQkd293FFS3t9(wz)B_w;VEH z;2IQ#8!rV&xsDvQ6VfuQiJ$U2?`OH+2?p&9P`r^ZH&cG%)!!IXuEtnW0lw7$IIEyj zQn~b57|KV}+;L`3&^>;0kDXtG2m$ -This repository holds the 51Degrees Java packages. Each package is still -published to Maven Central under its own coordinates, and the examples show -how a project of your own uses them. +**Java packages for the Pipeline API, Device Detection, IP Intelligence and Location** + +Our User-Agent parser for Java supports Device Detection, browser detection, mobile detection, and crawler detection on-premise or in the cloud. IP Intelligence adds IP geolocation, including IP to country data. + +It reads User-Agent and User-Agent Client Hints, HTTP request headers that carry browser and device information. For cloud bot detection, Human Confidence reports confidence that a person is viewing the page. + +## Install + +Each product is a separate Maven Central package. For Device Detection, cloud or on-premise: + +```xml + + + com.51degrees + device-detection + 4.5.x + +``` + +The product guides below list the other packages. To build from source, see [Getting the code](#getting-the-code). + +Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). + +Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, `location-java`, `device-detection-java-examples`, and `ip-intelligence-java-examples`, this repository continues their work. | Product | Read first | Documentation | | --- | --- | --- | @@ -67,9 +88,9 @@ mvn install -DskipTests -pl device-detection -am mvn surefire:test ``` -The cloud tests need a resource key, passed as `-DTestResourceKey=`. The +The cloud tests need a Resource Key, passed as `-DTestResourceKey=`. The on-premise IP Intelligence tests need the Lite or Enterprise data file, which -is not in the repository; see `ip-intelligence/README.md` for how to get it. +is not in the repository. See `ip-intelligence/README.md` for how to get it. Tests find data files by searching from the root of this repository, so run Maven from here rather than from inside a package folder. @@ -106,6 +127,16 @@ with the plain version, such as `4.5.127`. ## History +| Earlier repository | What it held | +| --- | --- | +| [pipeline-java](https://github.com/51Degrees/pipeline-java) | The Pipeline API | +| [device-detection-java](https://github.com/51Degrees/device-detection-java) | Device Detection | +| [ip-intelligence-java](https://github.com/51Degrees/ip-intelligence-java) | IP Intelligence | +| [location-java](https://github.com/51Degrees/location-java) | Location | +| [device-detection-java-examples](https://github.com/51Degrees/device-detection-java-examples) | Device Detection examples | +| [ip-intelligence-java-examples](https://github.com/51Degrees/ip-intelligence-java-examples) | IP Intelligence examples | + + The folders `pipeline`, `device-detection`, `ip-intelligence`, `location` and the two example folders were brought in from the separate repositories [pipeline-java](https://github.com/51Degrees/pipeline-java), @@ -120,6 +151,14 @@ SHA. The history of a file before the import is under its old path, so read it from the imported commit, for example `git log aff7c544 -- pom.xml` for what is now `pipeline/pom.xml`. -## Licence +## License [European Union Public Licence v1.2](LICENSE) + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- [Bots and humans](https://51degrees.com/bots-and-humans?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- [Documentation](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=find-out-more) +- https://github.com/51Degrees/51degrees-cxx From 99217cd6dfd9b27ad88f0dbc63e88838936ea1d2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 18:36:58 +0000 Subject: [PATCH 02/14] Check that only rc merges into the default branch (#29) --- .github/workflows/only-rc-into-main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/only-rc-into-main.yml diff --git a/.github/workflows/only-rc-into-main.yml b/.github/workflows/only-rc-into-main.yml new file mode 100644 index 000000000..69e19ca3a --- /dev/null +++ b/.github/workflows/only-rc-into-main.yml @@ -0,0 +1,11 @@ +name: Only rc into main + +# A pull request into the default branch must come from rc. See +# 51Degrees/common-ci .github/workflows/only-rc-into-main.yml. +on: + pull_request: + +jobs: + rc: + name: Only rc into main + uses: 51Degrees/common-ci/.github/workflows/only-rc-into-main.yml@main From 9ffbcf87e4830e7bea685dda4449b60e84d80ae2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 19:53:17 +0000 Subject: [PATCH 03/14] Point links to the archived Java repositories at their folders here (#31) pipeline-java, device-detection-java, ip-intelligence-java, location-java, device-detection-java-examples and ip-intelligence-java-examples are archived, and the note at the top of each README names the folder of this repository that now holds its code. Four READMEs and 33 example source files still linked those repositories, so each of the 39 links now points at the same file, folder or README heading on main here. Only the URLs change. The History section of the root README, two Java string constants, the creator context demo page and its test, and the four parent poms that the open package text pull request (#27) changes keep their links. --- device-detection/README.md | 4 ++-- examples/device-detection/README.md | 2 +- .../devicedetection/examples/console/GettingStartedCloud.java | 2 +- .../examples/console/GettingStartedOnPrem.java | 2 +- .../devicedetection/examples/console/MatchMetrics.java | 2 +- .../devicedetection/examples/console/MetadataCloud.java | 2 +- .../devicedetection/examples/console/MetadataOnPrem.java | 2 +- .../devicedetection/examples/console/NativeModelCloud.java | 2 +- .../devicedetection/examples/console/OfflineProcessing.java | 2 +- .../examples/console/PerformanceBenchmark.java | 2 +- .../fiftyone/devicedetection/examples/console/TacCloud.java | 2 +- .../devicedetection/examples/console/UpdateDataFile.java | 2 +- .../devicedetection/examples/web/GettingStartedWebCloud.java | 2 +- .../devicedetection/examples/web/GettingStartedWebOnPrem.java | 2 +- .../ipintelligence/examples/console/CompareOnPrem.java | 2 +- .../examples/console/GetAllPropertiesCloud.java | 2 +- .../ipintelligence/examples/console/GettingStartedCloud.java | 2 +- .../examples/console/GettingStartedCloudMixed.java | 2 +- .../ipintelligence/examples/console/GettingStartedMixed.java | 2 +- .../ipintelligence/examples/console/GettingStartedOnPrem.java | 2 +- .../ipintelligence/examples/console/MetadataCloud.java | 2 +- .../ipintelligence/examples/console/MetadataOnPrem.java | 2 +- .../ipintelligence/examples/console/MetricsOnPrem.java | 2 +- .../ipintelligence/examples/console/OfflineProcessing.java | 2 +- .../ipintelligence/examples/console/PerformanceBenchmark.java | 2 +- .../ipintelligence/examples/console/SuspiciousOnPrem.java | 2 +- .../ipintelligence/examples/console/UpdateDataFile.java | 2 +- .../ipintelligence/examples/web/GettingStartedApi.java | 2 +- .../examples/web/GettingStartedWebCloudMixed.java | 2 +- .../ipintelligence/examples/web/GettingStartedWebCloud.java | 2 +- .../ipintelligence/examples/web/GettingStartedWebMixed.java | 2 +- .../ipintelligence/examples/web/GettingStartedWebOnPrem.java | 2 +- ip-intelligence/README.md | 2 +- .../java/fiftyone/geolocation/examples/CombiningServices.java | 2 +- .../fiftyone/geolocation/examples/cloud/GettingStarted.java | 2 +- pipeline/README.md | 4 ++-- .../java/pipeline/developerexamples/usagesharing/Main.java | 2 +- 37 files changed, 39 insertions(+), 39 deletions(-) diff --git a/device-detection/README.md b/device-detection/README.md index 575ed61c9..c393a561e 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -212,14 +212,14 @@ or shaded. The names come from an `Automatic-Module-Name` manifest entry, so the jars remain ordinary non-modular jars, build identically on every JDK from 8 upwards, and nothing changes for consumers who stay on the classpath. See the -[pipeline-java README](https://github.com/51Degrees/pipeline-java#native-library-access) +[pipeline-java README](https://github.com/51Degrees/51degrees-java/blob/main/pipeline/README.md#native-library-access) for the pipeline module names and for how to configure element builders when a jar is on the module path. ## Examples Examples can be found in -[device-detection-java-examples](https://github.com/51Degrees/device-detection-java-examples) +[device-detection-java-examples](https://github.com/51Degrees/51degrees-java/tree/main/examples/device-detection) repository. ## Tests diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 8d38975ed..5d2edd339 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -150,4 +150,4 @@ A fat jar cannot narrow this any further, because everything inside it is part o unnamed module. To grant native access to 51Degrees code alone you need the individual jars, with `pipeline.engines.fiftyone` and `device-detection.hash.engine.on-premise` on the module path - see the -[device-detection-java README](https://github.com/51Degrees/device-detection-java#native-library-access). +[device-detection-java README](https://github.com/51Degrees/51degrees-java/blob/main/device-detection/README.md#native-library-access). diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java index 573ea86b4..c97a1868a 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java @@ -158,7 +158,7 @@ private static void analyzeEvidence(Map evidence, * * @include{doc} example-getting-started-cloud.txt * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/main/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java). * * @include{doc} example-require-resourcekey.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 113be2a40..9d8819565 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -184,6 +184,6 @@ private static void analyzeEvidence(Map evidence, * @example console/GettingStartedOnPrem.java * @include{doc} example-getting-started-onpremise.txt *

- * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java). * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index 26a305721..a54bfe71c 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -235,7 +235,7 @@ static void run(String dataFile, List> evidenceList, * [performance options](/documentation/4.5/_device_detection__features__performance_options.html) * page. * # Location - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java). * */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java index aaebc3151..8ca360fab 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java @@ -147,7 +147,7 @@ private static void outputProperties(DeviceDetectionCloudEngine engine, PrintWri * keys that, when added to the evidence collection in flow data, could have some impact on the * result returned by device detection. * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java). * * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 677b322e4..a58e739ab 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -231,7 +231,7 @@ private static String truncateToNl(String text) { * keys that, when added to the evidence collection in flow data, could have some impact on the * result returned by device detection. * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/main/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java). * * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index 0a5f472b0..e920f8c9d 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -133,5 +133,5 @@ static void analyseNativeModel(String nativemodel, Pipeline pipeline, PrintWrite * pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). Once this is done, a resource key with the * properties required by this example can be created at [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java). */ \ No newline at end of file diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index d044ae365..031c63368 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -262,7 +262,7 @@ private static Map getPopulatedProperties(DeviceData data, Strin * @example console/OfflineProcessing.java * @include{doc} example-offline-processing-hash.txt *

- * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java). * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index 72172d4e2..aeaf3180c 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -403,7 +403,7 @@ public PerformanceConfiguration(Constants.PerformanceProfiles profile, * and [hash dataset options](/documentation/4.5/_device_detection__hash.html#DeviceDetection_Hash_DataSetProduction_Performance) * for more information about adjusting performance. * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java). * * @include{doc} example-require-datafile.txt */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index 0db97d885..91213032b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -138,7 +138,7 @@ static void analyseTac(String tac, Pipeline pipeline, PrintWriter output) throws * [pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). Once this is done, a resource key with the * properties required by this example can be created [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java). * * @include{doc} example-require-resourcekey.txt */ \ No newline at end of file diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index 54c34bf39..6dae1a226 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -162,7 +162,7 @@ .setUpdateRandomisationMax(10*60) * ``` * # Location - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java/blob/master/device-detection-java-examples/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java). * * @include{doc} example-require-licensekey.txt */ diff --git a/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java b/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java index 83c9c0c31..53046d3b9 100644 --- a/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java +++ b/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java @@ -133,7 +133,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { * * @include{doc} example-getting-started-web.txt * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/web/getting-started.cloud/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java). * * @include{doc} example-require-resourcekey.txt * diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index c9a780e9b..22b31c746 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -145,7 +145,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { * * @include{doc} example-getting-started-web.txt * - * This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-java-examples/blob/master/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java). * * @include{doc} example-require-datafile.txt * diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index c411b9a9d..06538b059 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -54,7 +54,7 @@ * The output CSV file contains the input truth and the result fields for easy * evaluation. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=contact-us). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java index a21d88e83..3a55152aa 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java @@ -31,7 +31,7 @@ * 2. How to process an IP address and retrieve all available properties * 3. How to enumerate through all the properties returned by the service * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java index ed23f80b5..fb63e8546 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java @@ -31,7 +31,7 @@ * 2. How to pass input data (evidence) to the Pipeline * 3. How to retrieve the results * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java index a5cbe3c67..e5bf11069 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java @@ -31,7 +31,7 @@ * 2. How to pass input data (User-Agent and IP address) to the Pipeline * 3. How to retrieve device and IP intelligence results from a single pipeline * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 8014fc2bf..9bab40ab5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -33,7 +33,7 @@ * 2. How to pass combined evidence (User-Agent and IP address) to the Pipeline * 3. How to retrieve results from both engines * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java). * * This example requires: * - A Device Detection data file (.hash format) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 1637f76c8..8c6e4d0b9 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -31,7 +31,7 @@ * 2. How to pass input data (evidence) to the Pipeline * 3. How to retrieve the results * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=header). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java index 15a1af9a4..00da31c62 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java @@ -39,7 +39,7 @@ * cloud. If you are only using a single product (for example, IP intelligence) then not all * of these keys will be relevant. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 4b3224857..7a96b562f 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -50,7 +50,7 @@ * The example will output the available properties along with details about their data types and descriptions. * This helps you understand what IP Intelligence data you can access for your use case. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 0c81a3b34..aaf6ddac4 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -38,7 +38,7 @@ * verifying the published metrics associated with 51Degrees' * IP intelligence service. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=contact-us). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 354ca5387..983d25997 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -42,7 +42,7 @@ * * Evidence files can be obtained from the [ip-intelligence-data repository](https://github.com/51Degrees/ip-intelligence-data). * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=header). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index a4180e6f2..2be3b4599 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -32,7 +32,7 @@ * This can help you optimize your IP Intelligence configuration for your specific use case * and understand the performance characteristics of different settings. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index 8e6ee75b5..dd46f9cc5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -33,7 +33,7 @@ * 2. How to combine diversity values with other network related values to * assess the likelihood of an IP address being something suspicious. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us). diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index 88ee407cd..144931e2c 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -151,7 +151,7 @@ .setUpdateRandomisationMax(10*60) * ``` * # Location - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java). * * To test this example, you need to: * 1. Host an IP Intelligence data file (.ipi) at a custom URL accessible to this application diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 588976708..2de381969 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -38,7 +38,7 @@ * A cloud example, such as GettingStartedCloud, can be pointed at this server as a * custom endpoint. * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java index 279133750..4090fc110 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java @@ -32,7 +32,7 @@ * 2. How evidence from the web request is automatically passed to both engines * 3. How to retrieve results from both engines in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java index eb2d74e97..dbcb50e7f 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java @@ -31,7 +31,7 @@ * 2. How evidence from the web request is automatically passed to the Pipeline * 3. How to retrieve the results in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java). * * To run this example, you will eventually need to create a Resource Key. A Resource Key is used as * shorthand to store the particular set of properties you are interested in as well as any diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 6be4130a2..9969e3262 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -32,7 +32,7 @@ * 2. How evidence from the web request is automatically passed to both engines * 3. How to retrieve results from both engines in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/main/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java). * * This example requires: * - A Device Detection data file (.hash format) diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index a5595d706..d73bafdf8 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -31,7 +31,7 @@ * 2. How evidence from the web request is automatically passed to the Pipeline * 3. How to retrieve the results in your web application * - * This example is available in full on [GitHub](https://github.com/51Degrees/ip-intelligence-java-examples/blob/master/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java). * * This example requires an enterprise IP Intelligence data file (.ipi). * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=header). diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index a3d470e8e..2b087fe20 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -104,7 +104,7 @@ This is not recommended unless absolutely necessary and should be used with caut ## Examples Examples can be found in -[ip-intelligence-java-examples](https://github.com/51Degrees/ip-intelligence-java-examples) +[ip-intelligence-java-examples](https://github.com/51Degrees/51degrees-java/tree/main/examples/ip-intelligence) repository. ## Tests diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java index 999ba8892..748499f05 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java @@ -53,7 +53,7 @@ * Getting started example of using 51Degrees geo-location alongside 51Degrees * device detection. * - * This example is available in full on [GitHub](https://github.com/51Degrees/location-java/blob/master/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java). * * Required Maven Dependencies: * - [com.51degrees.geo-location](https://central.sonatype.com/artifact/com.51degrees/geo-location/) diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java index be73df1ef..553517c6a 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java @@ -42,7 +42,7 @@ * * Getting started example of using cloud geo-location. * - * This example is available in full on [GitHub](https://github.com/51Degrees/location-java/blob/master/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java). * * Required Maven Dependencies: * - [com.51degrees.geo-location](https://central.sonatype.com/artifact/com.51degrees/geo-location/) diff --git a/pipeline/README.md b/pipeline/README.md index 718333c20..a860e40c0 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -94,7 +94,7 @@ consumers who stay on the classpath. Device detection users need to grant access to a second module as well. The JNI `native` declarations live in `device-detection.hash.engine.on-premise`, and JEP 472 checks binding them against the module that declares them - see the -[device-detection-java README](https://github.com/51Degrees/device-detection-java#native-library-access). +[device-detection-java README](https://github.com/51Degrees/51degrees-java/blob/main/device-detection/README.md#native-library-access). ### Configuring builders from the module path @@ -122,7 +122,7 @@ Most packages contain tests which use junit and mockito. These tests can be run There are several examples available in the `pipeline.developer-examples` folder that demonstrate how to make use of the Pipeline API in isolation. These are described in the table below. If you want examples that demonstrate how to use 51Degrees products such as device detection, -then these are available in the corresponding [repository](https://github.com/51Degrees/device-detection-java) +then these are available in the corresponding [repository](https://github.com/51Degrees/51degrees-java/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=examples). | Example | Description | diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java index 06c4f4ce4..407759b4d 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java @@ -58,7 +58,7 @@ * For details of what each setting does, see the * [Share Usage reference](/documentation/4.5/_pipeline_api__features__usage_sharing.html) * - * This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-java/blob/main/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java). + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java). * * Expected output: * ``` From 5ac0cd496356e65eb6a6c2ba4490dd72a8d252cc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:00:22 +0000 Subject: [PATCH 04/14] Drop the moved example folders and fix the native model example name (#33) The examples moved to examples/, which the device detection and IP Intelligence Doxyfiles already read, so the old folders go. Point the location IMAGE_PATH at the shared documentation images, and name the native model example console/NativeModelCloud.java in place of the doubled example command. --- device-detection/docs/Doxyfile | 5 +---- .../devicedetection/examples/console/NativeModelCloud.java | 2 +- ip-intelligence/docs/Doxyfile | 1 - location/docs/Doxyfile | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/device-detection/docs/Doxyfile b/device-detection/docs/Doxyfile index 2c34c2ac9..0cbff9b26 100644 --- a/device-detection/docs/Doxyfile +++ b/device-detection/docs/Doxyfile @@ -820,7 +820,6 @@ WARN_LOGFILE = INPUT = ../device-detection/src/main \ ../device-detection.cloud/src/main \ - ../device-detection.shell.examples/src/main \ ../device-detection.hash.engine.on-premise/src/main \ ../device-detection.shared/src/main \ ../../examples/device-detection \ @@ -897,9 +896,7 @@ EXCLUDE_SYMBOLS = # that contain example code fragments that are included (see the \include # command). -EXAMPLE_PATH = ../device-detection.shell.examples \ - ../device-detection.web.examples \ - ../../examples/device-detection \ +EXAMPLE_PATH = ../../examples/device-detection \ ../../../documentation/src/snippets # If the value of the EXAMPLE_PATH tag contains directories, you can use the diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index e920f8c9d..754f0596a 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -119,7 +119,7 @@ static void analyseNativeModel(String nativemodel, Pipeline pipeline, PrintWrite } } /*! - * @example @example console/NativeModel.java + * @example console/NativeModelCloud.java * * This example shows how to use the 51Degrees Cloud service to look up the details of a device * based on a given 'native model name'. Native model name is a string of characters that are diff --git a/ip-intelligence/docs/Doxyfile b/ip-intelligence/docs/Doxyfile index 6b9d53dae..836e6d2b9 100644 --- a/ip-intelligence/docs/Doxyfile +++ b/ip-intelligence/docs/Doxyfile @@ -820,7 +820,6 @@ WARN_LOGFILE = INPUT = ../ip-intelligence/src/main \ ../ip-intelligence.cloud/src/main \ - ../ip-intelligence.shell.examples/src/main \ ../ip-intelligence.engine.on-premise/src/main \ ../ip-intelligence.shared/src/main \ ../../examples/ip-intelligence \ diff --git a/location/docs/Doxyfile b/location/docs/Doxyfile index f6c4b5094..953e37e3b 100644 --- a/location/docs/Doxyfile +++ b/location/docs/Doxyfile @@ -917,7 +917,7 @@ EXAMPLE_RECURSIVE = YES # that contain images that are to be included in the documentation (see the # \image command). -IMAGE_PATH = images +IMAGE_PATH = ../../../documentation/docs/images # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program From edbf8fcffa3564e0eae1e2b517d15f862aca90e6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 22:30:05 +0000 Subject: [PATCH 05/14] Link this repository from the native library message and the demo page (#32) When the native library fails to load, both on-premise engines suggest rebuilding the binary from the source code and instructions in device-detection-java or ip-intelligence-java, which are archived. The instructions for building the native binary are now in the README of device-detection/ and ip-intelligence/ here, beside the engine source, so the message links those folders. The creator context demo page linked pipeline-java as the source of the demo and of the code it runs on, and now links pipeline/ here. The test that checks the page ends with somewhere to go next asserts the new address. --- .../hash/engine/onpremise/interop/Constants.java | 2 +- .../ipintelligence/engine/onpremise/interop/Constants.java | 2 +- .../src/main/resources/fodid/creator-context/page.html | 2 +- .../developerexamples/fodid/CreatorContextPageTests.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java index af5116e58..1fa22d3ef 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/Constants.java @@ -29,7 +29,7 @@ public class Constants { "possible, you may be able to use a different library by " + "rebuilding the binary on the target system using the source " + "code and instructions available on " + - "https://github.com/51degrees/device-detection-java."; + "https://github.com/51Degrees/51degrees-java/tree/main/device-detection."; /** * Message returned when the native library could not be loaded. diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java index a51211766..7c09eb920 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/Constants.java @@ -29,7 +29,7 @@ public class Constants { "possible, you may be able to use a different library by " + "rebuilding the binary on the target system using the source " + "code and instructions available on " + - "https://github.com/51degrees/ip-intelligence-java."; + "https://github.com/51Degrees/51degrees-java/tree/main/ip-intelligence."; /** * Message returned when the native library could not be loaded. diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html index 0690d416a..41821e1ed 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/page.html @@ -147,7 +147,7 @@

Find out more

The source of this demo and of the code it runs on.

diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java index 9459f60b3..61d32d13c 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextPageTests.java @@ -303,7 +303,7 @@ public void The_Page_Ends_With_Find_Out_More() throws Exception { String page = pageText(); assertTrue(page.contains("Find out more"), "the page says where next"); assertTrue(page.contains("utm_campaign=51degrees-java"), "campaign tag"); - assertTrue(page.contains("https://github.com/51Degrees/pipeline-java"), + assertTrue(page.contains("https://github.com/51Degrees/51degrees-java/tree/main/pipeline"), "the repository is linked"); } } From 6b2a9dea5276757a7fff40efd1368c4b451d1218 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 23:33:58 +0000 Subject: [PATCH 06/14] Link the hardware profile engine to its specification page (#34) --- .../cloud/flowelements/HardwareProfileCloudEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java index bf288c8ee..ed7910e5a 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/HardwareProfileCloudEngine.java @@ -35,7 +35,7 @@ * Engine that takes the JSON response from the {@link CloudRequestEngine} and * uses it to populate a {@link MultiIPIDataCloud} instance for easier * consumption. - * @see Specification + * @see Specification */ public class HardwareProfileCloudEngine extends PropertyKeyedCloudEngineBase { From ebb595b78b3c4ddd6fb9436348586b4235499ff4 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:31:27 +0100 Subject: [PATCH 07/14] Merged Pull Request '#36 update-packages/rc->rc: Update packages' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- examples/device-detection/pom.xml | 2 +- examples/ip-intelligence/pom.xml | 4 ++-- examples/ip-intelligence/web/getting-started.api/pom.xml | 2 +- location/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index 47e7b11a2..d7617e6e6 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -50,7 +50,7 @@ 4.13.2 true 1.30 - 4.5.126 + 4.5.128 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 9653a86aa..945f34223 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -50,8 +50,8 @@ 4.13.2 true 1.30 - 4.5.96 - 4.5.126 + 4.5.128 + 4.5.128 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/web/getting-started.api/pom.xml b/examples/ip-intelligence/web/getting-started.api/pom.xml index ce2f2b0b1..5cb1db7d1 100644 --- a/examples/ip-intelligence/web/getting-started.api/pom.xml +++ b/examples/ip-intelligence/web/getting-started.api/pom.xml @@ -37,7 +37,7 @@ 2.0.20 1.3.16 - 4.5.48 + 4.5.128 20231013 diff --git a/location/pom.xml b/location/pom.xml index ac0e576ed..e3e332f3f 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -60,7 +60,7 @@ 1.8 ${project.version} - ${project.version} + 4.5.128 4.13.2 1.7.36 From 282fc0b25b677faf73eb9453861bffec449a432e Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:18:03 +0100 Subject: [PATCH 08/14] Merged Pull Request '#35 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cxx b/cxx index e0a6561f5..1bec7b570 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit e0a6561f521476151fbd4ba4710b94edf3c0c0b5 +Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 From 0da326d6bc307cf070952f90a7e3768ce1e73d8a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:07:30 +0000 Subject: [PATCH 09/14] Use the data repository for the examples, tests and CI (#37) --- AGENTS.md | 11 +- README.md | 25 +++- ci/README.md | 8 +- ci/fetch-assets.ps1 | 16 ++- cxx | 2 +- device-detection/README.md | 19 +-- .../devicedetection/cloud/TestsBase.java | 4 +- .../hash/engine/onpremise/TestsBase.java | 4 +- .../onpremise/flowelements/EngineTests.java | 6 +- .../shared/testhelpers/FileUtils.java | 12 +- .../devicedetection/DeviceDetectionTests.java | 4 +- examples/device-detection/README.md | 11 +- .../console/GettingStartedOnPrem.java | 10 +- .../examples/console/MatchMetrics.java | 3 +- .../examples/console/MetadataOnPrem.java | 10 +- .../examples/console/OfflineProcessing.java | 17 +-- .../console/PerformanceBenchmark.java | 5 +- .../examples/console/ShareUsageExample.java | 8 +- .../examples/console/UpdateDataFile.java | 6 +- .../main/resources/gettingStartedOnPrem.xml | 2 +- .../console/GettingStartedOnPremTest.java | 2 +- .../examples/console/MetadataOnPremTest.java | 2 +- .../console/OfflineProcessingTest.java | 2 +- .../examples/shared/DataFileHelper.java | 111 +++++++++++++-- .../examples/web/GettingStartedWebOnPrem.java | 15 +- .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 2 +- .../web/GettingStartedWebOnPremTest.java | 2 +- .../web/OnPremDataFileOptionTest.java | 2 +- examples/ip-intelligence/.gitignore | 2 - examples/ip-intelligence/README.md | 44 +++--- .../examples/console/CompareOnPrem.java | 4 +- .../examples/console/GettingStartedMixed.java | 9 +- .../examples/console/OfflineProcessing.java | 6 +- .../main/resources/gettingStartedMixed.xml | 4 +- .../main/resources/gettingStartedOnPrem.xml | 2 +- .../examples/shared/DataFileHelper.java | 128 +++++++++++++++--- .../examples/web/GettingStartedApi.java | 2 +- .../examples/web/GettingStartedWebMixed.java | 9 +- .../main/webapp/WEB-INF/51Degrees-Mixed.xml | 4 +- .../main/webapp/WEB-INF/51Degrees-OnPrem.xml | 2 +- .../examples/web/HtmlContentHelper.java | 4 +- ip-intelligence/README.md | 7 +- .../ip-intelligence.translation/pom.xml | 12 +- 43 files changed, 412 insertions(+), 148 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d163ccbfe..5a1d6f620 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,9 +22,18 @@ Tests and examples find data files with `FileFinder`, which searches below `project.root`. The surefire configuration sets that to `maven.multiModuleProjectDirectory`, which Maven takes from the nearest `.mvn` folder above the working directory. The only `.mvn` folder is at the root, so -the search covers `cxx/device-detection-data` and `cxx/ip-intelligence-data` +the search covers `cxx/data/device-detection` and `cxx/data/ip-intelligence` from wherever Maven is run. Do not add a `.mvn` folder below the root. +Those folders are the data repository, a submodule of cxx, where each free +data file is a `.link` until `get-data-files.ps1` or `get-data-files.sh` +downloads it. `ci/fetch-assets.ps1` runs the script, and the examples' +`DataFileHelper.findDataFile` runs it for a file that is missing. Look for a +free file by its folder and name, such as +`device-detection/51Degrees-LiteV4.1.hash`, so the search does not stop at a +file of the same name elsewhere, such as a Git LFS pointer in an old +`cxx/device-detection-data` folder. + ## Native builds Each on-premise engine compiles its own copy of the common C code from `cxx`, diff --git a/README.md b/README.md index 4ba31cf22..479f1ff36 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, ` | `location` | The Location packages, `com.51degrees:geo-location*` | | `examples/device-detection` | Console and web examples for Device Detection | | `examples/ip-intelligence` | Console and web examples for IP Intelligence, including the mixed examples that run both products | -| `cxx` | Submodule of [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx), with the C and C++ code the on-premise engines are built from and the data file submodules | +| `cxx` | Submodule of [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx), with the C and C++ code the on-premise engines are built from and the [data](https://github.com/51Degrees/data) submodule in `cxx/data` | | `pom.xml` | The reactor that builds everything here. It is never published | | `ci` | The scripts the shared 51Degrees CI workflows call | @@ -55,10 +55,20 @@ git clone --recurse-submodules https://github.com/51Degrees/51degrees-java.git cd 51degrees-java ``` -The free Device Detection Lite data file in `cxx/device-detection-data` is -stored with [Git LFS](https://git-lfs.github.com/). Run `git lfs pull` in that -folder to fetch it. The IP Intelligence ASN file in `cxx/ip-intelligence-data` -is plain git. +The free data files come from the [data](https://github.com/51Degrees/data) +repository, which is the `cxx/data` submodule. Each one is listed there as a +`.link` file, and a script downloads them all into place: + +``` +cxx/data/get-data-files.sh +``` + +On Windows, run `cxx\data\get-data-files.ps1` in PowerShell instead. The +examples run the script themselves for a free file they need that is missing. + +In a clone made before the data repository was added, delete the +`cxx/device-detection-data` and `cxx/ip-intelligence-data` folders once cxx no +longer includes them, because Git leaves a removed submodule's folder on disk. ## Building @@ -89,8 +99,9 @@ mvn surefire:test ``` The cloud tests need a Resource Key, passed as `-DTestResourceKey=`. The -on-premise IP Intelligence tests need the Lite or Enterprise data file, which -is not in the repository. See `ip-intelligence/README.md` for how to get it. +on-premise tests need the free data files, which the script in +[Getting the code](#getting-the-code) downloads. For the Enterprise IP +Intelligence data file, see `ip-intelligence/README.md`. Tests find data files by searching from the root of this repository, so run Maven from here rather than from inside a package folder. diff --git a/ci/README.md b/ci/README.md index 1041809a8..a537b0271 100644 --- a/ci/README.md +++ b/ci/README.md @@ -21,8 +21,12 @@ repositories used to do for one package tree each. workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). - [fetch-assets.ps1](fetch-assets.ps1) links every data file it downloads into - `cxx/device-detection-data` or `cxx/ip-intelligence-data`, where the tests - and examples find them. + `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests + and examples find them. Those are the folders of the + [data](https://github.com/51Degrees/data) repository, which is a submodule + of cxx. The script then runs the data repository's `get-data-files.ps1`, + which keeps the linked files and downloads the free files that are not + assets, which is the Asn file. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 23f509ea1..4062061b1 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -7,9 +7,11 @@ param ( $ErrorActionPreference = "Stop" # Tests and examples find data files by searching below the repository root, -# so every asset is linked into the data folders of the cxx submodule. -$deviceDetectionData = "$PSScriptRoot/../cxx/device-detection-data" -$ipIntelligenceData = "$PSScriptRoot/../cxx/ip-intelligence-data" +# so every asset is linked into the folders of the data repository, which is +# the data submodule of cxx. +$data = "$PSScriptRoot/../cxx/data" +$deviceDetectionData = "$data/device-detection" +$ipIntelligenceData = "$data/ip-intelligence" $deviceDetectionAssets = 'TAC-HashV41.hash', '51Degrees-LiteV4.1.hash', '20000 Evidence Records.yml', '20000 User Agents.csv' ./steps/fetch-assets.ps1 -DeviceDetection:$DeviceDetection -DeviceDetectionUrl:$DeviceDetectionUrl -Assets $deviceDetectionAssets @@ -24,6 +26,14 @@ New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-Enterprise # link it as that. New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$ipIntelligenceData/51Degrees-LiteV41.ipi" +# The data repository's script downloads every linked file that is not +# already there. The files linked above are kept, so this gets the free files +# that are not assets, which is the Asn file. +& "$data/get-data-files.ps1" +if (-not $?) { + throw "The data repository's get-data-files.ps1 could not download every file" +} + Write-Host "Assets hashes:" Get-FileHash -Algorithm MD5 -Path assets/* diff --git a/cxx b/cxx index 1bec7b570..e866f6d3c 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 +Subproject commit e866f6d3cea936587790f87967eff7863778b30e diff --git a/device-detection/README.md b/device-detection/README.md index c393a561e..73722fcb5 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -28,23 +28,24 @@ how to use this. #### On-Premise -If you are using on-premise detection, a "Lite" version of the data required is packaged -in this repository. It contains only a limited set of "essential" device detection properties. +If you are using on-premise detection, a free "Lite" version of the data required is +available from the [data](https://github.com/51Degrees/data) repository, which is the +`cxx/data` folder of this repository. It contains only a limited set of "essential" +device detection properties. You may want to license our complete data file containing all properties. [Details of our licenses](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) are available on our website. -If you want to use the lite file, you will need to install [GitLFS](https://git-lfs.github.com/), then: +To get the Lite file, run the data repository's download script from the root of +this repository: ``` -git lfs install +cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash ``` -Then, navigate to 'device-detection.hash.engine.on-premise/src/main/cxx/device-detection-cxx/device-detection-data' and execute: - -``` -git lfs pull -``` +On Windows, run `cxx\data\get-data-files.ps1 51Degrees-LiteV4.1.hash` in PowerShell +instead. The file is saved in `cxx/data/device-detection`, where the tests and +examples find it. ## Installation diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java index ac8007cd1..44d233d6a 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; import fiftyone.pipeline.util.FileFinder; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; public class TestsBase { @@ -43,7 +43,7 @@ protected UserAgentGenerator getUserAgents() { protected void testInitialize() throws Exception { wrapper = new WrapperCloud(); userAgents = new UserAgentGenerator( - FileFinder.getFilePath(UA_FILE_NAME)); + FileFinder.getFilePath(UA_FILE_PATH)); } public void testCleanup() { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java index 0351da463..f3826ee0e 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java @@ -26,7 +26,7 @@ import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.Constants; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; public class TestsBase { @@ -47,7 +47,7 @@ protected void testInitialize(Constants.PerformanceProfiles profile) throws Exce FileUtils.getHashFile(), profile); userAgents = new UserAgentGenerator( - getFilePath(UA_FILE_NAME)); + getFilePath(UA_FILE_PATH)); } public void testCleanup() { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index b73fbd83c..0b58bc098 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -33,7 +33,7 @@ import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.Constants; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -66,7 +66,7 @@ private void testDataSourceTier(String fileName) throws Exception { String tier = engine.getDataSourceTier(); engine.close(); - if (fileName.equals(LITE_HASH_DATA_FILE_NAME)) { + if (fileName.equals(LITE_HASH_DATA_FILE_PATH)) { assertEquals("Lite", tier); } else { @@ -81,7 +81,7 @@ private void testDataSourceTier(String fileName) throws Exception { */ @Test public void Engine_Hash_GetDataSourceTier_Lite() throws Exception { - testDataSourceTier(LITE_HASH_DATA_FILE_NAME); + testDataSourceTier(LITE_HASH_DATA_FILE_PATH); } /** diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java index 8eff15a70..fe9e13a9e 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java @@ -47,6 +47,14 @@ public class FileUtils { private static File UA_FILE; public static final String EVIDENCE_FILE_NAME = "20000 Evidence Records.yml"; private static Optional EVIDENCE_FILE; + // Where each free file is in the data repository. A search for the path + // rather than the name does not stop at another file of the same name, + // such as a Git LFS pointer in an old device-detection-data folder. + public static final String LITE_HASH_DATA_FILE_PATH = + "device-detection/" + LITE_HASH_DATA_FILE_NAME; + public static final String UA_FILE_PATH = "device-detection/" + UA_FILE_NAME; + public static final String EVIDENCE_FILE_PATH = + "device-detection/" + EVIDENCE_FILE_NAME; /** * Helper to find the location of an Enterprise or Lite Hash file in the default search scope @@ -78,7 +86,7 @@ public static File getHashFile() { } catch (Exception ex) { try { // following throws exception if not found - HASH_DATA_FILE = Optional.of(getFilePath(LITE_HASH_DATA_FILE_NAME)); + HASH_DATA_FILE = Optional.of(getFilePath(LITE_HASH_DATA_FILE_PATH)); return HASH_DATA_FILE.get(); } catch (Exception exc) { HASH_DATA_FILE = Optional.empty(); @@ -93,7 +101,7 @@ public static File getEvidenceFile() { return EVIDENCE_FILE.orElse(null); } try { - EVIDENCE_FILE = Optional.of(getFilePath(EVIDENCE_FILE_NAME)); + EVIDENCE_FILE = Optional.of(getFilePath(EVIDENCE_FILE_PATH)); return EVIDENCE_FILE.get(); } catch (Exception e) { EVIDENCE_FILE = Optional.empty(); diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java index 7d1bac24d..a59a0a6d6 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java @@ -52,7 +52,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; import static fiftyone.pipeline.core.Constants.EVIDENCE_HTTPHEADER_PREFIX; import static fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; @@ -99,7 +99,7 @@ public class DeviceDetectionTests { @BeforeClass public static void initClass() throws IOException { userAgents = new UserAgentGenerator( - FileFinder.getFilePath(UA_FILE_NAME)); + FileFinder.getFilePath(UA_FILE_PATH)); } private static String reportErrors(Collection errors) { diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 5d2edd339..1f42ecd77 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -21,6 +21,14 @@ Among other things, the examples illustrate: - use of device detection pipeline for off-line processing tasks - configuring device detection trade-offs between speed and conserving memory +## Data files + +The on-premise examples use the free Lite data file and the evidence files in +`cxx/data/device-detection`, a folder of the +[data](https://github.com/51Degrees/data) repository. An example that cannot +find one of these files runs the data repository's download script for it, +and logs the command it runs. + ## Cloud resource keys You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) @@ -114,7 +122,8 @@ PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-ex The on-premise web example takes its data file from the `51DEGREES_DD_PATH` environment variable, then from `TestDataFile`, and otherwise uses the free Lite file -in `device-detection-data`. +in `cxx/data/device-detection`, which it downloads before the server starts if it is +missing. ### Native library access diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 9d8819565..332ef1dca 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -58,12 +58,13 @@ public class GettingStartedOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, by default, the 51degrees "Lite" file needs to be somewhere in the project - space, or you may specify another file as a command line parameter. + /* In this example, by default, the 51degrees "Lite" file is used from the data repository in + cxx/data, and is downloaded if it is missing, or you may specify another file as a command line + parameter. Note that the Lite data file is only used for illustration, and has limited accuracy and capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ - public static String LITE_V_4_1_HASH = "51Degrees-LiteV4.1.hash"; + public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static String ENTERPRISE_HASH = "TAC-HashV41.hash"; public static void main(String[] args) throws Exception { @@ -87,7 +88,8 @@ public static void run(String dataFile, OutputStream outputStream) throws Exception { logger.info("Running GettingStarted example"); try { - String dataFileLocation = getFilePath(dataFile).getAbsolutePath(); + String dataFileLocation = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); // the location of the test data file is interpolated in the pipeline // configuration file System.setProperty("TestDataFile", dataFileLocation); diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index a54bfe71c..4c0aa55a1 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -27,7 +27,6 @@ import fiftyone.devicedetection.examples.shared.EvidenceHelper; import fiftyone.devicedetection.hash.engine.onpremise.data.DeviceDataHash; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.core.data.ElementPropertyMetaData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -80,7 +79,7 @@ static void run(String dataFile, List> evidenceList, boolean showDescs, OutputStream out) throws Exception { PrintWriter writer = new PrintWriter(out, true); if (Objects.isNull(dataFile)) { - dataFile = FileUtils.getHashFileName(); + dataFile = DataFileHelper.getDataFileLocation(null); } logger.info("Constructing pipeline from file " + dataFile); diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index a58e739ab..1dd9b1d21 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -45,12 +45,13 @@ public class MetadataOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, by default, the 51degrees "Lite" file needs to be somewhere in the project - space, or you may specify another file as a command line parameter. + /* In this example, by default, the 51degrees "Lite" file is used from the data repository in + cxx/data, and is downloaded if it is missing, or you may specify another file as a command line + parameter. Note that the Lite data file is only used for illustration, and has limited accuracy and capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ - public static String LITE_V_4_1_HASH = "51Degrees-LiteV4.1.hash"; + public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -63,7 +64,8 @@ public static void run(String dataFile, OutputStream output) throws Exception { logger.info("Running MetadataOnPrem example"); String dataFileLocation; try { - dataFileLocation = getFilePath(dataFile).getAbsolutePath(); + dataFileLocation = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { DataFileHelper.cantFindDataFile(dataFile); throw e; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index 031c63368..8fc928a56 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -67,22 +67,22 @@ */ public class OfflineProcessing { static final Logger logger = LoggerFactory.getLogger(OfflineProcessing.class); - // This 51degrees "Lite" file (distributed with the source) needs to - // be somewhere in the project space + // This 51degrees "Lite" file is used from the data repository in + // cxx/data, and is downloaded if it is missing // // Note that the Lite data file is only used for illustration, and has // limited accuracy and capabilities. Find out about the Enterprise data // file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash public static final String LITE_V_4_1_HASH = - "device-detection-data/51Degrees-LiteV4.1.hash"; - // This 51degrees file of 20,000 examples (distributed with the source) - // needs to be somewhere in the project space + "device-detection/51Degrees-LiteV4.1.hash"; + // This 51degrees file of 20,000 examples is used from the same folder, + // and is downloaded if it is missing public static final String HEADER_EVIDENCE_YML = - "device-detection-data/20000 Evidence Records.yml"; + "device-detection/20000 Evidence Records.yml"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - File evidenceFile = getFilePath(HEADER_EVIDENCE_YML); + File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(LITE_V_4_1_HASH, Files.newInputStream(evidenceFile.toPath()), System.out); } @@ -100,7 +100,8 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws String detectionFile; try { - detectionFile = getFilePath(dataFile).getAbsolutePath(); + detectionFile = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { DataFileHelper.cantFindDataFile(dataFile); throw e; diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index aeaf3180c..8c6f2b0fe 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -388,8 +388,9 @@ public PerformanceConfiguration(Constants.PerformanceProfiles profile, * @example PerformanceBenchmark.java * The example illustrates a "clock-time" benchmark for assessing detection speed. * - * Using a YAML formatted evidence file - "20000 Evidence Records.yml" - supplied with the - * distribution or can be obtained from the [data repository on Github](https://github.com/51Degrees/device-detection-data/blob/master/20000%20Evidence%20Records.yml). + * Using a YAML formatted evidence file - "20000 Evidence Records.yml" - which is downloaded by + * the scripts in the [data repository on GitHub](https://github.com/51Degrees/data), and by the + * example itself when the file is missing. * * It's important to understand the trade-offs between performance, memory usage and accuracy, that * the 51Degrees pipeline configuration makes available, and this example shows a range of diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java index 9e9ba2320..621ef0761 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; @@ -75,9 +76,10 @@ public class ShareUsageExample { static final Logger logger = LoggerFactory.getLogger(ShareUsageExample.class); - // Evidence file with 20,000 examples + // Evidence file with 20,000 examples, from the data repository in + // cxx/data. The example downloads the file if it is missing. public static final String HEADER_EVIDENCE_YML = - "device-detection-data/20000 Evidence Records.yml"; + "device-detection/20000 Evidence Records.yml"; // Configurable settings private static final int MINIMUM_ENTRIES_PER_MESSAGE = 10; // Send after 10 entries (default is 50) @@ -100,7 +102,7 @@ public class ShareUsageExample { public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - File evidenceFile = getFilePath(HEADER_EVIDENCE_YML); + File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(Files.newInputStream(evidenceFile.toPath())); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index 6dae1a226..d3babcd27 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -73,9 +73,9 @@ * ``` * ## Lite Data File * Lite data files (free-to-use, limited capabilities, no license key required) are created roughly - * once a month and cannot be updated using auto-update, they may be downloaded from - * [Github](https://github.com/51Degrees/device-detection-data) and are included with - * source distributions of this software. + * once a month and cannot be updated using auto-update, they may be downloaded with the + * scripts in the [data repository](https://github.com/51Degrees/data) on GitHub, which is + * the cxx/data folder of the source. * # Update on Start-Up * You can configure the pipeline builder to download an Enterprise data file on start-up. * ## Pre-Requisites diff --git a/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml b/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml index fed362bfd..ee634b786 100644 --- a/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml +++ b/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml @@ -5,7 +5,7 @@ false false - ${TestDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash} + ${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash} LowMemory DeviceDetectionHashEngine diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java index 1c272b52e..779a70fc2 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java @@ -31,7 +31,7 @@ public class GettingStartedOnPremTest { @Test public void gettingStartedOnPremTest() throws Exception { GettingStartedOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFileName(), EvidenceHelper.setUpEvidence(), System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java index 32d3b8882..a128b1393 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java @@ -29,7 +29,7 @@ public class MetadataOnPremTest { @Test public void gettingMetaDataOnPremTest() throws Exception { MetadataOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFileName(), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java index 80d767c58..4861f14d6 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java @@ -32,7 +32,7 @@ public class OfflineProcessingTest { @Test public void offlineProcessingTest() throws Exception { OfflineProcessing.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFileName(), new FileInputStream(Objects.requireNonNull(FileUtils.getEvidenceFile())), System.out); diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java index c5aab3928..337f6e165 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java @@ -32,10 +32,15 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.List; +import java.util.Locale; import java.util.Objects; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -43,6 +48,12 @@ public class DataFileHelper { static Logger logger = LoggerFactory.getLogger(DataFileHelper.class); + /** + * Sent with each download, so the Distributor can count the downloads + * made by these examples. + */ + private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; @@ -90,9 +101,10 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { "than {} days old. A more recent data file " + "may be needed to correctly detect the latest " + "devices, browsers, etc.", daysOld); - logger.info("The latest 'Lite' data file is available " + - "from the device-detection-data repository on GitHub " + - "https://github.com/51Degrees/device-detection-data. " + + logger.info("Run get-data-files.ps1 -Update or " + + "get-data-files.sh -u in cxx/data, which is the data " + + "repository https://github.com/51Degrees/data, to get " + + "the latest 'Lite' data file. " + "Find out about the Enterprise data file, which " + "includes automatic daily updates, on our pricing " + "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-age-warning"); @@ -102,9 +114,10 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { public static void cantFindDataFile(String dataFile) { logger.error("Could not find the data file '{}' which must be " + "somewhere in the project space to be found.", dataFile); - logger.info("The latest 'Lite' data file is available " + - "from the device-detection-data repository on GitHub " + - "https://github.com/51Degrees/device-detection-data. " + + logger.info("The free 'Lite' data file and the evidence files " + + "are downloaded by get-data-files.ps1 or get-data-files.sh " + + "in cxx/data, which is the data repository " + + "https://github.com/51Degrees/data. " + "Find out about the Enterprise data file, which includes " + "automatic daily updates, on our pricing " + "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-not-found"); @@ -120,12 +133,12 @@ public static void cantFindDataFile(String dataFile) { @SuppressWarnings("RedundantThrows") public static File getEvidenceFile(String evidenceFilename) throws Exception { if (Objects.isNull(evidenceFilename)) { - evidenceFilename = FileUtils.EVIDENCE_FILE_NAME; + evidenceFilename = FileUtils.EVIDENCE_FILE_PATH; } File evidenceFile; try { - evidenceFile = FileFinder.getFilePath(evidenceFilename); + evidenceFile = findDataFile(evidenceFilename); } catch (Exception e) { logger.error("Could not find evidence file {}", evidenceFilename); throw e; @@ -134,7 +147,8 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { } /** - * Tries to find the passed file, or if null a default file + * Tries to find the passed file, or if null the first data file found, + * or the free Lite file when there is none * @param dataFilename a filename to find * @return a full pathname * @throws Exception if the file was not found @@ -144,13 +158,90 @@ public static String getDataFileLocation(String dataFilename) throws Exception { if (Objects.isNull(dataFilename)) { dataFilename = FileUtils.getHashFileName(); } + if (Objects.isNull(dataFilename)) { + dataFilename = FileUtils.LITE_HASH_DATA_FILE_PATH; + } String dataFileLocation; try { - dataFileLocation = getFilePath(dataFilename).getAbsolutePath(); + dataFileLocation = findDataFile(dataFilename).getAbsolutePath(); } catch (Exception e) { DataFileHelper.cantFindDataFile(dataFilename); throw e; } return dataFileLocation; } + + /** + * Finds a file below the project root, as + * {@link FileFinder#getFilePath(String)} does. A free data file that is + * missing has a {@code .link} in the data repository, so the data + * repository's script is run to download it, and the file is looked for + * again. + * @param file the name of the file, or the end of its path + * @return the file + * @throws IllegalArgumentException if the file was not found, naming the + * command that downloads it when there is one + */ + public static File findDataFile(String file) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + File link; + try { + link = getFilePath(file + ".link"); + } catch (IllegalArgumentException noLink) { + throw notFound; + } + String command = runDownload(link); + try { + return getFilePath(file); + } catch (IllegalArgumentException stillNotFound) { + String message = "Could not download '" + file + + "'. Run this command to download it: " + command; + logger.error(message); + throw new IllegalArgumentException(message, stillNotFound); + } + } + } + + /** + * Runs the data repository's script for the file that a link is for, + * and waits for it to finish. The data repository is the folder above + * the one the link is in. + * @param link the {@code .link} file + * @return the command, as it would be typed + */ + private static String runDownload(File link) { + String name = link.getName().substring( + 0, link.getName().length() - ".link".length()); + File data = link.getAbsoluteFile().getParentFile().getParentFile(); + List command; + if (System.getProperty("os.name", "") + .toLowerCase(Locale.ROOT).startsWith("windows")) { + command = Arrays.asList("powershell", "-NoProfile", + "-ExecutionPolicy", "Bypass", "-File", + new File(data, "get-data-files.ps1").getPath(), name, + "-Source", DOWNLOAD_SOURCE); + } else { + command = Arrays.asList("sh", + new File(data, "get-data-files.sh").getPath(), + "-s", DOWNLOAD_SOURCE, name); + } + List typed = new ArrayList<>(); + for (String argument : command) { + typed.add(argument.contains(" ") ? + "\"" + argument + "\"" : argument); + } + String line = String.join(" ", typed); + logger.info("'{}' is missing, so it is being downloaded by running {}", + name, line); + try { + new ProcessBuilder(command).inheritIO().start().waitFor(); + } catch (IOException e) { + logger.error("Could not run {}. {}", line, e.getMessage()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return line; + } } diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index 22b31c746..8387720c6 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.web; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; @@ -34,6 +35,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.File; import java.io.PrintWriter; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; @@ -48,7 +50,8 @@ *

* The data file is taken from the 51DEGREES_DD_PATH environment variable, then * from TestDataFile (an environment variable or system property), and otherwise - * the free Lite file in device-detection-data is used. + * the free Lite file in cxx/data/device-detection is used, which is downloaded + * before the server starts if it is missing. *

* The server listens on port 8081 and stops when Enter is pressed. If the PORT * environment variable is set, the server listens on that port instead and runs @@ -63,6 +66,16 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); logger.info("Running Example {}", GettingStartedWebOnPrem.class); + // If the data file the configuration names is not at that path, it + // is searched for, and the free Lite file is downloaded if missing. + String dataFile = PipelineOptionsFactory.getOptionsFromFile( + getFilePath(resourceBase) + "/WEB-INF/51Degrees-OnPrem.xml") + .findAndSubstitute("DeviceDetectionHashEngine", "DataFile"); + if (!new File(dataFile).exists()) { + System.setProperty("TestDataFile", + DataFileHelper.getDataFileLocation(dataFile)); + } + String portEnv = System.getenv("PORT"); if (portEnv != null) { // Automated runs (e.g. the unified Selenium suite) inject a port and have no diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index f785a6607..bba6dd60e 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -32,7 +32,7 @@ variable, then from TestDataFile (an environment variable or system property), and otherwise the free Lite file is used. --> - ${51DEGREES_DD_PATH:-${TestDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash}} + ${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} LowMemory DeviceDetectionHashEngine diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java index 70636a724..d6a94896f 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java @@ -43,7 +43,7 @@ public class GettingStartedWebOnPremTest { @BeforeClass public static void startJetty() throws Exception { System.setProperty("TestDataFile", FileUtils.getHashFile() == null - ? FileUtils.LITE_HASH_DATA_FILE_NAME + ? FileUtils.LITE_HASH_DATA_FILE_PATH : FileUtils.getHashFile().getAbsolutePath()); SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); } diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java index 7c898e848..108fbd615 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java @@ -43,7 +43,7 @@ public class OnPremDataFileOptionTest { private static final String DD_PATH = "51DEGREES_DD_PATH"; private static final String TEST_DATA_FILE = "TestDataFile"; private static final String LITE = - "cxx/device-detection-data/51Degrees-LiteV4.1.hash"; + "cxx/data/device-detection/51Degrees-LiteV4.1.hash"; private String savedDdPath; private String savedTestDataFile; diff --git a/examples/ip-intelligence/.gitignore b/examples/ip-intelligence/.gitignore index 092783861..23e4fdfef 100644 --- a/examples/ip-intelligence/.gitignore +++ b/examples/ip-intelligence/.gitignore @@ -48,5 +48,3 @@ SESSIONS.ser 51Degrees-IPIV4EnterpriseIpiV41.ipi.gz 51Degrees-IPIV4AsnIpiV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi.gz -# device detection data file used by the mixed examples -device-detection-data/ diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 022af4b4a..3ebef2ce6 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -10,16 +10,20 @@ These examples are not distributed as maven jars and need to be built by you. ## Required Files -See [ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data/) -repository for instructions on obtaining the necessary data files for on-premise IP intelligence. - -**IMPORTANT**: The enterprise IP Intelligence data file must be placed in the `ip-intelligence-data` -directory at the root of this repository. The expected filename is `51Degrees-EnterpriseIpiV41.ipi`. - -Two free data files are also usable for many of the examples. The ASN file -(`51Degrees-IPIV4AsnIpiV41.ipi`) is included in the `ip-intelligence-data` submodule, and the -Lite file (`51Degrees-LiteV41.ipi`) can be downloaded by running -`ip-intelligence-data/get-lite-file-from-azure.ps1` (or `.sh`). When the enterprise file is +The data files for on-premise IP intelligence are kept in `cxx/data/ip-intelligence`, a +folder of the [data](https://github.com/51Degrees/data) repository, which is a submodule of +cxx. + +**IMPORTANT**: The enterprise IP Intelligence data file must be placed in the +`cxx/data/ip-intelligence` directory of this repository. The expected filename is +`51Degrees-EnterpriseIpiV41.ipi`. + +Two free data files are also usable for many of the examples, the ASN file +(`51Degrees-IPIV4AsnIpiV41.ipi`) and the Lite file (`51Degrees-LiteV41.ipi`). Run +`cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download +them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. An example +that needs one of them and cannot find it runs the script for that file, and logs the +command it runs. When the enterprise file is absent, the tests for examples that work with any data file fall back to the ASN file and then the Lite file. The OfflineProcessing and PerformanceBenchmark examples need the RegisteredName property, which the ASN file does not contain, so their tests fall back to @@ -27,12 +31,16 @@ the Lite file only. Tests for examples needing enterprise-only properties (Metri and Suspicious) are skipped instead. ``` -ip-intelligence-java-examples/ -├── ip-intelligence-data/ -│ └── 51Degrees-EnterpriseIpiV41.ipi -├── console/ -├── web/ -└── shared/ +51degrees-java/ +├── cxx/ +│ └── data/ +│ └── ip-intelligence/ +│ └── 51Degrees-EnterpriseIpiV41.ipi +└── examples/ + └── ip-intelligence/ + ├── console/ + ├── web/ + └── shared/ ``` The examples locate the data file in the following order: @@ -40,9 +48,9 @@ The examples locate the data file in the following order: 1. The `51DEGREES_IPI_PATH` environment variable or system property, which can be set to an explicit path to the data file. 2. A search of the project folder hierarchy for the expected file name. -3. The expected location `ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi` +3. The expected location `cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi` (the configuration files for the getting started examples fall back to the - free Lite file `ip-intelligence-data/51Degrees-LiteV41.ipi`). + free Lite file `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi`). This project contains sub-modules - **console**, giving examples that are intended to be run from the command line/console and **web**, illustrating use diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index 06538b059..b8df87518 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -112,10 +112,10 @@ public class CompareOnPrem { /** * Relative path to the CSV file of truth records distributed with the - * ip-intelligence-data submodule. + * data repository, in cxx/data/ip-intelligence. */ public static final String GEOIP_COMPARISON_EVIDENCE_REL_PATH = - "ip-intelligence-data/geoip_comparison_evidence.csv"; + "ip-intelligence/geoip_comparison_evidence.csv"; /** * Default name of the output file written to the working directory. diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 9bab40ab5..07f33c130 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -83,7 +83,7 @@ public class GettingStartedMixed { private static final Logger logger = LoggerFactory.getLogger(GettingStartedMixed.class); - public static final String DD_DATA_FILE_DEFAULT = "device-detection-data/51Degrees-LiteV4.1.hash"; + public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; public static final String IPI_DATA_FILE_DEFAULT = DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; public static void main(String[] args) throws Exception { @@ -271,11 +271,12 @@ private static String resolveDataFile(String dataFile) throws IOException { if (Files.exists(relativePath)) { return relativePath.toAbsolutePath().toString(); } - // Try FileFinder (searches within project directory) + // Search within the project directory, downloading the free Lite + // file if it is missing try { - return getFilePath(dataFile).getAbsolutePath(); + return DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { - throw new IOException("Data file not found: " + dataFile); + throw new IOException("Data file not found: " + dataFile, e); } } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 983d25997..608989b73 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -40,7 +40,7 @@ * advanced operation this example can be used to experiment with tuning IP Intelligence for * performance using Performance Profile settings. * - * Evidence files can be obtained from the [ip-intelligence-data repository](https://github.com/51Degrees/ip-intelligence-data). + * Evidence files can be obtained from the [data repository](https://github.com/51Degrees/data). * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). * @@ -99,10 +99,10 @@ public class OfflineProcessing { // be somewhere in the project space // // For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=datadir - private static final String dataDir = "ip-intelligence-data"; + private static final String dataDir = "ip-intelligence"; // This 51degrees file of 20,000 examples (distributed with the source) // needs to be somewhere in the project space. Additional evidence files - // can be obtained from the [ip-intelligence-data repository](https://github.com/51Degrees/ip-intelligence-data) + // can be obtained from the [data repository](https://github.com/51Degrees/data) public static final String HEADER_EVIDENCE_YML = dataDir + "/evidence.yml"; diff --git a/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml b/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml index c6dba02ab..cfb3acbcf 100644 --- a/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml +++ b/examples/ip-intelligence/console/src/main/resources/gettingStartedMixed.xml @@ -5,7 +5,7 @@ false false - ${TestDeviceDetectionDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash} + ${TestDeviceDetectionDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash} LowMemory DeviceDetectionHashEngine @@ -14,7 +14,7 @@ false false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-LiteV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml b/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml index 7f1171389..f156a40b5 100644 --- a/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml +++ b/examples/ip-intelligence/console/src/main/resources/gettingStartedOnPrem.xml @@ -6,7 +6,7 @@ false false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-LiteV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index d448834d4..4cfa22723 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -39,7 +39,11 @@ import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.List; +import java.util.Locale; import java.util.Objects; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -48,31 +52,38 @@ public class DataFileHelper { static Logger logger = LoggerFactory.getLogger(DataFileHelper.class); /** - * Relative path to the enterprise IP Intelligence data file from the examples repository root. + * Sent with each download, so the Distributor can count the downloads + * made by these examples. + */ + private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + + /** + * Folder and name of the enterprise IP Intelligence data file, which + * belongs in cxx/data/ip-intelligence, a folder of the data repository. * This path is used by both console and web examples for testing. - * The data file is downloaded to this location by CI scripts. + * The data file is linked to this location by CI scripts. */ - public static final String ENTERPRISE_DATA_FILE_REL_PATH = "ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi"; + public static final String ENTERPRISE_DATA_FILE_REL_PATH = "ip-intelligence/51Degrees-EnterpriseIpiV41.ipi"; /** - * Relative path to the freely downloadable Lite IP Intelligence data file. - * Run ip-intelligence-data/get-lite-file-from-azure.ps1 (or .sh) to obtain it. + * Folder and name of the free Lite IP Intelligence data file, which the + * data repository's get-data-files script downloads to that folder. * It contains the RegisteredCountry, RegisteredName and RegisteredOwner properties. */ - public static final String LITE_DATA_FILE_REL_PATH = "ip-intelligence-data/51Degrees-LiteV41.ipi"; + public static final String LITE_DATA_FILE_REL_PATH = "ip-intelligence/51Degrees-LiteV41.ipi"; /** - * Relative path to the ASN IP Intelligence data file which is part of the - * ip-intelligence-data submodule. It contains the Asn and AsnName properties. + * Folder and name of the free ASN IP Intelligence data file, which the + * same script downloads. It contains the Asn and AsnName properties. */ - public static final String ASN_DATA_FILE_REL_PATH = "ip-intelligence-data/51Degrees-IPIV4AsnIpiV41.ipi"; + public static final String ASN_DATA_FILE_REL_PATH = "ip-intelligence/51Degrees-IPIV4AsnIpiV41.ipi"; /** * Find the best IP Intelligence data file available, preferring the - * enterprise file, then the ASN file from the ip-intelligence-data - * submodule, then the Lite file. Suitable for examples that work with - * whatever properties the data file contains, where any data file - * allows the example to run. + * enterprise file, then the free ASN file, then the free Lite file, + * and downloading a free file that is missing. Suitable for examples + * that work with whatever properties the data file contains, where any + * data file allows the example to run. * @return a resolved data file path, or null when none is available */ public static String findAvailableDataFile() { @@ -180,7 +191,7 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { File evidenceFile; try { - evidenceFile = FileFinder.getFilePath(evidenceFilename); + evidenceFile = findDataFile(evidenceFilename); } catch (Exception e) { logger.error("Could not find evidence file {}", evidenceFilename); throw e; @@ -244,13 +255,92 @@ public static String getDataFileLocation(String dataFilename) throws Exception { if (Files.exists(dataPath)) { return dataPath.toAbsolutePath().toString(); } - DataFileHelper.cantFindDataFile(dataFilename); - throw new IOException("Data file not found at relative path: " + dataFilename + - ". Searched in project directories and current working directory." + - " An explicit path can be supplied via the " + IPI_PATH_ENV_VAR + - " environment variable."); + // Download it, if it is a free data file + try { + return findDataFile(dataFilename).getAbsolutePath(); + } catch (IllegalArgumentException notFound) { + DataFileHelper.cantFindDataFile(dataFilename); + throw new IOException("Data file not found at relative path: " + dataFilename + + ". Searched in project directories and current working directory." + + " An explicit path can be supplied via the " + IPI_PATH_ENV_VAR + + " environment variable.", notFound); + } } } } } + + /** + * Finds a file below the project root, as + * {@link FileFinder#getFilePath(String)} does. A free data file that is + * missing has a {@code .link} in the data repository, so the data + * repository's script is run to download it, and the file is looked for + * again. + * @param file the name of the file, or the end of its path + * @return the file + * @throws IllegalArgumentException if the file was not found, naming the + * command that downloads it when there is one + */ + public static File findDataFile(String file) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + File link; + try { + link = getFilePath(file + ".link"); + } catch (IllegalArgumentException noLink) { + throw notFound; + } + String command = runDownload(link); + try { + return getFilePath(file); + } catch (IllegalArgumentException stillNotFound) { + String message = "Could not download '" + file + + "'. Run this command to download it: " + command; + logger.error(message); + throw new IllegalArgumentException(message, stillNotFound); + } + } + } + + /** + * Runs the data repository's script for the file that a link is for, + * and waits for it to finish. The data repository is the folder above + * the one the link is in. + * @param link the {@code .link} file + * @return the command, as it would be typed + */ + private static String runDownload(File link) { + String name = link.getName().substring( + 0, link.getName().length() - ".link".length()); + File data = link.getAbsoluteFile().getParentFile().getParentFile(); + List command; + if (System.getProperty("os.name", "") + .toLowerCase(Locale.ROOT).startsWith("windows")) { + command = Arrays.asList("powershell", "-NoProfile", + "-ExecutionPolicy", "Bypass", "-File", + new File(data, "get-data-files.ps1").getPath(), name, + "-Source", DOWNLOAD_SOURCE); + } else { + command = Arrays.asList("sh", + new File(data, "get-data-files.sh").getPath(), + "-s", DOWNLOAD_SOURCE, name); + } + List typed = new ArrayList<>(); + for (String argument : command) { + typed.add(argument.contains(" ") ? + "\"" + argument + "\"" : argument); + } + String line = String.join(" ", typed); + logger.info("'{}' is missing, so it is being downloaded by running {}", + name, line); + try { + new ProcessBuilder(command).inheritIO().start().waitFor(); + } catch (IOException e) { + logger.error("Could not run {}. {}", line, e.getMessage()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return line; + } } diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 2de381969..6e3fa2e8a 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -94,7 +94,7 @@ * and a JSON builder element which serialises the results. *

* The data file can be supplied as a command line argument. When no argument is given - * the enterprise data file is looked for in the ip-intelligence-data directory. + * the enterprise data file is looked for in the cxx/data/ip-intelligence directory. *

* Note that the .NET version of this example also includes a device detection engine * in the same pipeline. This Java version is IP Intelligence only. diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 9969e3262..5b7331767 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -87,7 +87,7 @@ public class GettingStartedWebMixed extends HttpServlet { private static final long serialVersionUID = 1734154705981153541L; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebMixed.class); - public static final String DD_DATA_FILE_DEFAULT = "device-detection-data/51Degrees-LiteV4.1.hash"; + public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -310,10 +310,13 @@ private static String resolveDataFile(String dataFile) throws IOException { if (Files.exists(relativePath)) { return relativePath.toAbsolutePath().toString(); } + // Search within the project directory, downloading the free Lite + // file if it is missing try { - return getFilePath(dataFile).getAbsolutePath(); + return fiftyone.ipintelligence.examples.shared.DataFileHelper + .findDataFile(dataFile).getAbsolutePath(); } catch (Exception e) { - throw new IOException("Data file not found: " + dataFile); + throw new IOException("Data file not found: " + dataFile, e); } } } diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml index 95f859502..127b4665d 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml @@ -27,7 +27,7 @@ false false - ${TestDeviceDetectionDataFile:-cxx/device-detection-data/51Degrees-LiteV4.1.hash} + ${TestDeviceDetectionDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash} LowMemory DeviceDetectionHashEngine @@ -36,7 +36,7 @@ false false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index d1bc62de7..c8e8d515b 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -29,7 +29,7 @@ false - ${TestDataFile:-cxx/ip-intelligence-data/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java index 2425e1be5..0d70be0f1 100644 --- a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java +++ b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java @@ -78,8 +78,8 @@ public static String getDataFileAgeWarning(FlowData flowData) { return "

\n" + " The IP intelligence data file is " + daysOld + " days old. " + "A more recent data file may be needed for the most accurate IP " + - "intelligence results. The latest Lite data file is available from the " + - "ip-intelligence-data " + + "intelligence results. The latest Lite data file is downloaded by the " + + "scripts in the data " + "repository on GitHub. Find out about the full Enterprise data file, " + "which includes automatic daily updates, on our " + "pricing page.\n" + diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index 2b087fe20..3d194b87b 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -33,14 +33,13 @@ how to use this. In order to perform IP intelligence on-premise, you will need to use a data file. -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data/) repository instructs how to obtain a 'Lite' data file, otherwise [contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) to obtain an 'Enterprise' data file. +The free 'Lite' and Asn data files come from the [data](https://github.com/51Degrees/data) repository, which is the `cxx/data` folder of this repository. Run `cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. Otherwise [contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) to obtain an 'Enterprise' data file. For tests, the data file can be supplied explicitly by setting the `51DEGREES_IPI_PATH` environment variable or system property to the path of the .ipi file. When it is not set, the project space is searched for the -expected file name, with the free 'Lite' file expected in the -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data/) -location. +expected file name, with the free 'Lite' file expected in +`cxx/data/ip-intelligence`. ## Installation diff --git a/ip-intelligence/ip-intelligence.translation/pom.xml b/ip-intelligence/ip-intelligence.translation/pom.xml index 83cbc8865..a620ab341 100644 --- a/ip-intelligence/ip-intelligence.translation/pom.xml +++ b/ip-intelligence/ip-intelligence.translation/pom.xml @@ -60,9 +60,11 @@ + they are copied in at build time from the data repository in + cxx/data/ip-intelligence/Translations, as the .NET project + embeds the same files. A folder that is not there is skipped + without an error, so TranslationTests, which read the files + from the classpath, are what show they were copied. --> org.apache.maven.plugins maven-resources-plugin @@ -78,13 +80,13 @@ ${project.build.outputDirectory}/fiftyone/ipintelligence/translation - ${project.basedir}/../../cxx/ip-intelligence-data/Translations + ${project.basedir}/../../cxx/data/ip-intelligence/Translations countrycodes.en_GB.yml - ${project.basedir}/../../cxx/ip-intelligence-data/Translations/OSM + ${project.basedir}/../../cxx/data/ip-intelligence/Translations/OSM countries.*.yml From 82cc6a7e830c904b5bf7a6b4f88f94e7ad234166 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:07:55 +0000 Subject: [PATCH 10/14] Say in AGENTS.md how to get the data, build and run the examples (#41) --- AGENTS.md | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5a1d6f620..971eb5d05 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,226 @@ # Working in this repository -Notes on what is not obvious from the code. Keep them short and current. -Building, testing and the layout are described in the [README](README.md). +If you have been asked to get an example running, follow +[Run the examples](#run-the-examples). It gives every step in order, from a +fresh clone to the output. The sections after it are notes on what is not +obvious from the code, for changing it. Keep them short and current. The +[README](README.md) describes the build, the tests and the layout. + +## Run the examples + +Run each command from the repository root unless the step says otherwise. +Device Detection's on-premise `GettingStartedOnPrem` is the quickest example +to show, because its data file is about 35 MB. IP Intelligence's on-premise +`GettingStartedOnPrem` needs a download of about 230 MB that unpacks to about +460 MB, so run it second. + +The examples use the packages published to Maven Central unless the `dev` +profile is active (see +[One reactor, six package trees](#one-reactor-six-package-trees)). Those +packages carry the native engines, so building only an example folder, as +below, needs no CMake and no C++ toolchain. `mvn install` at the root builds +the native engines from `cxx` and does need them, as the README says. + +### Prerequisites + +- Git. +- A JDK, 8 or later. CI tests 8, 11, 17 and 21 (`ci/options.json`). +- Maven 3.8 or later, or the `mvnw` wrapper at the root. +- Network access to GitHub, to Maven Central for the packages the examples + use, and to `distributor.51degrees.com` for the data files. +- For the data files, `curl`, `gzip` and `md5sum` (`md5` on macOS) on Linux + and macOS, where the examples download with the shell script, and Windows + PowerShell 5.1 or PowerShell 7 on Windows, where they use the PowerShell + script. + +Clone the repository with its submodules. + +``` +git clone --recurse-submodules https://github.com/51Degrees/51degrees-java.git +cd 51degrees-java +``` + +In a clone made without `--recurse-submodules`, fetch the submodules. + +``` +git submodule update --init --recursive +``` + +### Data files + +The examples use the free Lite data files, which need no licence key. They are +not stored in this repository. The `cxx` submodule holds the +[data](https://github.com/51Degrees/data) repository as `cxx/data`, which has a +link to each file, and a script downloads them into `cxx/data/device-detection` +and `cxx/data/ip-intelligence`. + +On Linux and macOS, run the shell script. + +``` +cxx/data/get-data-files.sh +``` + +On Windows, run the PowerShell script. Windows PowerShell can refuse to run a +script file under the policy Windows sets by default, so run it the way the +examples do. + +``` +powershell -NoProfile -ExecutionPolicy Bypass -File cxx/data/get-data-files.ps1 +``` + +Where PowerShell 7 is installed, `pwsh cxx/data/get-data-files.ps1` does the +same on any system. Either script downloads every free file, and keeps a file +that is already there. To download one file, give its name, as in +`cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash`. An example whose free +file is missing runs the script for that file itself. + +#### Check the files + +- `cxx/data/device-detection/51Degrees-LiteV4.1.hash` is more than 10 MB. +- `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi` is more than 100 MB. + +### Build + +Build the console examples of a product, with the examples' shared module, in +that product's examples folder. Each build makes a jar that holds the example +and everything it needs. + +``` +cd examples/device-detection +mvn -pl console -am package -DskipTests +cd ../.. +cd examples/ip-intelligence +mvn -pl console -am package -DskipTests +cd ../.. +``` + +With the wrapper, run `../../mvnw` in place of `mvn`, or `..\..\mvnw.cmd` on +Windows. The jars are +`examples/device-detection/console/target/device-detection-java-examples.console--jar-with-dependencies.jar` +and +`examples/ip-intelligence/console/target/ip-intelligence-java-examples.console--jar-with-dependencies.jar`, +where `` is the `` in that examples folder's `pom.xml`, +`4.5.127-SNAPSHOT` on `rc` today. + +### Run + +Run the examples from the repository root. They find `logback.xml`, the data +file and their options file with `FileFinder`, which searches only below the +folder Java runs in (`pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/FileFinder.java`). +When a free data file is not there, the example finds its `.link` the same way +and runs the download script for it (`findDataFile` in each examples folder's +`shared` module, `DataFileHelper.java`), with Windows PowerShell on Windows and +`sh` on Linux and macOS. The same commands work on Windows, Linux and macOS. + +``` +java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedOnPrem +``` + +The IP Intelligence example looks for the paid Enterprise file, +`ip-intelligence/51Degrees-EnterpriseIpiV41.ipi`, so give it the Lite file as +its first argument. + +``` +java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.GettingStartedOnPrem cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +``` + +The example downloads the Lite file first when it is not there. In place of +the argument, the `51DEGREES_IPI_PATH` environment variable or system property +can give the absolute path of a data file that is already there. + +On Java 24 or later, add `--enable-native-access=ALL-UNNAMED` before the class +name, as +[examples/device-detection/README.md](examples/device-detection/README.md#native-library-access) +explains, or Java warns about the native library. + +### Correct output + +When an example downloads its data file, a Logback line first reports +`'' is missing, so it is being downloaded by running` and the command, +and the script's `Downloading` and `Saved` lines follow. + +The Device Detection example processes five sets of evidence. For each it +prints `Input values:` and the evidence, then `Results:` with +`Mobile Device`, `Platform Name`, `Platform Version`, `Browser Name` and +`Browser Version`. Logback lines come before and after. The last ones report +the data file and end the run, with these messages. + +``` +Used a 'Lite' data file, created , days ago, from location '' +This example is using the 'Lite ' data file. This is used for illustration, ... +All done +``` + +A warning that the data file is more than 28 days old may come too, which is +expected with the Lite file. The exit code is 0. + +The IP Intelligence example looks up `116.154.188.222`, `45.236.48.61` and an +IPv6 address. For each it prints `Input values:` with `query.client-ip`, then +`Results:` with a line for each property, from `RegisteredName` to +`TimeZoneOffset`. The Lite file has RegisteredCountry, RegisteredName and +RegisteredOwner, so a line for a property it does not have reads +`Unknown. The property '' is not available in this data file.` followed +by a link, and the first such line also logs a warning that some properties +need a paid subscription. It then logs the data file in the same way, and +`All done`. + +The paid data files have many more properties, as the +[pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=correct-output) +describes. + +### Cloud examples + +The cloud examples need a resource key, which a person creates in the +configurator, so ask for one if none is set. A key with the properties most of +the Device Detection examples use can be made +[here](https://configure.51degrees.com/jqz435Nc?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=cloud-examples). +The Device Detection cloud example takes the key as its first argument, or +from a `TestResourceKey` environment variable or system property. + +``` +java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedCloud +``` + +It prints the same `Input values:` and `Results:` blocks as the on-premise +example. The IP Intelligence cloud examples read their key from +`51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and +`examples/ip-intelligence/README.md` gives the keys to create for them. + +### Pitfalls + +- **A data file that is not there.** `cxx/data` holds a `.link` for each file + rather than the file. An example whose file is missing runs the download + script for it, so it needs the network and the tools under "Prerequisites". + Check the file sizes as above. +- **Missing submodules.** The example then logs `Could not find the data file + 'device-detection/51Degrees-LiteV4.1.hash' which must be somewhere in the + project space to be found.` Run `git submodule update --init --recursive`. A + zip download of the repository has no submodules, so clone with Git. +- **Running from another folder.** `FileFinder` looks only below the folder + Java runs in, so `java` started in `examples/device-detection`, as the + examples' README shows, cannot find the data file or its link in `cxx`. + Start it from the repository root. The same applies to `mvn exec:java`, + which runs in the folder Maven starts in. +- **The version in the jar name.** The examples' READMEs show jar names with + `4.4.20` and `4.4.19`, which are out of date. Use the name in `target`. +- **The IP Intelligence example without an argument.** It searches for the + Enterprise file, logs `Failed to find IP Intelligence data file` and stops, + even when the Lite file is in place. +- **The `dev` profile.** `-Pdev`, which CI uses, builds the examples against + the libraries here, which needs `mvn install` at the root first and so the + native build, with CMake 3.10 or later and a C++ toolchain. It is not needed + to run the examples. +- **Platforms the packages cover.** The published + `device-detection.hash.engine.on-premise` 4.5.128 jar has native libraries + for Windows x64 and x86, Linux x64 and aarch64, and macOS x64 and aarch64. + The `ip-intelligence.engine.on-premise` 4.5.128 jar has the same, without + Windows x86. On a platform a jar lacks, the example cannot load the engine. +- **What CI does not test.** On a pull request CI builds and tests the + examples with the `dev` profile (`ci/build-project.ps1` and + `ci/run-unit-tests.ps1`), and against the published packages only when it + publishes (`ci/run-integration-tests.ps1`). It never runs the console + example jars as above, and the example tests use the paid data files when + CI has them. ## One reactor, six package trees @@ -53,6 +272,14 @@ The Windows pre-build scripts take a toolset and SDK version from the pom. JavaScript templates and the OWID library are submodules inside `pipeline`, where the Pipeline API expects them. +## Releases + +Pull requests go to `rc`, not `main`, and stay drafts until they have been +reviewed, because the nightly merges any pull request into `rc` that is not a +draft and has no review requested. When a release is wanted, one pull request +merges `rc` into `main`, and the "Only rc into main" check fails a pull request +into `main` from any other branch. + ## Writing Plain words, no em dashes, sentence case headings. Nothing in this public From 086322c51744b8b6e51ca9bc6f76fe82fa69f829 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:08:22 +0000 Subject: [PATCH 11/14] Point package metadata at this repository and fix stale links (#42) --- ci/README.md | 2 +- device-detection/README.md | 8 ++--- .../device-detection.cloud/pom.xml | 2 +- .../pom.xml | 2 +- .../device-detection.shared/pom.xml | 2 +- device-detection/device-detection/pom.xml | 2 +- device-detection/pom.xml | 8 +++-- examples/device-detection/README.md | 29 ++++++++----------- .../examples/console/GettingStartedCloud.java | 2 +- .../console/GettingStartedOnPrem.java | 2 +- .../examples/console/MetadataOnPrem.java | 2 +- .../examples/console/MinimalExample.java | 2 +- .../examples/console/NativeModelCloud.java | 2 +- .../examples/console/TacCloud.java | 2 +- examples/ip-intelligence/README.md | 8 ++--- .../examples/console/CompareOnPrem.java | 2 +- .../examples/console/GettingStartedCloud.java | 2 +- .../console/GettingStartedOnPrem.java | 2 +- .../examples/console/MetricsOnPrem.java | 2 +- .../examples/console/SuspiciousOnPrem.java | 6 ++-- ip-intelligence/README.md | 8 ++--- ip-intelligence/ip-intelligence.cloud/pom.xml | 2 +- .../ip-intelligence.engine.on-premise/pom.xml | 2 +- .../ip-intelligence.shared/pom.xml | 2 +- .../ip-intelligence.translation/pom.xml | 2 +- ip-intelligence/ip-intelligence/pom.xml | 2 +- ip-intelligence/pom.xml | 8 +++-- location/README.md | 6 ++-- location/pom.xml | 6 ++-- pipeline/README.md | 4 +-- pipeline/pipeline.caching/pom.xml | 2 +- pipeline/pipeline.cloudrequestengine/pom.xml | 2 +- .../cloudrequestengine/Constants.java | 2 +- pipeline/pipeline.common/pom.xml | 2 +- pipeline/pipeline.core/pom.xml | 2 +- pipeline/pipeline.did/pom.xml | 6 +++- pipeline/pipeline.engines.fiftyone/pom.xml | 2 +- pipeline/pipeline.engines/pom.xml | 2 +- .../fiftyone/pipeline/engines/Constants.java | 4 +-- pipeline/pipeline.javascriptbuilder/pom.xml | 2 +- pipeline/pipeline.jsonbuilder/pom.xml | 2 +- pipeline/pipeline.translation/pom.xml | 2 +- .../pipeline.web.mvc/pom.xml | 2 +- .../pipeline.web.shared/pom.xml | 2 +- .../pipeline.web/pom.xml | 2 +- pipeline/pom.xml | 6 ++-- 46 files changed, 90 insertions(+), 83 deletions(-) diff --git a/ci/README.md b/ci/README.md index a537b0271..bf6b2a62b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -55,7 +55,7 @@ To run the on-premise tests: To run the cloud tests: -- `_51DEGREES_RESOURCE_KEY_BESPOKE`: [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) +- `_51DEGREES_RESOURCE_KEY_BESPOKE`: [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) passed to the tests as `TestResourceKey` and `SuperResourceKey`. - `_51DEGREES_LICENSE_KEY_51DID`: license key for the 51Did cloud tests, together with the `_51DEGREES_RESOURCE_KEY_BASIC` and diff --git a/device-detection/README.md b/device-detection/README.md index 73722fcb5..5441e364a 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -11,7 +11,7 @@ This repository contains Java implementation of the device detection [specificat ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -21,9 +21,9 @@ The Java API can either use our cloud service to get its data or it can use a lo #### Cloud -You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) +You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on +our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise @@ -225,7 +225,7 @@ repository. ## Tests -You will need [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) +You will need [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) (see above) to complete the tests and run examples which include exercising the cloud API. To verify the code: diff --git a/device-detection/device-detection.cloud/pom.xml b/device-detection/device-detection.cloud/pom.xml index 4e6c839b7..ddfbaf069 100644 --- a/device-detection/device-detection.cloud/pom.xml +++ b/device-detection/device-detection.cloud/pom.xml @@ -12,7 +12,7 @@ device-detection.cloud 51Degrees :: Device Detection :: Cloud Cloud-based device detection for Java that uses the 51Degrees cloud service via the Pipeline API. Parses HTTP headers to identify device hardware, operating systems, browsers, and crawlers. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection.cloud-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.hash.engine.on-premise/pom.xml b/device-detection/device-detection.hash.engine.on-premise/pom.xml index 87250bfe8..7d141d302 100644 --- a/device-detection/device-detection.hash.engine.on-premise/pom.xml +++ b/device-detection/device-detection.hash.engine.on-premise/pom.xml @@ -12,7 +12,7 @@ device-detection.hash.engine.on-premise 51Degrees :: Device Detection :: Hash :: On Premise High-performance on-premise hash-based device detection engine for Java, built for the 51Degrees Pipeline API. Parses HTTP headers locally to identify device hardware, operating systems, browsers, and crawlers, with no external service calls. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection.hash.engine.on-premise-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.shared/pom.xml b/device-detection/device-detection.shared/pom.xml index 202797eb3..7d2618c67 100644 --- a/device-detection/device-detection.shared/pom.xml +++ b/device-detection/device-detection.shared/pom.xml @@ -12,7 +12,7 @@ device-detection.shared 51Degrees :: Device Detection :: Shared Shared components and utilities for building 51Degrees Pipeline API device detection engines in Java. Supports parsing HTTP headers to identify device type, model, OS, browser, and crawler information—an alternative approach to UAParser, DeviceAtlas, and WURFL-based solutions. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection.shared-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection/pom.xml b/device-detection/device-detection/pom.xml index 7ffa7b7d3..96eefb171 100644 --- a/device-detection/device-detection/pom.xml +++ b/device-detection/device-detection/pom.xml @@ -13,7 +13,7 @@ device-detection 51Degrees :: Device Detection Parse HTTP headers to detect the device type, model, OS, browser, and crawler information. This is an alternative to popular UAParser, DeviceAtlas, and WURFL packages. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=device-detection-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/device-detection/pom.xml b/device-detection/pom.xml index 8aef20aea..ce56dd201 100644 --- a/device-detection/pom.xml +++ b/device-detection/pom.xml @@ -1,4 +1,4 @@ - 4.0.0 @@ -9,7 +9,11 @@ pom 51Degrees :: Pipeline :: Device Detection - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java + + 51Degrees + https://51degrees.com/device-detection?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.device-detection&utm_term=organization + Device detection engines for the 51Degrees Pipeline API diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 1f42ecd77..4c80193e5 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -31,9 +31,9 @@ and logs the command it runs. ## Cloud resource keys -You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) +You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on +our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on how to use this. A resource key configured with the properties needed @@ -49,23 +49,18 @@ resource keys and license keys at our [pricing page](https://51degrees.com/prici ## Running examples with changes to Pipeline packages -A common use case is to make a change to the Pipeline logic in -device-detection-java and then use these examples to observe the results of the -change. +A common use case is to make a change to the Device Detection or Pipeline +packages in this repository and then use these examples to observe the +results of the change. -By default, the examples are configured to use the packages from Maven central. -In order to produce and use local packages instead: +By default, the examples use the packages from Maven Central. To build them +against the packages in this repository instead, run Maven from the root of +the repository with the `dev` profile, as the +[root README](../../README.md#examples) describes: -- Clone and make your changes to device-detection-java -- Set the version of the device detection packages that we're going to create to 0.0.0: - `mvn versions:set-property -Dproperty="project.version" "-DnewVersion=0.0.0"` -- Create and install the packages locally (skipping tests is needed): - `mvn clean install [-DskipTests]` -- Modify the POM for the examples to reference these new local packages. This can - be done by editing the POM directly or by using the command line: - `mvn versions:set-property -Dproperty="device-detection.version" "-DnewVersion=0.0.0"` - -The same principle can be applied to incorporate changes in pipeline-java if needed. +``` +mvn install -DskipTests -Pdev +``` ## Examples diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java index c97a1868a..846e1fcb4 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java @@ -84,7 +84,7 @@ public static void run(String resourceKey, /* In this example, we use the DeviceDetectionPipelineBuilder and configure it in code. For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=run + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=run Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ try (Pipeline pipeline = new DeviceDetectionPipelineBuilder() diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 332ef1dca..44b7227fb 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -102,7 +102,7 @@ public static void run(String dataFile, options contained in the file "gettingStartedOnPrem.xml". For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=run + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=run Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ // the configuration file is in the resources directory diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 1dd9b1d21..5b6939705 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -81,7 +81,7 @@ public static void run(String dataFile, OutputStream output) throws Exception { // example. See the documentation for more detail on this and other // configuration options: // https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=run - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=run + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=run .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) // inhibit auto-update of the data file for this test .setAutoUpdate(false) diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java index 4b996daba..8c5fb5206 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java @@ -29,7 +29,7 @@ /** * This is the code for the minimal example displayed in the configurator. Please see - * + * * Getting Started Console Cloud for a fuller example. */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index 754f0596a..3c6c724ac 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -56,7 +56,7 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // This example creates the pipeline and engines in code. For a demonstration // of how to do this using a configuration file instead, see the TacCloud example. // For more information about builders in general see the documentation at - // https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=run + // https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=run resourceKey = getOrSetSuperResourceKey(resourceKey, NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME); try (PrintWriter output = new PrintWriter(os)) { diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index 91213032b..ee5ee1e0b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -72,7 +72,7 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // For a demonstration of how to do this in code instead, see the // NativeModelLookup example. // For more information about builders in general see the documentation at - // https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=run + // https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=run // the configuration file is in the resources directory File optionsFile = getFilePath("tacCloud.xml"); diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 3ebef2ce6..9344df4e0 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -129,14 +129,14 @@ to get a paid subscription with more properties. ## Running Examples and Tests -**IMPORTANT**: All examples and tests must be run from the repository root directory (`ip-intelligence-java-examples/`). +**IMPORTANT**: All examples and tests must be run from this folder, `examples/ip-intelligence`. ### Running Tests To run all tests: ```bash -# From the repository root directory +# From examples/ip-intelligence mvn test ``` @@ -155,14 +155,14 @@ mvn test -pl web/getting-started.onprem Console examples can be run directly using Maven exec plugin: ```bash -# From the repository root directory +# From examples/ip-intelligence mvn compile exec:java -pl console -Dexec.mainClass="fiftyone.ipintelligence.examples.console.GettingStartedOnPrem" ``` Web examples: ```bash -# From the repository root directory +# From examples/ip-intelligence mvn compile exec:java -pl web/getting-started.onprem -Dexec.mainClass="fiftyone.ipintelligence.examples.web.GettingStartedWebOnPrem" ``` diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index b8df87518..4b994e3b9 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -202,7 +202,7 @@ public static void run(String dataFile, // We use the max performance profile for optimal lookup // speed in this example. See the documentation for more detail // on this and other configuration options. - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) // inhibit sharing usage for this example .setShareUsage(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java index fb63e8546..684990a91 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java @@ -125,7 +125,7 @@ public static void run(String resourceKey, /* In this example, we use the IPIntelligencePipelineBuilder and configure it in code. For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java&utm_term=configuration-builders + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java&utm_term=configuration-builders Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ IPIntelligenceCloudPipelineBuilder builder = new IPIntelligencePipelineBuilder() diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 8c6e4d0b9..41a7712a5 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -139,7 +139,7 @@ public static void run(String dataFile, options contained in the file "gettingStartedOnPrem.xml". For more information about pipelines in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=run + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=run Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ // the configuration file is in the resources directory diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index aaf6ddac4..6c66b486f 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -343,7 +343,7 @@ public static void run(String dataFile, // detection speed in this example. See the // documentation for more detail on this and other // configuration options. - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) // inhibit sharing usage for this example .setShareUsage(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index dd46f9cc5..3df68b732 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -120,7 +120,7 @@ public static void run(String dataFile, /* In this example, we use the IPIntelligencePipelineBuilder and configure it in code. For more information about builders in general see the documentation at - https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=configuration-builders + https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=configuration-builders Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ try (Pipeline pipeline = new IPIntelligencePipelineBuilder() @@ -128,8 +128,8 @@ public static void run(String dataFile, // We use the max performance profile for optimal lookup speed in this // example. See the documentation for more detail on this and other // configuration options. - // https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates - // https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing + // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates + // https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) // inhibit sharing usage for this example, usually this should be set to "true" .setShareUsage(false) diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index 3d194b87b..8f4534ed9 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -11,7 +11,7 @@ This repository contains Java implementation of the IP Intelligence [specificati ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -23,9 +23,9 @@ The Java API can either use our cloud service to get its data or it can use a lo The cloud service for IP Intelligence is currently in development. -You will require [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) +You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon), see our [documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) on +our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) on how to use this. #### On-Premise @@ -108,7 +108,7 @@ repository. ## Tests -You will need [resource keys](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) +You will need [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) (see above) to complete the tests and run examples which include exercising the cloud API. The tests look for a resource key in the following order: diff --git a/ip-intelligence/ip-intelligence.cloud/pom.xml b/ip-intelligence/ip-intelligence.cloud/pom.xml index 9a1d434f4..cfc6ffa32 100644 --- a/ip-intelligence/ip-intelligence.cloud/pom.xml +++ b/ip-intelligence/ip-intelligence.cloud/pom.xml @@ -12,7 +12,7 @@ ip-intelligence.cloud 51Degrees :: IP Intelligence :: Cloud Provides IP Intelligence for the 51Degrees Pipeline API using the 51Degrees Cloud service. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.cloud-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml index 00b37b646..0c48477de 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml +++ b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml @@ -12,7 +12,7 @@ ip-intelligence.engine.on-premise 51Degrees :: IP Intelligence :: On Premise Provides the On-premise IP Intelligence engine for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.engine.on-premise-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.shared/pom.xml b/ip-intelligence/ip-intelligence.shared/pom.xml index 71ab0f830..3f2271d0a 100644 --- a/ip-intelligence/ip-intelligence.shared/pom.xml +++ b/ip-intelligence/ip-intelligence.shared/pom.xml @@ -12,7 +12,7 @@ ip-intelligence.shared 51Degrees :: IP Intelligence :: Shared Shared code used by 51Degrees IP Intelligence modules for the Pipeline API. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.shared-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.translation/pom.xml b/ip-intelligence/ip-intelligence.translation/pom.xml index a620ab341..98c629255 100644 --- a/ip-intelligence/ip-intelligence.translation/pom.xml +++ b/ip-intelligence/ip-intelligence.translation/pom.xml @@ -14,7 +14,7 @@ Translation engines that turn the weighted ISO country codes from 51Degrees IP Intelligence into localized, ordered country name and code lists. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence.translation-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence/pom.xml b/ip-intelligence/ip-intelligence/pom.xml index b22ba399b..2baf5e0fd 100644 --- a/ip-intelligence/ip-intelligence/pom.xml +++ b/ip-intelligence/ip-intelligence/pom.xml @@ -13,7 +13,7 @@ ip-intelligence 51Degrees :: IP Intelligence Provides IP Intelligence for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=ip-intelligence-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/pom.xml b/ip-intelligence/pom.xml index 4327c14b2..16a23b1a3 100644 --- a/ip-intelligence/pom.xml +++ b/ip-intelligence/pom.xml @@ -1,4 +1,4 @@ - 4.0.0 @@ -9,7 +9,11 @@ pom 51Degrees :: Pipeline :: IP Intelligence - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java + + 51Degrees + https://51degrees.com/ip-intelligence?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.ip-intelligence&utm_term=organization + IP Intelligence engines for the 51Degrees Pipeline API diff --git a/location/README.md b/location/README.md index b48074eaa..031f81f12 100644 --- a/location/README.md +++ b/location/README.md @@ -10,16 +10,16 @@ This repository contains the geo-location engines for the Java implementation of ## Dependencies -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ### Data -You will require a [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) +You will require a [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) to use the Cloud API. You can create resource keys using our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data), see our -[documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) on how to use this. +[documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) on how to use this. ## Installation diff --git a/location/pom.xml b/location/pom.xml index e3e332f3f..1939c6520 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -20,7 +20,7 @@ ~ such notice(s) shall fulfill the requirements of that article. --> - 4.0.0 @@ -31,12 +31,12 @@ pom 51Degrees :: Pipeline :: Geo Location - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java Geo location engines for the 51Degrees Pipeline API 51Degrees - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url-2 + https://51degrees.com/geolocation?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.geo-location&utm_term=organization diff --git a/pipeline/README.md b/pipeline/README.md index a860e40c0..8370cc09b 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -16,7 +16,7 @@ Reference documentation for the Java implementation can be found on the ## Pre-requisites -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pre-requisites) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pre-requisites) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -123,7 +123,7 @@ There are several examples available in the `pipeline.developer-examples` folder how to make use of the Pipeline API in isolation. These are described in the table below. If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [repository](https://github.com/51Degrees/51degrees-java/tree/main/device-detection) -and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=examples). +and on our [website](https://51degrees.com/documentation/_device_detection__examples__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=examples). | Example | Description | |----------------------------------------------------|-------------| diff --git a/pipeline/pipeline.caching/pom.xml b/pipeline/pipeline.caching/pom.xml index fba348a5b..2286dd55d 100644 --- a/pipeline/pipeline.caching/pom.xml +++ b/pipeline/pipeline.caching/pom.xml @@ -33,7 +33,7 @@ pipeline.caching 51Degrees :: Pipeline :: Caching - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.caching-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.cloudrequestengine/pom.xml b/pipeline/pipeline.cloudrequestengine/pom.xml index 38d1e2115..2871e320d 100644 --- a/pipeline/pipeline.cloudrequestengine/pom.xml +++ b/pipeline/pipeline.cloudrequestengine/pom.xml @@ -34,7 +34,7 @@ pipeline.cloudrequestengine 51Degrees :: Pipeline :: Cloud Request Engine Cloud request engine for the 51Degrees Pipeline API - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index 2e982f7e1..593c618d6 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -46,7 +46,7 @@ public class Messages { + "because your resource key does not include access to any " + "properties under '%s'. For more details on resource keys, " + "see our explainer: " - + "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; public static final String ProcessCloudEngineNotImplemented = "This method should be overridden in the class derived from " diff --git a/pipeline/pipeline.common/pom.xml b/pipeline/pipeline.common/pom.xml index 68da0f135..cf2089abf 100644 --- a/pipeline/pipeline.common/pom.xml +++ b/pipeline/pipeline.common/pom.xml @@ -33,7 +33,7 @@ pipeline.common 51Degrees :: Pipeline :: Common - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.common-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.core/pom.xml b/pipeline/pipeline.core/pom.xml index 3da59fe4c..3e30f3f64 100644 --- a/pipeline/pipeline.core/pom.xml +++ b/pipeline/pipeline.core/pom.xml @@ -38,7 +38,7 @@ 51Degrees :: Pipeline :: Core Core library for the 51Degrees Pipeline API - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.core-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java diff --git a/pipeline/pipeline.did/pom.xml b/pipeline/pipeline.did/pom.xml index 21a2074c1..c5ba43ddc 100644 --- a/pipeline/pipeline.did/pom.xml +++ b/pipeline/pipeline.did/pom.xml @@ -39,7 +39,11 @@ value returned by the 51Degrees Cloud service. Parses the OWID envelope and exposes the Flags, License Id and match key plus the identifier type. Compare match keys, never envelopes. - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.did-pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java + + 51Degrees + https://51degrees.com/developers/51did?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.did&utm_term=organization + - 4.0.0 @@ -31,12 +31,12 @@ pom 51Degrees :: Pipeline - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url + https://github.com/51Degrees/51degrees-java The 51Degrees Pipeline API provides architecture for consuming real-time digital data services. 51Degrees - https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pom.xml&utm_term=url-2 + https://51degrees.com/build-your-own?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline&utm_term=organization From 377323863270fdd3362483330f2691e58e1f30b8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 18:08:44 +0000 Subject: [PATCH 12/14] Describe the rc approach to releases in AGENTS.md (#39) From 59e13f17fb17fa8d16f9db5e2020e360f7fc34a8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 20:51:42 +0000 Subject: [PATCH 13/14] Check the callback URL before the JavaScript builder writes it (#40) --- .../flowelements/CallbackUrl.java | 358 ++++++++++++++++++ .../JavaScriptBuilderElement.java | 72 ++-- .../javascriptbuilder/JavaScriptUrlTests.java | 129 +++++++ .../flowelements/CallbackUrlTests.java | 283 ++++++++++++++ 4 files changed, 816 insertions(+), 26 deletions(-) create mode 100644 pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java create mode 100644 pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java new file mode 100644 index 000000000..4118f86f9 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java @@ -0,0 +1,358 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.flowelements; + +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * The callback URL the client script posts to for updated JSON. The protocol + * and host usually come from the request, and the script holds the URL in a + * single-quoted string, so the URL is checked before it is written into the + * script. A URL that is not valid is refused. In the user information, in + * anything after the host and port, and in the endpoint, the characters a + * URL cannot hold or that could end the string or the script are + * percent-encoded. + * @see Specification + */ +final class CallbackUrl { + + /** + * The largest port number. + */ + private static final int MAX_PORT = 65535; + + /** + * The most digits a port can have once its leading zeros are dropped. + */ + private static final int MAX_PORT_DIGITS = 5; + + /** + * Written in place of a lone surrogate, which has no UTF-8 encoding, as a + * browser writes it. + */ + private static final int REPLACEMENT_CHARACTER = 0xFFFD; + + /** + * The printable ASCII characters that are percent-encoded, besides the + * control characters, the space and everything outside ASCII. + */ + private static final String ENCODED = "\"'<>\\^`{|}"; + + private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray(); + + /** + * Text made only of the Unicode White_Space characters, which counts as + * a part that is not set. + */ + private static final Pattern BLANK = Pattern.compile( + "[\\t\\n\\x0B\\f\\r \\u0085\\u00A0\\u1680\\u2000-\\u200A" + + "\\u2028\\u2029\\u202F\\u205F\\u3000]*"); + + /** + * A scheme as RFC 3986 section 3.1 defines one, being a letter followed + * by letters, digits, '+', '-' and '.'. + */ + private static final Pattern SCHEME = + Pattern.compile("[A-Za-z][A-Za-z0-9+.-]*"); + + /** + * A label of a host name, being ASCII letters, digits, '-' and '_'. + */ + private static final Pattern LABEL = Pattern.compile("[A-Za-z0-9_-]+"); + + /** + * An IPv6 zone identifier, being unreserved characters as RFC 3986 + * section 2.3 defines them. + */ + private static final Pattern ZONE = Pattern.compile("[A-Za-z0-9._~-]+"); + + /** + * A port as it follows the host, being ':' and any number of digits. + */ + private static final Pattern PORT = Pattern.compile(":[0-9]*"); + + /** + * One group of an IPv6 address. + */ + private static final Pattern HEX_GROUP = + Pattern.compile("[0-9A-Fa-f]{1,4}"); + + /** + * An IPv4 address as RFC 3986 section 3.2.2 defines one, being four + * numbers from 0 to 255 with no leading zeros. + */ + private static final Pattern IPV4 = Pattern.compile( + "(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])" + + "(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}"); + + /** + * Whether the URL can be written into the script. + */ + final boolean valid; + + /** + * The URL to write into the script when it is valid, and otherwise the + * URL as it was built, for the log. + */ + final String url; + + private CallbackUrl(boolean valid, String url) { + this.valid = valid; + this.url = url; + } + + /** + * Builds the callback URL from its parts and checks it. One slash + * separates the host and the endpoint, being added when neither has one + * and removed when both do. The protocol and the host are written in + * lower case and the rest as given, apart from the percent-encoding. + * @param protocol the protocol, such as https + * @param host the host, which can start with user information and end + * with a port, a path, a query or a fragment + * @param endpoint the endpoint + * @return the callback URL, or null when the protocol, host or endpoint + * is not set + */ + static CallbackUrl of(String protocol, String host, String endpoint) { + if (isBlank(protocol) || isBlank(host) || isBlank(endpoint)) { + return null; + } + boolean hostSlash = host.endsWith("/"); + boolean endpointSlash = endpoint.startsWith("/"); + if (hostSlash == false && endpointSlash == false) { + endpoint = "/" + endpoint; + } else if (hostSlash && endpointSlash) { + endpoint = endpoint.substring(1); + } + String built = protocol + "://" + host + endpoint; + if (SCHEME.matcher(protocol).matches() == false) { + return new CallbackUrl(false, built); + } + int end = authorityEnd(host); + String authority = checkedAuthority(host.substring(0, end)); + if (authority == null) { + return new CallbackUrl(false, built); + } + return new CallbackUrl( + true, + protocol.toLowerCase(Locale.ROOT) + "://" + authority + + encoded(host.substring(end)) + encoded(endpoint)); + } + + /** + * Whether a part of the URL is not set, being null, empty or only + * whitespace. + * @param text the part + * @return true if the part is not set + */ + static boolean isBlank(String text) { + return text == null || BLANK.matcher(text).matches(); + } + + /** + * Where the authority ends, which is at the first '/', '?' or '#' of the + * host, or at its end. + */ + private static int authorityEnd(String host) { + for (int i = 0; i < host.length(); i++) { + char c = host.charAt(i); + if (c == '/' || c == '?' || c == '#') { + return i; + } + } + return host.length(); + } + + /** + * The user information, host and port as they are written into the URL, + * or null when they are not valid. The user information ends at the + * first '@', so a second '@' is part of the host and refused. + */ + private static String checkedAuthority(String authority) { + int at = authority.indexOf('@'); + if (at < 0) { + return checkedHostPort(authority); + } + String hostPort = checkedHostPort(authority.substring(at + 1)); + return hostPort == null + ? null + : encoded(authority.substring(0, at)) + "@" + hostPort; + } + + /** + * The host and optional port in lower case, or null when they are not + * valid. The host is an IPv6 address in brackets with an optional zone + * identifier, or a name of labels separated by dots with a dot allowed + * at the end. The port is ':' followed by digits up to 65535, or by + * nothing. + */ + private static String checkedHostPort(String hostPort) { + String host; + String port; + if (hostPort.startsWith("[")) { + int close = hostPort.indexOf(']'); + if (close < 0) { + return null; + } + String address = hostPort.substring(1, close); + int zone = address.indexOf("%25"); + if (zone >= 0) { + if (ZONE.matcher(address.substring(zone + 3)).matches() == + false) { + return null; + } + address = address.substring(0, zone); + } + if (isIpv6(address) == false) { + return null; + } + host = hostPort.substring(0, close + 1); + port = hostPort.substring(close + 1); + } else { + int colon = hostPort.indexOf(':'); + int end = colon < 0 ? hostPort.length() : colon; + host = hostPort.substring(0, end); + port = hostPort.substring(end); + String name = host.endsWith(".") + ? host.substring(0, host.length() - 1) + : host; + for (String label : name.split("\\.", -1)) { + if (LABEL.matcher(label).matches() == false) { + return null; + } + } + } + if (port.isEmpty() == false && isPort(port) == false) { + return null; + } + return host.toLowerCase(Locale.ROOT) + port; + } + + /** + * Whether the text is ':' followed by digits whose value is no more than + * 65535, or by nothing. + */ + private static boolean isPort(String text) { + if (PORT.matcher(text).matches() == false) { + return false; + } + String digits = text.substring(1).replaceFirst("^0+", ""); + return digits.length() <= MAX_PORT_DIGITS && + (digits.isEmpty() || Integer.parseInt(digits) <= MAX_PORT); + } + + /** + * Whether the text is an IPv6 address as RFC 3986 section 3.2.2 defines + * one. That is eight groups of one to four hexadecimal digits separated + * by ':', where one '::' can stand for one or more groups of zeros and + * the last two groups can be written as an IPv4 address. + */ + private static boolean isIpv6(String text) { + int gap = text.indexOf("::"); + if (gap < 0) { + return groups(text, true) == 8; + } + if (text.indexOf("::", gap + 1) >= 0) { + return false; + } + int head = groups(text.substring(0, gap), false); + int tail = groups(text.substring(gap + 2), true); + return head >= 0 && tail >= 0 && head + tail <= 7; + } + + /** + * The number of 16 bit groups in a run of groups separated by ':', or -1 + * when the run is not valid. An empty run has none, and an IPv4 address + * at the end of the run, where one is allowed, counts as two. + */ + private static int groups(String run, boolean ipv4AtEnd) { + if (run.isEmpty()) { + return 0; + } + String[] parts = run.split(":", -1); + int count = 0; + for (int i = 0; i < parts.length; i++) { + if (HEX_GROUP.matcher(parts[i]).matches()) { + count++; + } else if (ipv4AtEnd && i == parts.length - 1 && + IPV4.matcher(parts[i]).matches()) { + count += 2; + } else { + return -1; + } + } + return count; + } + + /** + * The text with each byte of its UTF-8 encoding percent-encoded where + * the byte is a control character, a space, outside ASCII or one of + * ENCODED, and with a '%' encoded where two hexadecimal digits do not + * follow it. + */ + private static String encoded(String text) { + byte[] bytes = utf8(text); + StringBuilder out = new StringBuilder(bytes.length); + for (int i = 0; i < bytes.length; i++) { + int b = bytes[i] & 0xFF; + boolean keep = b == '%' + ? i + 2 < bytes.length && + isHexDigit(bytes[i + 1]) && isHexDigit(bytes[i + 2]) + : b > ' ' && b < 0x7F && ENCODED.indexOf(b) < 0; + if (keep) { + out.append((char) b); + } else { + out.append('%') + .append(HEX_DIGITS[b >> 4]) + .append(HEX_DIGITS[b & 0x0F]); + } + } + return out.toString(); + } + + private static boolean isHexDigit(byte b) { + return (b >= '0' && b <= '9') || + (b >= 'A' && b <= 'F') || + (b >= 'a' && b <= 'f'); + } + + /** + * The UTF-8 encoding of the text, with each lone surrogate written as + * U+FFFD. + */ + private static byte[] utf8(String text) { + StringBuilder whole = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + int c = text.codePointAt(i); + i += Character.charCount(c); + whole.appendCodePoint( + c >= Character.MIN_SURROGATE && c <= Character.MAX_SURROGATE + ? REPLACEMENT_CHARACTER + : c); + } + return whole.toString().getBytes(StandardCharsets.UTF_8); + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 5ad34579e..f8d828dc5 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -42,6 +42,7 @@ import java.io.*; import java.net.URLEncoder; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static fiftyone.pipeline.core.Constants.*; @@ -68,6 +69,12 @@ public class JavaScriptBuilderElement protected final boolean enableCookies; private final Mustache mustache; + /** + * Set once a callback URL that is not valid has been logged, because a + * misconfigured element would otherwise log on every request. + */ + private final AtomicBoolean invalidUrlLogged = new AtomicBoolean(false); + /** * The object name is written into the script as the name of a global * variable, as a session storage key and inside string literals, with no @@ -392,35 +399,48 @@ private String serializeParameters(Map parameters) { return jsonObject.toString(0); } + /** + * Gets the callback URL, or null when the protocol, host or endpoint is + * not set or the URL is not valid, as {@link CallbackUrl} decides. A URL + * that is not valid is logged as an error the first time only, quoted + * and escaped so that it cannot start a new log line. + */ private String getUrl(String protocol, String host) { - String url = null; - if (protocol != null && !protocol.isEmpty() && - host != null && !host.isEmpty() && - endpoint != null && !endpoint.isEmpty()) { - boolean contextRootPopulated = contextRoot != null && - !contextRoot.isEmpty() && !contextRoot.equals("/"); - - // Make sure that each part of the URL except host starts with a '/' - // and each part except endpoint does NOT end with one. - if (endpoint.charAt(0) != '/') { - endpoint = "/" + endpoint; - } - if (contextRootPopulated && contextRoot.charAt(0) != '/') { - contextRoot = "/" + contextRoot; - } - if (host.charAt(host.length() - 1) == '/') { - host = host.substring(0, host.length() - 1); - } - if (contextRootPopulated && contextRoot.charAt(contextRoot.length() - 1) == '/') { - contextRoot = contextRoot.substring(0, contextRoot.length() - 1); - } - - url = protocol + "://" + host + - (contextRootPopulated ? contextRoot : "") + - endpoint; + CallbackUrl url = CallbackUrl.of(protocol, host, getPath()); + if (url == null) { + return null; + } + if (url.valid) { + return url.url; } + if (invalidUrlLogged.compareAndSet(false, true)) { + logger.error("The generated URL " + JSONObject.quote(url.url) + + " is not valid, so the JavaScript has no callback URL. " + + "Check the JavaScript builder's protocol, host, context " + + "root and endpoint, and the protocol and host evidence. " + + "This is reported once."); + } + return null; + } - return url; + /** + * Gets the endpoint with the context root in front of it, where there is + * a context root. The context root starts with a '/' and does not end + * with one, and a '/' joins it to the endpoint. + */ + private String getPath() { + boolean contextRootPopulated = contextRoot != null && + contextRoot.isEmpty() == false && contextRoot.equals("/") == false; + if (contextRootPopulated == false || CallbackUrl.isBlank(endpoint)) { + return endpoint; + } + String root = contextRoot.charAt(0) == '/' + ? contextRoot + : "/" + contextRoot; + if (root.charAt(root.length() - 1) == '/') { + root = root.substring(0, root.length() - 1); + } + return root + (endpoint.charAt(0) == '/' ? endpoint : "/" + endpoint); } /** diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index 4f3fc0dea..a94604104 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.javascriptbuilder; +import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -145,4 +146,132 @@ public void Url_KeepsContextRoot() throws Exception { assertEquals("https://example.com/app/json", url); } } + + /** + * Loggers for the tests that read what the element logged. No logger is + * behind them, so each one keeps the name it was created with. + */ + private final TestLoggerFactory hostLoggerFactory = + new TestLoggerFactory(null); + + /** + * Renders one script for each host given, all through the same element, + * with the host taken from the request's evidence. + */ + private List renderWithHosts(String... hosts) throws Exception { + TestLoggerFactory factory = hostLoggerFactory; + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(factory) + .setEndpoint("/json") + .build(); + Pipeline pipeline = new PipelineBuilder(factory) + .addFlowElement(new SequenceElementBuilder(factory).build()) + .addFlowElement(new JsonBuilderElementBuilder(factory).build()) + .addFlowElement(jsElement) + .build(); + List scripts = new ArrayList<>(); + for (String host : hosts) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(Constants.EVIDENCE_HOST_KEY, host); + flowData.process(); + scripts.add( + flowData.getFromElement(jsElement).getJavaScript()); + } + } + return scripts; + } + + /** + * The errors the JavaScript builder element logged. + */ + private List builderErrors() { + List errors = new ArrayList<>(); + for (TestLogger logger : hostLoggerFactory.loggers) { + if (JavaScriptBuilderElement.class.getName().equals( + logger.getName())) { + errors.addAll(logger.errorsLogged); + } + } + return errors; + } + + /** + * A host the check refuses gives a script with no callback, and the + * refusal is logged as an error the first time only, whatever the + * requests in between. + */ + @Test + public void Url_RefusedHost_NoCallbackAndLoggedOnce() throws Exception { + List scripts = renderWithHosts( + "exa mple.com", "exa mple.com", "example.com", "exa mple.com"); + + for (int i = 0; i < scripts.size(); i++) { + String script = scripts.get(i); + assertTrue(script.contains("new fiftyoneDegreesManager()"), + "The script was not rendered."); + if (i == 2) { + assertFalse(requestUrls(script).isEmpty(), + "The valid host lost its callback."); + } else { + assertTrue(requestUrls(script).isEmpty(), + "A refused URL reached the script."); + assertFalse(script.contains("exa mple.com"), + "The refused host reached the script."); + } + } + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + assertTrue(errors.get(0).contains("\"https://exa mple.com/json\""), + errors.get(0)); + } + + /** + * The URL is quoted and escaped in the log, so a line break in it cannot + * start a new log line. + */ + @Test + public void Url_RefusedHost_LoggedOnOneLine() throws Exception { + renderWithHosts("a\nb.com\r\u2028"); + + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + String error = errors.get(0); + for (char c : new char[] {'\n', '\r', '\u2028'}) { + assertTrue(error.indexOf(c) < 0, error); + } + assertTrue(error.contains("a\\nb.com\\r\\u2028"), error); + } + + /** + * A host made only of whitespace is not set, so the script has no + * callback and nothing is logged. + */ + @Test + public void Url_BlankHost_NoCallbackAndNothingLogged() throws Exception { + String script = renderWithHosts(" ").get(0); + + assertTrue(requestUrls(script).isEmpty(), + "A blank host gave a callback."); + assertTrue(builderErrors().isEmpty(), + String.join("\n", builderErrors())); + } + + /** + * A host from the request reaches the script as the check writes it, + * in lower case and with anything that could end the string the script + * holds it in percent-encoded. + */ + @Test + public void Url_HostFromEvidence_WrittenAsChecked() throws Exception { + String script = + renderWithHosts("EXAMPLE.com/');alert(1);//").get(0); + + List urls = requestUrls(script); + assertFalse(urls.isEmpty(), "No request URL found in the script."); + for (String url : urls) { + assertEquals("https://example.com/%27);alert(1);//json", url); + } + assertFalse(script.contains("');alert(1)"), + "The host ended the string."); + } } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java new file mode 100644 index 000000000..fb7f6d91d --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java @@ -0,0 +1,283 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.flowelements; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Checks the callback URL against the cases every implementation of the + * JavaScript builder shares, in the same order, so that each one refuses + * and writes the same URLs. + */ +public class CallbackUrlTests { + + private static final String ENDPOINT = "/51dpipeline/json"; + + private static CallbackUrl url(String protocol, String host) { + return CallbackUrl.of(protocol, host, ENDPOINT); + } + + private static void assertValid(String expected, CallbackUrl url, + String given) { + assertNotNull(url, given + " gave no URL"); + assertTrue(url.valid, given + " was refused as " + url.url); + assertEquals(expected, url.url, given); + } + + private static void assertInvalid(CallbackUrl url, String given) { + assertNotNull(url, given + " gave no URL"); + assertFalse(url.valid, given + " was accepted as " + url.url); + } + + @Test + public void CallbackUrl_AddsASingleSlash() { + assertValid( + "https://example.com/51dpipeline/json", + CallbackUrl.of("https", "example.com", "51dpipeline/json"), + "an endpoint without a slash"); + } + + @Test + public void CallbackUrl_KeepsASingleSlashOnTheEndpoint() { + assertValid( + "https://example.com/51dpipeline/json", + CallbackUrl.of("https", "example.com", "/51dpipeline/json"), + "an endpoint with a slash"); + } + + @Test + public void CallbackUrl_CollapsesADoubleSlash() { + assertValid( + "http://example.com/json", + CallbackUrl.of("http", "example.com/", "/json"), + "a slash on both"); + } + + @Test + public void CallbackUrl_MissingWhenAPartIsEmpty() { + assertNull(CallbackUrl.of("https", "", "/json")); + assertNull(CallbackUrl.of("", "example.com", "/json")); + assertNull(CallbackUrl.of("https", "example.com", "")); + assertNull(CallbackUrl.of("https", " ", "/json")); + } + + static Stream acceptedHosts() { + return Stream.of( + Arguments.of("EXAMPLE.com", + "https://example.com/51dpipeline/json"), + Arguments.of("example.com:8080", + "https://example.com:8080/51dpipeline/json"), + Arguments.of("example.com:", + "https://example.com:/51dpipeline/json"), + Arguments.of("example.com:65535", + "https://example.com:65535/51dpipeline/json"), + Arguments.of("a_b.com", + "https://a_b.com/51dpipeline/json"), + Arguments.of("-ab.com", + "https://-ab.com/51dpipeline/json"), + Arguments.of("127.0.0.1:80", + "https://127.0.0.1:80/51dpipeline/json"), + Arguments.of("[::1]", + "https://[::1]/51dpipeline/json"), + Arguments.of("[::1]:8080", + "https://[::1]:8080/51dpipeline/json"), + Arguments.of("[fe80::1%25eth0]", + "https://[fe80::1%25eth0]/51dpipeline/json"), + Arguments.of("user:pw@example.com", + "https://user:pw@example.com/51dpipeline/json"), + Arguments.of("u x@example.com", + "https://u%20x@example.com/51dpipeline/json"), + Arguments.of("u\tx@example.com", + "https://u%09x@example.com/51dpipeline/json"), + Arguments.of("example.com.", + "https://example.com./51dpipeline/json"), + Arguments.of("example.com/path", + "https://example.com/path/51dpipeline/json"), + Arguments.of("example.com?x", + "https://example.com?x/51dpipeline/json"), + Arguments.of("example.com/a b", + "https://example.com/a%20b/51dpipeline/json"), + Arguments.of("example.com/a%zzb", + "https://example.com/a%25zzb/51dpipeline/json"), + Arguments.of("example.com/a%41b", + "https://example.com/a%41b/51dpipeline/json"), + Arguments.of("example.com/\u00fc", + "https://example.com/%C3%BC/51dpipeline/json"), + Arguments.of("example.com/a\nb", + "https://example.com/a%0Ab/51dpipeline/json"), + Arguments.of("example.com/a\u2028b", + "https://example.com/a%E2%80%A8b/51dpipeline/json")); + } + + @ParameterizedTest(name = "[{index}]") + @MethodSource("acceptedHosts") + public void CallbackUrl_AcceptsHost(String host, String expected) { + assertValid(expected, url("https", host), "host " + host); + } + + static Stream refusedHosts() { + return Stream.of( + "example.com:65536", + "example.com:99999", + "example.com:abc", + "exa mple.com", + "example.com');alert(1);//", + "a\"b.com", + "a'b.com", + "a hostsThatCouldEndTheString() { + return Stream.of( + "example.com/');alert(1);//", + "example.com?');alert(1);//", + "example.com#');alert(1);//", + "u'x@example.com", + "u\\x@example.com", + "example.com/a\\b", + "example.com/"); + } + + /** + * A quote, a backslash or a line break never reaches the script, wherever + * in the URL it was given. + */ + @ParameterizedTest(name = "[{index}]") + @MethodSource("hostsThatCouldEndTheString") + public void CallbackUrl_NothingGivenCanEndTheString(String host) { + CallbackUrl url = url("https", host); + assertNotNull(url, "host " + host); + assertTrue(url.valid, "host " + host + " was refused"); + for (char c : new char[] { + '\'', '"', '\\', '<', '>', '\n', '\r', '\u2028', '\u2029'}) { + assertTrue(url.url.indexOf(c) < 0, + "host " + host + " gave " + url.url); + } + } + + @Test + public void CallbackUrl_EncodesTheEndpoint() { + assertValid( + "https://example.com/json%27x", + CallbackUrl.of("https", "example.com", "/json'x"), + "the endpoint /json'x"); + } + + /** + * A host must be ASCII, so an internationalized name is given as + * A-labels, and anything else outside ASCII is refused. + */ + @ParameterizedTest(name = "[{index}]") + @ValueSource(strings = { + "b\u00fccher.de", "a\u00a0b.com", "a\u200bb.com", "a\u2028b.com"}) + public void CallbackUrl_RefusesHostOutsideAscii(String host) { + assertInvalid(url("https", host), "host " + host); + } + + /** + * Not one of the shared cases, because only some languages' strings can + * hold a lone surrogate. It has no UTF-8 encoding, so it is written as + * U+FFFD, as a browser writes it. + */ + @Test + public void CallbackUrl_WritesALoneSurrogateAsTheReplacement() { + assertValid( + "https://example.com/%EF%BF%BD/51dpipeline/json", + url("https", "example.com/" + (char) 0xD800), + "a lone surrogate"); + } +} From 68101da77992cf79df74660a3594ef1132a02490 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 23:29:36 +0000 Subject: [PATCH 14/14] Close every FlowData, Pipeline and metadata collection, and fail the build when one is left open (#44) --- ci/pmd-resource-ruleset.xml | 99 +++ .../onpremise/data/ComponentMetaDataHash.java | 3 +- .../onpremise/data/ProfileMetaDataHash.java | 16 +- .../onpremise/data/PropertyMetaDataHash.java | 3 +- .../hash/engine/onpremise/WrapperHash.java | 7 +- .../onpremise/data/DataValidatorHash.java | 13 +- .../onpremise/data/MetaDataHashTests.java | 8 +- .../onpremise/data/MetaDataHasherHash.java | 75 +- .../onpremise/flowelements/EngineTests.java | 7 +- .../shared/testhelpers/Wrapper.java | 7 +- .../testhelpers/data/MetaDataHasher.java | 8 +- .../shared/testhelpers/data/ValueTests.java | 7 +- .../CloudRequestOriginTests.java | 5 +- ...DetectionOnPremisePipelineBuilderTest.java | 211 +++--- device-detection/pom.xml | 35 + .../examples/console/MetadataOnPrem.java | 135 ++-- .../examples/console/ShareUsageExample.java | 5 +- examples/device-detection/pom.xml | 35 + .../console/GettingStartedCloudMixed.java | 3 + .../examples/console/MetadataOnPrem.java | 103 +-- .../examples/console/MetricsOnPrem.java | 10 +- examples/ip-intelligence/pom.xml | 35 + .../examples/web/GettingStartedApi.java | 15 +- .../web/GettingStartedWebCloudMixed.java | 12 +- .../examples/web/GettingStartedWebCloud.java | 16 +- .../examples/web/GettingStartedWebMixed.java | 12 +- .../examples/web/GettingStartedWebOnPrem.java | 16 +- .../onpremise/data/ComponentMetaDataIPI.java | 3 +- .../onpremise/data/ProfileMetaDataIPI.java | 16 +- .../onpremise/data/PropertyMetaDataIPI.java | 3 +- .../engine/onpremise/WrapperHash.java | 7 +- .../onpremise/data/MetaDataHashTests.java | 8 +- .../onpremise/data/MetaDataHasherHash.java | 75 +- .../onpremise/flowelements/EngineTests.java | 7 +- .../shared/testhelpers/Wrapper.java | 7 +- .../testhelpers/data/MetaDataHasher.java | 8 +- .../CloudRequestOriginTests.java | 5 +- ...elligenceOnPremisePipelineBuilderTest.java | 148 ++-- ip-intelligence/pom.xml | 2 + .../examples/CombiningServices.java | 18 +- .../examples/cloud/GettingStarted.java | 13 +- location/pom.xml | 35 + .../CloudAspectEngineBaseTests.java | 33 +- .../flowelements/CloudRequestEngineTests.java | 71 +- .../core/flowelements/PipelineBuilder.java | 63 +- .../core/flowelements/FlowDataTests.java | 9 +- .../flowelements/ParallelElementsTests.java | 55 +- .../flowelements/PipelineBuilderTests.java | 149 ++-- .../core/flowelements/PipelineTests.java | 45 +- .../integration/PipelineIntegrationTests.java | 34 +- .../flowelements/RecordingElementBuilder.java | 47 ++ .../developerexamples/cloudengine/Main.java | 49 +- .../developerexamples/flowelement/Main.java | 31 +- .../onpremiseengine/Main.java | 32 +- .../developerexamples/usagesharing/Main.java | 40 +- .../fiftyone/data/ComponentMetaData.java | 3 +- .../data/FiftyOneAspectPropertyMetaData.java | 3 +- ...FiftyOneAspectPropertyMetaDataDefault.java | 6 +- .../fiftyone/data/ProfileMetaData.java | 6 +- .../flowelements/SetHeadersElementTests.java | 96 +-- .../flowelements/ShareUsageElementTests.java | 685 +++++++++--------- .../performance/ShareUsageOverheadTests.java | 103 +-- .../javascriptbuilder/CookieTests.java | 15 +- .../javascriptbuilder/JavaScriptUrlTests.java | 23 +- .../jsonbuilder/JsonBuilderTests.java | 8 +- .../mvc/components/FiftyOneInterceptor.java | 25 +- .../web/mvc/FiftyOneInterceptorTests.java | 15 + .../fiftyone/pipeline/web/PipelineFilter.java | 41 +- .../services/PipelineResultServiceCore.java | 23 +- .../pipeline/web/PipelineFilterTests.java | 41 +- pipeline/pom.xml | 35 + 71 files changed, 1864 insertions(+), 1178 deletions(-) create mode 100644 ci/pmd-resource-ruleset.xml create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java diff --git a/ci/pmd-resource-ruleset.xml b/ci/pmd-resource-ruleset.xml new file mode 100644 index 000000000..5f99fad8e --- /dev/null +++ b/ci/pmd-resource-ruleset.xml @@ -0,0 +1,99 @@ + + + + + Fails the build when a FlowData, a Pipeline or a collection of + metadata is not closed on every path. The native results in a FlowData + and the native metadata collections each hold the engine's data set, + so while one is open, closing the Pipeline cannot free the data set. + Java never frees the native results by itself. + + Where something is closed by its owner elsewhere, such as the FlowData + the web integration closes at the end of a request, the code says so + with @SuppressWarnings and a comment naming the owner. + + + + + Every FlowData created is closed by a try-with-resources statement, + unless it is returned to the caller or kept in a field. + + 3 + + + + + + + + + + + + Every CloseableIterable a method returns, such as the profiles or + the components of an engine, is closed by a try-with-resources + statement, unless it is returned to the caller. + + 3 + + + + + + + + + + + + A local variable holding a Pipeline that a builder has just built + is a try-with-resources resource, is closed in a finally block of + the same method, or is returned to the caller. + + 3 + + + + + + + + + + diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java index 30d18118b..63cfdbf6f 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ComponentMetaDataHash.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.hash.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.interop.PropertyIterable; import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ComponentMetaDataSwig; @@ -77,7 +78,7 @@ public ProfileMetaData getDefaultProfile() { } @Override - public Iterable getProperties() { + public CloseableIterable getProperties() { return new PropertyIterable( engine, properties, diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java index 7603773ae..afadcd5b2 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/ProfileMetaDataHash.java @@ -114,11 +114,19 @@ public ComponentMetaData getComponent() { @Override public String getName() { List values = new ArrayList<>(); - for (ValueMetaData value : getValues()) { - if (value.getProperty().getDisplayOrder() > 0 && - value.getName().equalsIgnoreCase("N/A") == false) { - values.add(value); + // The native collection holds the data set until it is closed. The + // values taken from it are separate objects, so they outlive it. + try (CloseableIterable profileValues = getValues()) { + for (ValueMetaData value : profileValues) { + if (value.getProperty().getDisplayOrder() > 0 && + value.getName().equalsIgnoreCase("N/A") == false) { + values.add(value); + } } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); } ValueMetaData[] valuesArray = new ValueMetaData[0]; valuesArray = values.toArray(valuesArray); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java index 3dbf2b5d5..6a6b5a92a 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/data/PropertyMetaDataHash.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.hash.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.interop.Swig; import fiftyone.devicedetection.hash.engine.onpremise.interop.ValueIterable; @@ -136,7 +137,7 @@ public ComponentMetaData getComponent() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return new ValueIterable( engine, engine.getMetaData().getValuesForProperty(source)); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java index 2d034f6ff..21bdb6fb7 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/WrapperHash.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -70,15 +71,15 @@ public Iterable getProperties() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return engine.getValues(); } - public Iterable getProfiles() { + public CloseableIterable getProfiles() { return engine.getProfiles(); } - public Iterable getComponents() { + public CloseableIterable getComponents() { return engine.getComponents(); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java index 336a52141..ddd55d599 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/DataValidatorHash.java @@ -29,6 +29,7 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.exceptions.NoValueException; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; @@ -91,8 +92,16 @@ public void validateData(FlowData data, boolean validEvidence) throws NoValueExc // rather than a fixed value, and that valid evidence produces at least // one result. int componentCount = 0; - for (ComponentMetaData component : engine.getComponents()) { - componentCount++; + // The collection holds the engine's data set until it is closed. + try (CloseableIterable components = + engine.getComponents()) { + for (ComponentMetaData component : components) { + componentCount++; + } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); } int matchedUserAgents = elementData.getUserAgents().getValue().size(); assertTrue( diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java index 4d0a9e220..8e4fa034a 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHashTests.java @@ -65,7 +65,7 @@ public void MetaData_Hash_Reload() throws ExecutionException, InterruptedExcepti * assumption rather than failing. */ @Test - public void MetaData_Performance_Properties() { + public void MetaData_Performance_Properties() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProperties(0, getWrapper()); @@ -83,7 +83,7 @@ public void MetaData_Performance_Properties() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Components() { + public void MetaData_Performance_Components() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashComponents(0, getWrapper()); @@ -101,7 +101,7 @@ public void MetaData_Performance_Components() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Values() { + public void MetaData_Performance_Values() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashValues(0, getWrapper()); @@ -119,7 +119,7 @@ public void MetaData_Performance_Values() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Profiles() { + public void MetaData_Performance_Profiles() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProfiles(0, getWrapper()); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java index f793a46fc..aa5a619ba 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java @@ -25,6 +25,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.Constants; import fiftyone.devicedetection.shared.testhelpers.Wrapper; import fiftyone.devicedetection.shared.testhelpers.data.MetaDataHasher; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -34,7 +35,7 @@ public class MetaDataHasherHash implements MetaDataHasher { @Override - public int hashProperties(int hash, Wrapper wrapper) { + public int hashProperties(int hash, Wrapper wrapper) throws Exception { int i = 0; Iterable properties = wrapper.getProperties(); for (FiftyOneAspectPropertyMetaData property : properties) { @@ -42,11 +43,13 @@ public int hashProperties(int hash, Wrapper wrapper) { hash ^= property.hashCode(); int j = 0; - for (ValueMetaData value : property.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable propertyValues = property.getValues()) { + for (ValueMetaData value : propertyValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; } - j++; } // Match metric property does not have component so don't include if (!property.getCategory().equals(Constants.MatchMetrics.CATEGORY)) { @@ -62,50 +65,60 @@ public int hashProperties(int hash, Wrapper wrapper) { } @Override - public int hashValues(int hash, Wrapper wrapper) { + public int hashValues(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ValueMetaData value : wrapper.getValues()) { - if (i % 100 == 0) { - hash ^= value.hashCode(); - hash ^= value.getProperty().hashCode(); + try (CloseableIterable values = wrapper.getValues()) { + for (ValueMetaData value : values) { + if (i % 100 == 0) { + hash ^= value.hashCode(); + hash ^= value.getProperty().hashCode(); + } + i++; } - i++; } return hash; } @Override - public int hashComponents(int hash, Wrapper wrapper) { - for (ComponentMetaData component : wrapper.getComponents()) { - hash ^= component.hashCode(); - int i = 0; - for (FiftyOneAspectPropertyMetaData property : component.getProperties()) { - if (i % 10 == 0) { - hash ^= property.hashCode(); + public int hashComponents(int hash, Wrapper wrapper) throws Exception { + try (CloseableIterable components = wrapper.getComponents()) { + for (ComponentMetaData component : components) { + hash ^= component.hashCode(); + int i = 0; + try (CloseableIterable componentProperties = component.getProperties()) { + for (FiftyOneAspectPropertyMetaData property : componentProperties) { + if (i % 10 == 0) { + hash ^= property.hashCode(); + } + i++; + } } - i++; + hash ^= component.getDefaultProfile().hashCode(); } - hash ^= component.getDefaultProfile().hashCode(); } return hash; } @Override - public int hashProfiles(int hash, Wrapper wrapper) { + public int hashProfiles(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ProfileMetaData profile : wrapper.getProfiles()) { - if (i % 100 == 0) { - hash ^= profile.hashCode(); - hash ^= profile.getComponent().hashCode(); - int j = 0; - for (ValueMetaData value : profile.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable profiles = wrapper.getProfiles()) { + for (ProfileMetaData profile : profiles) { + if (i % 100 == 0) { + hash ^= profile.hashCode(); + hash ^= profile.getComponent().hashCode(); + int j = 0; + try (CloseableIterable profileValues = profile.getValues()) { + for (ValueMetaData value : profileValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; + } } - j++; } + i++; } - i++; } return hash; } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index 0b58bc098..14b30de62 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -62,9 +62,10 @@ private void testDataSourceTier(String fileName) throws Exception { assumeTrue(e.getMessage(), false); } - DeviceDetectionHashEngine engine = createEngine(dataFile); - String tier = engine.getDataSourceTier(); - engine.close(); + String tier; + try (DeviceDetectionHashEngine engine = createEngine(dataFile)) { + tier = engine.getDataSourceTier(); + } if (fileName.equals(LITE_HASH_DATA_FILE_PATH)) { assertEquals("Lite", tier); diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java index 45fc54abe..82a7b6454 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Wrapper.java @@ -24,6 +24,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -38,11 +39,11 @@ public interface Wrapper extends Closeable { Iterable getProperties(); - Iterable getValues(); + CloseableIterable getValues(); - Iterable getProfiles(); + CloseableIterable getProfiles(); - Iterable getComponents(); + CloseableIterable getComponents(); FiftyOneAspectEngine getEngine(); } \ No newline at end of file diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java index 49240aecd..17fd7796e 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/MetaDataHasher.java @@ -25,11 +25,11 @@ import fiftyone.devicedetection.shared.testhelpers.Wrapper; public interface MetaDataHasher { - int hashProperties(int hash, Wrapper wrapper); + int hashProperties(int hash, Wrapper wrapper) throws Exception; - int hashValues(int hash, Wrapper wrapper); + int hashValues(int hash, Wrapper wrapper) throws Exception; - int hashComponents(int hash, Wrapper wrapper); + int hashComponents(int hash, Wrapper wrapper) throws Exception; - int hashProfiles(int hash, Wrapper wrapper); + int hashProfiles(int hash, Wrapper wrapper) throws Exception; } diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java index 148e3a6e0..6826fb4d2 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/data/ValueTests.java @@ -30,6 +30,7 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; @@ -67,8 +68,10 @@ public static void matchedUserAgents(Wrapper wrapper) throws Exception { // depends on the data file, so assert the bound here and validate // every matched substring below. int componentCount = 0; - for (ComponentMetaData component : wrapper.getComponents()) { - componentCount++; + try (CloseableIterable components = wrapper.getComponents()) { + for (ComponentMetaData component : components) { + componentCount++; + } } int matchedCount = device.getUserAgents().getValue().size(); assertTrue( diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java index 72e4e2335..bb311b1b3 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudRequestOriginTests.java @@ -86,12 +86,11 @@ public void ResourceKeyWithOrigin() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); try { - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory, http) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory, http) .useCloud(resourceKey) .setCloudRequestOrigin(this.origin) .build(); - - try(FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { data.addEvidence("query.user-agent", "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL " + "Build/OPD1.170816.004) AppleWebKit/537.36 " + diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java index c9430a527..21b9e5bd4 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java @@ -87,12 +87,13 @@ private ConfigHashSwig getConfigHashSwig(Pipeline pipeline) throws NoSuchFieldEx @Test public void setFilename_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setFilename(HASH_DATA_FILE_NAME, true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(HASH_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); - assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(HASH_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); + assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + } } @Test @@ -118,210 +119,232 @@ public void setFilename_OtherExtension_ExceptionThrown() { @Test public void setEngineData_PropertyPropagatedToDataFileConfiguration() throws Exception { byte[] data = Files.readAllBytes(datafile.toPath()); - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(new byte[]{}, Enums.DeviceDetectionAlgorithm.Hash) .setEngineData(data) .setShareUsage(false) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(data, dataFileConfiguration.getData()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(data, dataFileConfiguration.getData()); + } } @Test public void setEngineData_FilenameIsSet_PropertyNotPropagated() throws Exception { byte[] data = Files.readAllBytes(datafile.toPath()); - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setEngineData(data) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertNull(dataFileConfiguration.getData()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertNull(dataFileConfiguration.getData()); + } } @Test public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(false) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(1, flowElements.size()); - assertFalse(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(1, flowElements.size()); + assertFalse(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setShareUsage_True_SharedFlowElementAdded() throws Exception { - Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(true) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(2, flowElements.size()); - assertTrue(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(2, flowElements.size()); + assertTrue(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setAutoUpdate(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + } } @Test public void setDataFileSystemWatcher_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataFileSystemWatcher(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + } } @Test public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); - Pipeline pipeline = builder + try (Pipeline ignored = builder .setAutoUpdate(true) .setDataUpdateService(dataUpdateServiceDefault) - .build(); - verify(dataUpdateServiceDefault).registerDataFile(any()); + .build()) { + verify(dataUpdateServiceDefault).registerDataFile(any()); + } } @Test public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getUpdateOnStartup()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getUpdateOnStartup()); + } } @Test public void setUpdatePollingInterval_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingInterval(10) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdatePollingIntervalMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingIntervalMillis(10000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdateRandomisationMax_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMax(123) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setUpdateRandomisationMaxMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMaxMillis(123000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setDataUpdateLicenseKey_PropertyPropagatedToDataFileConfiguration() throws Exception { String key = "licence_key"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateLicenseKey(key) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + } } @Test public void setConcurrency_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setConcurrency(4) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertEquals(4, swig.getConcurrency()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertEquals(4, swig.getConcurrency()); + } } @Test public void setDifference_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDifference(123) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertEquals(123, swig.getDifference()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertEquals(123, swig.getDifference()); + } } @Test public void setAllowUnmatched_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setAllowUnmatched(true) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertTrue(swig.getAllowUnmatched()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertTrue(swig.getAllowUnmatched()); + } } @Test public void setDrift_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDrift(123) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertEquals(123, swig.getDrift()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertEquals(123, swig.getDrift()); + } } @Test public void setUsePerformanceGraph_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUsePerformanceGraph(true) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertTrue(swig.getUsePerformanceGraph()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertTrue(swig.getUsePerformanceGraph()); + } } @Test public void setUsePredictiveGraph_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUsePredictiveGraph(true) - .build(); - ConfigHashSwig swig = getConfigHashSwig(pipeline); - assertTrue(swig.getUsePredictiveGraph()); + .build()) { + ConfigHashSwig swig = getConfigHashSwig(pipeline); + assertTrue(swig.getUsePredictiveGraph()); + } } @Test public void setDataUpdateUrl_PropertyPropagatedToDataFileConfiguration() throws Exception { String url = "https://update.me.com"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrl(url) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + } } @Test public void setDataUpdateVerifyMd5_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateVerifyMd5(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getVerifyMd5()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getVerifyMd5()); + } } @Test public void setDataUpdateUrlFormatter_PropertyPropagatedToDataFileConfiguration() throws Exception { DataUpdateUrlFormatter urlFormatter = mock(DataUpdateUrlFormatter.class); - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrlFormatter(urlFormatter) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + } } } \ No newline at end of file diff --git a/device-detection/pom.xml b/device-detection/pom.xml index ce56dd201..b255d1e53 100644 --- a/device-detection/pom.xml +++ b/device-detection/pom.xml @@ -56,6 +56,7 @@ 3.7.1 6.5.3 0.7.0 + 3.26.0 true false @@ -308,6 +309,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 5b6939705..350172405 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -21,11 +21,13 @@ * ********************************************************************* */ package fiftyone.devicedetection.examples.console; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; import fiftyone.pipeline.engines.fiftyone.data.ValueMetaData; @@ -134,71 +136,92 @@ private static void outputEvidenceKeyDetails(DeviceDetectionHashEngine ddEngine, } private static void outputProfileDetails(DeviceDetectionHashEngine ddEngine, - PrintWriter output) { - // Group the profiles by component and then output the number of profiles + PrintWriter output) throws Exception { + // Group the profiles by component and then output the number of profiles // for each component. - Map> groups = - StreamSupport.stream(ddEngine.getProfiles().spliterator(), false) - .collect(Collectors.groupingBy(p -> p.getComponent().getName())); - groups.forEach((k,v)->output.format("%s Profiles: %d\n", k , v.size())); + // + // A try-with-resource block MUST be used for the collections of + // metadata returned by the engine. Each one holds a reference to the + // engine's data set until it is closed, so if it is left open the + // data set cannot be freed when the engine is closed. + Map groups; + try (CloseableIterable profiles = ddEngine.getProfiles()) { + groups = StreamSupport.stream(profiles.spliterator(), false) + .collect(Collectors.groupingBy( + p -> p.getComponent().getName(), + Collectors.counting())); + } + groups.forEach((k,v)->output.format("%s Profiles: %d\n", k , v)); } // Output the component name as well as a list of all the associated properties. // If we're outputting to console then we also add some formatting to make it // more readable. - private static void outputComponents(DeviceDetectionHashEngine ddEngine, PrintWriter output){ - ddEngine.getComponents().forEach(c -> { - output.println("Component - "+ c.getName()); - outputProperties(c, output); - }); + private static void outputComponents(DeviceDetectionHashEngine ddEngine, PrintWriter output) + throws Exception { + // See outputProfileDetails for why this is a try-with-resource block. + try (CloseableIterable components = ddEngine.getComponents()) { + for (ComponentMetaData c : components) { + output.println("Component - "+ c.getName()); + outputProperties(c, output); + } + } } - private static void outputProperties(ComponentMetaData component, PrintWriter output) { - if (component.getProperties().iterator().hasNext() == false) { - output.println(" ... no properties"); - return; + private static void outputProperties(ComponentMetaData component, PrintWriter output) + throws Exception { + // The properties of a component, and the values of a property, are + // collections that are closed in the same way. + try (CloseableIterable properties = + component.getProperties()) { + if (properties.iterator().hasNext() == false) { + output.println(" ... no properties"); + return; + } + for (FiftyOneAspectPropertyMetaData property : properties) { + // Output some details about the property. + // If we're outputting to console then we also add some formatting to make it + // more readable. + output.format(" Property - %s [Category: %s] (%s)\n " + + "Description: %s\n", + property.getName(), + property.getCategory(), + property.getType().getName(), + property.getDescription()); + + // Next, output a list of the possible values this property can have. + // Most properties in the Device Metrics category do not have defined + // values so exclude them. + if (property.getCategory().equals("Device Metrics")==false) { + StringBuilder values = new StringBuilder(" Possible " + + "values: "); + try (CloseableIterable propertyValues = + property.getValues()) { + Spliterator spliterator2 = + propertyValues.spliterator(); + StreamSupport.stream(spliterator2, false) + .limit(20) + .forEach(value -> { + // add value + values.append(truncateToNl(value.getName())); + // add description if exists + String d = value.getDescription(); + if (Objects.nonNull(d) && d.isEmpty() == false) { + values.append("(") + .append(d) + .append(")"); + } + values.append(","); + }); + + if (spliterator2.estimateSize() > 20) { + values.append(" + more ..."); + } + } + output.println(values); + } + } } - component.getProperties() - .forEach(property-> { - // Output some details about the property. - // If we're outputting to console then we also add some formatting to make it - // more readable. - output.format(" Property - %s [Category: %s] (%s)\n " + - "Description: %s\n", - property.getName(), - property.getCategory(), - property.getType().getName(), - property.getDescription()); - - // Next, output a list of the possible values this property can have. - // Most properties in the Device Metrics category do not have defined - // values so exclude them. - if (property.getCategory().equals("Device Metrics")==false) { - StringBuilder values = new StringBuilder(" Possible " + - "values: "); - Spliterator spliterator2 = - property.getValues().spliterator(); - StreamSupport.stream(spliterator2, false) - .limit(20) - .forEach(value -> { - // add value - values.append(truncateToNl(value.getName())); - // add description if exists - String d = value.getDescription(); - if (Objects.nonNull(d) && d.isEmpty() == false) { - values.append("(") - .append(d) - .append(")"); - } - values.append(","); - }); - - if (spliterator2.estimateSize() > 20) { - values.append(" + more ..."); - } - output.println(values); - } - }); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java index 621ef0761..50d44c364 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java @@ -137,9 +137,12 @@ public static void run(InputStream is) throws Exception { // .setIncludedQueryStringParameter("campaign") .build(); - // Build a pipeline with only the ShareUsageElement + // Build a pipeline with only the ShareUsageElement. The pipeline is + // asked to close the element, which sends any data still queued, as + // it is not closed anywhere else. try (Pipeline pipeline = new PipelineBuilder() .addFlowElement(shareUsageElement) + .setAutoCloseElements(true) .build()) { logger.info("Pipeline built successfully with ShareUsageElement"); diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index d7617e6e6..fa55e5534 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -62,6 +62,7 @@ 2.0.20 1.3.16 + 3.26.0 @@ -165,6 +166,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java index e5bf11069..efca03fdc 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java @@ -144,10 +144,13 @@ public static void run(String resourceKey, IPIntelligenceCloudEngine ipIntelligenceEngine = new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + // The engines are not closed anywhere else, so the pipeline is asked + // to close them when it is closed. try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(cloudRequestEngine) .addFlowElement(deviceDetectionEngine) .addFlowElement(ipIntelligenceEngine) + .setAutoCloseElements(true) .build()) { // carry out some sample detections diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 7a96b562f..6e127722d 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -21,6 +21,7 @@ * ********************************************************************* */ package fiftyone.ipintelligence.examples.console; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; @@ -175,59 +176,67 @@ private static void outputComponents(IPIntelligenceOnPremiseEngine engine, Print throws Exception { // See outputProfileDetails for why this is a try-with-resource block. try (CloseableIterable components = engine.getComponents()) { - components.forEach(c -> { + for (ComponentMetaData c : components) { output.println("Component - "+ c.getName()); outputProperties(c, output); - }); + } } } - private static void outputProperties(ComponentMetaData component, PrintWriter output) { - if (component.getProperties().iterator().hasNext() == false) { - output.println(" ... no properties"); - return; + private static void outputProperties(ComponentMetaData component, PrintWriter output) + throws Exception { + // The properties of a component, and the values of a property, are + // collections that are closed in the same way. + try (CloseableIterable properties = + component.getProperties()) { + if (properties.iterator().hasNext() == false) { + output.println(" ... no properties"); + return; + } + for (FiftyOneAspectPropertyMetaData property : properties) { + // Output some details about the property. + // If we're outputting to console then we also add some formatting to make it + // more readable. + output.format(" Property - %s [Category: %s] (%s)\n " + + "Description: %s\n", + property.getName(), + property.getCategory(), + property.getType().getName(), + property.getDescription()); + + // Next, output a list of the possible values this property can have. + // Most properties in the Metrics category do not have defined + // values so exclude them. + if (property.getCategory().equals("Metrics")==false) { + StringBuilder values = new StringBuilder(" Possible " + + "values: "); + try (CloseableIterable propertyValues = + property.getValues()) { + Spliterator spliterator2 = + propertyValues.spliterator(); + StreamSupport.stream(spliterator2, false) + .limit(20) + .forEach(value -> { + // add value + values.append(truncateToNl(value.getName())); + // add description if exists + String d = value.getDescription(); + if (Objects.nonNull(d) && d.isEmpty() == false) { + values.append("(") + .append(d) + .append(")"); + } + values.append(","); + }); + + if (spliterator2.estimateSize() > 20) { + values.append(" + more ..."); + } + } + output.println(values); + } + } } - component.getProperties() - .forEach(property-> { - // Output some details about the property. - // If we're outputting to console then we also add some formatting to make it - // more readable. - output.format(" Property - %s [Category: %s] (%s)\n " + - "Description: %s\n", - property.getName(), - property.getCategory(), - property.getType().getName(), - property.getDescription()); - - // Next, output a list of the possible values this property can have. - // Most properties in the Metrics category do not have defined - // values so exclude them. - if (property.getCategory().equals("Metrics")==false) { - StringBuilder values = new StringBuilder(" Possible " + - "values: "); - Spliterator spliterator2 = - property.getValues().spliterator(); - StreamSupport.stream(spliterator2, false) - .limit(20) - .forEach(value -> { - // add value - values.append(truncateToNl(value.getName())); - // add description if exists - String d = value.getDescription(); - if (Objects.nonNull(d) && d.isEmpty() == false) { - values.append("(") - .append(d) - .append(")"); - } - values.append(","); - }); - - if (spliterator2.estimateSize() > 20) { - values.append(" + more ..."); - } - output.println(values); - } - }); } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 6c66b486f..347c35b60 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -507,11 +507,13 @@ private static String getValue(ProfileMetaData profile, String name) String result = null; // Every value is closed, not only the first, as each one wraps a // native object of its own. - for (ValueMetaData value : profile.getValues(name)) { - if (result == null) { - result = value.getName(); + try (CloseableIterable values = profile.getValues(name)) { + for (ValueMetaData value : values) { + if (result == null) { + result = value.getName(); + } + value.close(); } - value.close(); } return result; } diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 945f34223..2e3624a91 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -63,6 +63,7 @@ 2.0.20 1.3.16 + 3.26.0 @@ -165,6 +166,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 6e3fa2e8a..9887afc57 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -127,16 +127,29 @@ public void init() throws ServletException { .setAutoUpdate(false) .build(dataFileLocation, false); // The SequenceElement provides the sequence evidence required by - // the JsonBuilderElement. + // the JsonBuilderElement. The pipeline closes all three elements, + // the engine and its data file included, when destroy closes it. pipeline = new PipelineBuilder(LoggerFactory.getILoggerFactory()) .addFlowElement(new SequenceElementBuilder( LoggerFactory.getILoggerFactory()).build()) .addFlowElement(engine) .addFlowElement(new JsonBuilderElementBuilder( LoggerFactory.getILoggerFactory()).build()) + .setAutoCloseElements(true) .build(); DataFileHelper.logDataFileInfo(engine); } catch (Exception e) { + // A servlet whose init fails is never destroyed, so whatever was + // built is closed here. + try { + if (pipeline != null) { + pipeline.close(); + } else if (engine != null) { + engine.close(); + } + } catch (Exception closeException) { + e.addSuppressed(closeException); + } throw new ServletException("Failed to create pipeline from data file '" + dataFileLocation + "'", e); } diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java index 4090fc110..977e3b3fa 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java @@ -118,7 +118,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // Get the IP address parameter from the request for custom lookup String inputIpAddress = request.getParameter("client-ip"); - // The detection has already been carried out by the PipelineFilter + // The detection has already been carried out by the PipelineFilter, + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -149,14 +151,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, deviceData, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java index dbcb50e7f..2e0fbb58f 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java @@ -47,8 +47,9 @@ * The `PipelineFilter` is used to intercept requests and perform IP Intelligence. The results * will be stored in the HttpServletRequest object. * - * The results of the lookup can be accessed by using a FlowDataProvider which - * is responsible for managing the lifecycle of the flowData - do NOT dispose + * The results of the lookup can be accessed by using a FlowDataProvider. The + * PipelineFilter closes the FlowData at the end of the request, so it is not + * closed by the example. * ```{java} * FlowData flowData = flowDataProvider.getFlowData(request); * IPIntelligenceData ipData = flowData.get(IPIntelligenceData.class); @@ -137,7 +138,8 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String inputIpAddress = request.getParameter("client-ip"); // The lookup has already been carried out by the PipelineFilter - // which is responsible for the lifecycle of the flowData - do NOT dispose + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -165,14 +167,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 5b7331767..b6addc8d7 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -140,7 +140,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // Get the IP address parameter from the request for custom lookup String inputIpAddress = request.getParameter("client-ip"); - // The detection has already been carried out by the PipelineFilter + // The detection has already been carried out by the PipelineFilter, + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -171,14 +173,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, deviceData, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index d73bafdf8..ec48e2df9 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -45,8 +45,9 @@ * The `PipelineFilter` is used to intercept requests and perform IP Intelligence. The results * will be stored in the HttpServletRequest object. * - * The results of the lookup can be accessed by using a FlowDataProvider which - * is responsible for managing the lifecycle of the flowData - do NOT dispose + * The results of the lookup can be accessed by using a FlowDataProvider. The + * PipelineFilter closes the FlowData at the end of the request, so it is not + * closed by the example. * ```{java} * FlowData flowData = flowDataProvider.getFlowData(request); * IPIntelligenceData ipData = flowData.get(IPIntelligenceData.class); @@ -155,7 +156,8 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String inputIpAddress = request.getParameter("client-ip"); // The lookup has already been carried out by the PipelineFilter - // which is responsible for the lifecycle of the flowData - do NOT dispose + // which closes the FlowData at the end of the request, so it is not + // closed here. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -183,14 +185,6 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String htmlTemplate = loadTemplate(resourceBase + "/WEB-INF/html/index.html"); String processedHtml = substituteTemplateValues(htmlTemplate, ipiData, targetIp, flowData); out.println(processedHtml); - } finally { - if (flowData != null) { - try { - flowData.close(); - } catch (Exception e) { - logger.warn("Error closing flow data", e); - } - } } } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java index 7ee5100d7..528e1d737 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ComponentMetaDataIPI.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.interop.PropertyIterable; import fiftyone.ipintelligence.engine.onpremise.interop.swig.ComponentMetaDataSwig; @@ -77,7 +78,7 @@ public ProfileMetaData getDefaultProfile() { } @Override - public Iterable getProperties() { + public CloseableIterable getProperties() { return new PropertyIterable( engine, properties, diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java index b9ee322df..3f2645723 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/ProfileMetaDataIPI.java @@ -114,11 +114,19 @@ public ComponentMetaData getComponent() { @Override public String getName() { List values = new ArrayList<>(); - for (ValueMetaData value : getValues()) { - if (value.getProperty().getDisplayOrder() > 0 && - value.getName().equalsIgnoreCase("N/A") == false) { - values.add(value); + // The native collection holds the data set until it is closed. The + // values taken from it are separate objects, so they outlive it. + try (CloseableIterable profileValues = getValues()) { + for (ValueMetaData value : profileValues) { + if (value.getProperty().getDisplayOrder() > 0 && + value.getName().equalsIgnoreCase("N/A") == false) { + values.add(value); + } } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); } ValueMetaData[] valuesArray = new ValueMetaData[0]; valuesArray = values.toArray(valuesArray); diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java index 527df5b74..eabb2aecb 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.engine.onpremise.data; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.interop.Swig; import fiftyone.ipintelligence.engine.onpremise.interop.ValueIterable; @@ -136,7 +137,7 @@ public ComponentMetaData getComponent() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return new ValueIterable( engine, engine.getMetaData().getValuesForProperty(source)); diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java index d9e861c0f..a001f6e34 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/WrapperHash.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -69,15 +70,15 @@ public Iterable getProperties() { } @Override - public Iterable getValues() { + public CloseableIterable getValues() { return engine.getValues(); } - public Iterable getProfiles() { + public CloseableIterable getProfiles() { return engine.getProfiles(); } - public Iterable getComponents() { + public CloseableIterable getComponents() { return engine.getComponents(); } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java index d229af08b..f9f9a03eb 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHashTests.java @@ -67,7 +67,7 @@ public void MetaData_Hash_Reload() throws ExecutionException, InterruptedExcepti * assumption rather than failing. */ @Test - public void MetaData_Performance_Properties() { + public void MetaData_Performance_Properties() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProperties(0, getWrapper()); @@ -85,7 +85,7 @@ public void MetaData_Performance_Properties() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Components() { + public void MetaData_Performance_Components() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashComponents(0, getWrapper()); @@ -103,7 +103,7 @@ public void MetaData_Performance_Components() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Values() { + public void MetaData_Performance_Values() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashValues(0, getWrapper()); @@ -121,7 +121,7 @@ public void MetaData_Performance_Values() { * assumption rather than failing. */ @Test - public void MetaData_Performance_Profiles() { + public void MetaData_Performance_Profiles() throws Exception { MetaDataHasherHash hasher = new MetaDataHasherHash(); long start = System.currentTimeMillis(); hasher.hashProfiles(0, getWrapper()); diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java index 4683157a5..2bd50f8b6 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/MetaDataHasherHash.java @@ -24,6 +24,7 @@ import fiftyone.ipintelligence.shared.testhelpers.Wrapper; import fiftyone.ipintelligence.shared.testhelpers.data.MetaDataHasher; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -33,7 +34,7 @@ public class MetaDataHasherHash implements MetaDataHasher { @Override - public int hashProperties(int hash, Wrapper wrapper) { + public int hashProperties(int hash, Wrapper wrapper) throws Exception { int i = 0; Iterable properties = wrapper.getProperties(); for (FiftyOneAspectPropertyMetaData property : properties) { @@ -41,11 +42,13 @@ public int hashProperties(int hash, Wrapper wrapper) { hash ^= property.hashCode(); int j = 0; - for (ValueMetaData value : property.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable propertyValues = property.getValues()) { + for (ValueMetaData value : propertyValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; } - j++; } if (property.getDefaultValue() != null) { hash ^= property.getDefaultValue().hashCode(); @@ -57,50 +60,60 @@ public int hashProperties(int hash, Wrapper wrapper) { } @Override - public int hashValues(int hash, Wrapper wrapper) { + public int hashValues(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ValueMetaData value : wrapper.getValues()) { - if (i % 100 == 0) { - hash ^= value.hashCode(); - hash ^= value.getProperty().hashCode(); + try (CloseableIterable values = wrapper.getValues()) { + for (ValueMetaData value : values) { + if (i % 100 == 0) { + hash ^= value.hashCode(); + hash ^= value.getProperty().hashCode(); + } + i++; } - i++; } return hash; } @Override - public int hashComponents(int hash, Wrapper wrapper) { - for (ComponentMetaData component : wrapper.getComponents()) { - hash ^= component.hashCode(); - int i = 0; - for (FiftyOneAspectPropertyMetaData property : component.getProperties()) { - if (i % 10 == 0) { - hash ^= property.hashCode(); + public int hashComponents(int hash, Wrapper wrapper) throws Exception { + try (CloseableIterable components = wrapper.getComponents()) { + for (ComponentMetaData component : components) { + hash ^= component.hashCode(); + int i = 0; + try (CloseableIterable componentProperties = component.getProperties()) { + for (FiftyOneAspectPropertyMetaData property : componentProperties) { + if (i % 10 == 0) { + hash ^= property.hashCode(); + } + i++; + } } - i++; + hash ^= component.getDefaultProfile().hashCode(); } - hash ^= component.getDefaultProfile().hashCode(); } return hash; } @Override - public int hashProfiles(int hash, Wrapper wrapper) { + public int hashProfiles(int hash, Wrapper wrapper) throws Exception { int i = 0; - for (ProfileMetaData profile : wrapper.getProfiles()) { - if (i % 100 == 0) { - hash ^= profile.hashCode(); - hash ^= profile.getComponent().hashCode(); - int j = 0; - for (ValueMetaData value : profile.getValues()) { - if (j % 10 == 0) { - hash ^= value.hashCode(); + try (CloseableIterable profiles = wrapper.getProfiles()) { + for (ProfileMetaData profile : profiles) { + if (i % 100 == 0) { + hash ^= profile.hashCode(); + hash ^= profile.getComponent().hashCode(); + int j = 0; + try (CloseableIterable profileValues = profile.getValues()) { + for (ValueMetaData value : profileValues) { + if (j % 10 == 0) { + hash ^= value.hashCode(); + } + j++; + } } - j++; } + i++; } - i++; } return hash; } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java index 4f0d9005b..e12f6cc10 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineTests.java @@ -62,9 +62,10 @@ private void testDataSourceTier(String fileName) throws Exception { assumeTrue(e.getMessage(), false); } - IPIntelligenceOnPremiseEngine engine = createEngine(dataFile); - String tier = engine.getDataSourceTier(); - engine.close(); + String tier; + try (IPIntelligenceOnPremiseEngine engine = createEngine(dataFile)) { + tier = engine.getDataSourceTier(); + } if (fileName.equals(LITE_IPI_DATA_FILE_NAME)) { assertEquals("Lite", tier); diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java index 39378f9fa..d74e56ff7 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/Wrapper.java @@ -24,6 +24,7 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; @@ -38,11 +39,11 @@ public interface Wrapper extends Closeable { Iterable getProperties(); - Iterable getValues(); + CloseableIterable getValues(); - Iterable getProfiles(); + CloseableIterable getProfiles(); - Iterable getComponents(); + CloseableIterable getComponents(); FiftyOneAspectEngine getEngine(); } \ No newline at end of file diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java index 43ffbbfa7..da6e9b7bd 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/data/MetaDataHasher.java @@ -25,11 +25,11 @@ import fiftyone.ipintelligence.shared.testhelpers.Wrapper; public interface MetaDataHasher { - int hashProperties(int hash, Wrapper wrapper); + int hashProperties(int hash, Wrapper wrapper) throws Exception; - int hashValues(int hash, Wrapper wrapper); + int hashValues(int hash, Wrapper wrapper) throws Exception; - int hashComponents(int hash, Wrapper wrapper); + int hashComponents(int hash, Wrapper wrapper) throws Exception; - int hashProfiles(int hash, Wrapper wrapper); + int hashProfiles(int hash, Wrapper wrapper) throws Exception; } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java index c4d13d293..54b262730 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudRequestOriginTests.java @@ -88,12 +88,11 @@ public void ResourceKeyWithOrigin() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); try { - Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory, http) + try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory, http) .useCloud(resourceKey) .setCloudRequestOrigin(this.origin) .build(); - - try(FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { data.addEvidence("query.client-ip", "8.8.8.8") .process(); } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java index 98b386e97..d220eb776 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java @@ -88,12 +88,13 @@ private ConfigIpiSwig getConfigIpiSwig(Pipeline pipeline) throws NoSuchFieldExce @Test public void setFilename_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setFilename(IPI_DATA_FILE_NAME, true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(IPI_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); - assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(IPI_DATA_FILE_NAME, dataFileConfiguration.getDataFilePath()); + assertTrue(dataFileConfiguration.getCreateTempDataCopy()); + } } @Test @@ -118,144 +119,159 @@ public void setFilename_OtherExtension_ExceptionThrown() { @Test public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { - Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) + try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(false) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(1, flowElements.size()); - assertFalse(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(1, flowElements.size()); + assertFalse(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setShareUsage_True_SharedFlowElementAdded() throws Exception { - Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) + try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) .setShareUsage(true) - .build(); - List flowElements = pipeline.getFlowElements(); - assertEquals(2, flowElements.size()); - assertTrue(flowElements.get(0) instanceof ShareUsageElement); + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(2, flowElements.size()); + assertTrue(flowElements.get(0) instanceof ShareUsageElement); + } } @Test public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setAutoUpdate(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); + } } @Test public void setDataFileSystemWatcher_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataFileSystemWatcher(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getFileSystemWatcherEnabled()); + } } @Test public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); - builder + try (Pipeline ignored = builder .setAutoUpdate(true) .setDataUpdateService(dataUpdateServiceDefault) - .build(); - verify(dataUpdateServiceDefault).registerDataFile(any()); + .build()) { + verify(dataUpdateServiceDefault).registerDataFile(any()); + } } @Test public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getUpdateOnStartup()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getUpdateOnStartup()); + } } @Test public void setUpdatePollingInterval_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingInterval(10) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdatePollingIntervalMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdatePollingIntervalMillis(10000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(10, dataFileConfiguration.getPollingIntervalSeconds()); + } } @Test public void setUpdateRandomisationMax_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMax(123) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setUpdateRandomisationMaxMillis_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setUpdateRandomisationMaxMillis(123000) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(123, dataFileConfiguration.getMaxRandomisationSeconds()); + } } @Test public void setDataUpdateLicenseKey_PropertyPropagatedToDataFileConfiguration() throws Exception { String key = "licence_key"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateLicenseKey(key) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getDataUpdateLicenseKeys().contains(key)); + } } @Test public void setConcurrency_PropertyPropagatedSwigConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setConcurrency(4) - .build(); - ConfigIpiSwig swig = getConfigIpiSwig(pipeline); - assertEquals(4, swig.getConcurrency()); + .build()) { + ConfigIpiSwig swig = getConfigIpiSwig(pipeline); + assertEquals(4, swig.getConcurrency()); + } } @Test public void setDataUpdateUrl_PropertyPropagatedToDataFileConfiguration() throws Exception { String url = "https://update.me.com"; - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrl(url) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(url, dataFileConfiguration.getDataUpdateUrl()); + } } @Test public void setDataUpdateVerifyMd5_PropertyPropagatedToDataFileConfiguration() throws Exception { - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateVerifyMd5(true) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertTrue(dataFileConfiguration.getVerifyMd5()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertTrue(dataFileConfiguration.getVerifyMd5()); + } } @Test public void setDataUpdateUrlFormatter_PropertyPropagatedToDataFileConfiguration() throws Exception { DataUpdateUrlFormatter urlFormatter = mock(DataUpdateUrlFormatter.class); - Pipeline pipeline = builder + try (Pipeline pipeline = builder .setDataUpdateUrlFormatter(urlFormatter) - .build(); - DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); - assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + .build()) { + DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); + assertEquals(urlFormatter, dataFileConfiguration.getUrlFormatter()); + } } } \ No newline at end of file diff --git a/ip-intelligence/pom.xml b/ip-intelligence/pom.xml index 16a23b1a3..1355b5932 100644 --- a/ip-intelligence/pom.xml +++ b/ip-intelligence/pom.xml @@ -179,6 +179,8 @@ ${maven.multiModuleProjectDirectory}/ci/pmd-ruleset.xml + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml true diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java index 748499f05..c3088a99f 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java @@ -74,12 +74,14 @@ * .build(Enums.GeoLocationProvider.FiftyOneDegrees); * ``` * - * 2. Build a new Pipeline containing the engines which were just created. + * 2. Build a new Pipeline containing the engines which were just created, + * and ask it to close them when it is closed. * ``` * Pipeline pipeline = new PipelineBuilder(loggerFactory) * .addFlowElement(cloudRequestEngine) * .addFlowElement(deviceDetectionEngine) * .addFlowElement(locationEngine) + * .setAutoCloseElements(true) * .build(); * ``` * @@ -92,9 +94,10 @@ * .build(); * ``` * 3. Create a new FlowData instance ready to be populated with evidence for - * the Pipeline. + * the Pipeline, in a try-with-resources block so that it is closed when it + * is finished with. * ``` - * FlowData data = pipeline.createFlowData(); + * try (FlowData data = pipeline.createFlowData()) { * ``` * * 4. Process a longitude, latitude and User-Agent to retrieve the values @@ -148,14 +151,16 @@ public static void main(String[] args) throws Exception { new GeoLocationCloudEngineBuilder(loggerFactory) .build(Enums.GeoLocationProvider.FiftyOneDegrees); - Pipeline pipeline = + // The engines are not closed anywhere else, so the pipeline is + // asked to close them when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(cloudRequestEngine) .addFlowElement(deviceDetectionEngine) .addFlowElement(locationEngine) + .setAutoCloseElements(true) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData .addEvidence(EVIDENCE_HEADER_USERAGENT_KEY, mobileUserAgent) .addEvidence(EVIDENCE_GEO_LAT_PARAM_KEY, "51.458048") @@ -173,7 +178,6 @@ public static void main(String[] args) throws Exception { System.out.println("Country: " + country.toString()); System.out.println("IsMobile: " + isMobile.toString()); } - pipeline.close(); } } diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java index 553517c6a..8c86dfe01 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java @@ -57,9 +57,10 @@ * ``` * * 2. Create a new FlowData instance ready to be populated with evidence for - * the Pipeline. + * the Pipeline, in a try-with-resources block so that it is closed when it + * is finished with. * ``` - * FlowData data = pipeline.createFlowData(); + * try (FlowData data = pipeline.createFlowData()) { * ``` * * 3. Process a longitude and latitude to retrieve the values associated with @@ -92,14 +93,13 @@ public static void main(String[] args) throws Exception { "property as it is used by this example."); } else { - Pipeline pipeline = new GeoLocationPipelineBuilder(loggerFactory) + try (Pipeline pipeline = new GeoLocationPipelineBuilder(loggerFactory) .useCloud(resourceKey, Enums.GeoLocationProvider.FiftyOneDegrees) .setEndPoint("https://cloud.51degrees.com/api/v4") .useLazyLoading(1000) .setAutoCloseElements(true) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(EVIDENCE_GEO_LAT_PARAM_KEY, "51.458048") .addEvidence(EVIDENCE_GEO_LON_PARAM_KEY, "-0.9822207999999999") .process(); @@ -115,8 +115,7 @@ public static void main(String[] args) throws Exception { System.out.println("Country: " + country.toString()); System.out.println(flowData.get(GeoData.class).getRegion()); - } - pipeline.close(); + } } } diff --git a/location/pom.xml b/location/pom.xml index 1939c6520..930a853f5 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -78,6 +78,7 @@ 3.0.0 3.0.1 0.7.0 + 3.26.0 true false @@ -300,6 +301,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + + diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 54b2c8f96..514a9369e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -181,9 +181,15 @@ public void init() { } @AfterEach - public void cleanUp() { - loggerFactory.assertMaxWarnings(maxWarnings); - loggerFactory.assertMaxErrors(maxErrors); + public void cleanUp() throws Exception { + try { + loggerFactory.assertMaxWarnings(maxWarnings); + loggerFactory.assertMaxErrors(maxErrors); + } finally { + if (pipeline != null) { + pipeline.close(); + } + } } private boolean propertiesContainName( @@ -370,9 +376,10 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .addFlowElement(mockRequestEngine) .addFlowElement(mockTestInstance) .build(); - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("query.user-agent", "iPhone"); - flowData.process(); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + } } catch (Exception e) { fail("An exception has occurred here."); @@ -422,9 +429,10 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .addFlowElement(mockRequestEngine) .addFlowElement(mockTestInstance) .build(); - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("query.user-agent", "iPhone"); - flowData.process(); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + } } catch (Exception e) { fail("An exception has occurred here."); @@ -480,9 +488,10 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .addFlowElement(mockTestInstance) .addFlowElement(mockRequestEngine) .build(); - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("query.user-agent", "iPhone"); - flowData.process(); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + } fail("Expected exception was not thrown"); } catch (Exception ex) { diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index c82645790..01b82a180 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -75,29 +75,30 @@ public void Process() throws Exception { .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine).build(); - FlowData data = pipeline.createFlowData(); - data.addEvidence("query.User-Agent", userAgent); - data.process(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", userAgent); + data.process(); - String result = data.getFromElement(engine).getJsonResponse(); - assertEquals("{'device':{'value':'1'}}", result); + String result = data.getFromElement(engine).getJsonResponse(); + assertEquals("{'device':{'value':'1'}}", result); - JSONObject jsonObj = new JSONObject(result); - assertEquals(1, jsonObj.getJSONObject("device").getInt("value")); + JSONObject jsonObj = new JSONObject(result); + assertEquals(1, jsonObj.getJSONObject("device").getInt("value")); - verify(httpClient, times(1)) // we expected a single external POST request - .postData( - argThat(c -> { - return c.getURL().equals(expectedUrl); // to this uri - }), - ArgumentMatchers.anyMap(), - argThat(bytes -> { - String string = new String(bytes); - return string.contains("resource=" + resourceKey) && // content contains resource key - string.contains("User-Agent=" + userAgent); // content contains user agent - })); + verify(httpClient, times(1)) // we expected a single external POST request + .postData( + argThat(c -> { + return c.getURL().equals(expectedUrl); // to this uri + }), + ArgumentMatchers.anyMap(), + argThat(bytes -> { + String string = new String(bytes); + return string.contains("resource=" + resourceKey) && // content contains resource key + string.contains("User-Agent=" + userAgent); // content contains user agent + })); + } } /** @@ -113,19 +114,20 @@ public void Process_SuppressProcessExceptions_Exceptions_Added_To_Errors() throw .setResourceKey(resourceKey) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .setSuppressProcessException(true) .addFlowElement(engine) .build(); - FlowData data = pipeline.createFlowData(); + FlowData data = pipeline.createFlowData()) { - data.addEvidence("query.User-Agent", userAgent); - data.process(); + data.addEvidence("query.User-Agent", userAgent); + data.process(); - assertFalse(data.getErrors().isEmpty()); - ArrayList errors = (ArrayList) data.getErrors(); - Throwable throwable = errors.get(0).getThrowable(); - assertInstanceOf(IOException.class, throwable); + assertFalse(data.getErrors().isEmpty()); + ArrayList errors = (ArrayList) data.getErrors(); + Throwable throwable = errors.get(0).getThrowable(); + assertInstanceOf(IOException.class, throwable); + } } /** @@ -330,11 +332,10 @@ public void evidencePrecidence(String name, boolean shouldWarn, String evidence1 .setResourceKey("resourcekey") .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(evidence1Parts[0], evidence1Parts[1]); flowData.addEvidence(evidence2Parts[0], evidence2Parts[1]); @@ -450,11 +451,10 @@ public void getFormData(String name, Map evidence, String expect .setResourceKey("resourcekey") .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .build(); - - try (FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { for (Map.Entry entry : evidence.entrySet()) { data.addEvidence(entry.getKey(), entry.getValue()); } @@ -479,11 +479,10 @@ public void getFormData_KeepsDotsInEvidenceName() throws Exception { .setResourceKey("resourcekey") .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .build(); - - try (FlowData data = pipeline.createFlowData()) { + FlowData data = pipeline.createFlowData()) { data.addEvidence("query.id.usage", "non-marketing"); data.addEvidence("header.id.email", "someone@example.com"); diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java index 20c846576..fbebda96e 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java @@ -122,9 +122,12 @@ public Pipeline buildFromConfiguration(PipelineOptions options) "The following builder parameters could not be processed: " + stringJoin(builderParams, ",")); } - } catch (PipelineConfigurationException ex) { - logger.debug("Problem with pipeline configuration, " + - "failed to create pipeline.", ex); + } catch (Exception | Error ex) { + if (ex instanceof PipelineConfigurationException) { + logger.debug("Problem with pipeline configuration, " + + "failed to create pipeline.", ex); + } + closeAfterFailure(flowElements, ex); throw ex; } // As the elements are all created within the builder, the user @@ -134,7 +137,32 @@ public Pipeline buildFromConfiguration(PipelineOptions options) // Build and return the pipeline using the list of flow elements // that have been created from the configuration options. - return build(); + try { + return build(); + } catch (Exception | Error ex) { + closeAfterFailure(flowElements, ex); + throw ex; + } + } + + /** + * Closes the elements built so far when building from configuration + * fails, as no pipeline exists to close them. A failure to close one is + * added to the original failure, which is the one reported. + * @param elements the elements built so far, emptied once closed + * @param failure the reason building failed + */ + private static void closeAfterFailure( + List elements, + Throwable failure) { + for (FlowElement element : elements) { + try { + element.close(); + } catch (Exception e) { + failure.addSuppressed(e); + } + } + elements.clear(); } /** @@ -377,17 +405,24 @@ private void addParallelElementsToList( // Iterate through the sub elements, creating them and // adding them to the list. int subCounter = 0; - for (ElementOptions subElement : elementOptions.subElements) { - if (subElement.subElements != null && - subElement.subElements.size() > 0) { - throw new PipelineConfigurationException( - "ElementOptions " + elementIndex + " contains nested " + - "sub elements. This is not supported."); - } else { - addElementToList(parallelElements, subElement, - "element " + subCounter + " in element " + elementIndex); + try { + for (ElementOptions subElement : elementOptions.subElements) { + if (subElement.subElements != null && + subElement.subElements.size() > 0) { + throw new PipelineConfigurationException( + "ElementOptions " + elementIndex + " contains nested " + + "sub elements. This is not supported."); + } else { + addElementToList(parallelElements, subElement, + "element " + subCounter + " in element " + elementIndex); + } + subCounter++; } - subCounter++; + } catch (RuntimeException | Error ex) { + // These elements are not in the builder's list yet, so they are + // closed here. + closeAfterFailure(parallelElements, ex); + throw ex; } // Now we've created all the elements, create the // ParallelElements instance and add it to the pipeline's diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java index 3e04cf231..613033fa2 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java @@ -318,14 +318,15 @@ public void FlowData_GetNotPresent_Element() { } @Test(expected = IllegalStateException.class) - public void FlowData_PipelineDisposed() { + public void FlowData_PipelineDisposed() throws Exception { when(pipeline.isClosed()).thenReturn(true); - FlowData flowData = new FlowDataDefault( + try (FlowData flowData = new FlowDataDefault( mock(Logger.class), pipeline, - new EvidenceDefault(mock(Logger.class))); + new EvidenceDefault(mock(Logger.class)))) { - flowData.process(); + flowData.process(); + } } @Test diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java index 8ddaa0435..617089b24 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java @@ -126,35 +126,36 @@ public TestElementData create(FlowData flowData) { Arrays.asList(element1, element2, element3)); - FlowData data = new FlowDataDefault( + try (FlowData data = new FlowDataDefault( mock(Logger.class), pipeline, - new EvidenceDefault(mock(Logger.class))); - data.process(); - - // Act - parallelElements.process(data); - - List startTimes = new ArrayList<>(); - startTimes.add((Date) data.get("element1").get("start")); - startTimes.add((Date) data.get("element2").get("start")); - startTimes.add((Date) data.get("element3").get("start")); - List endTimes = new ArrayList<>(); - endTimes.add((Date) data.get("element1").get("end")); - endTimes.add((Date) data.get("element2").get("end")); - endTimes.add((Date) data.get("element3").get("end")); - - // Assert - assertTrue("Expected no errors", - data.getErrors() == null || data.getErrors().size() == 0); - for (Date start : startTimes) { - for (Date end : endTimes) { - assertTrue( - "The elements were not processed sequentially. Times were: " + - startTimes.get(0) + " => " + endTimes.get(0) + "\n" + - startTimes.get(1) + " => " + endTimes.get(1) + "\n" + - startTimes.get(2) + " => " + endTimes.get(2), - start.before(end)); + new EvidenceDefault(mock(Logger.class)))) { + data.process(); + + // Act + parallelElements.process(data); + + List startTimes = new ArrayList<>(); + startTimes.add((Date) data.get("element1").get("start")); + startTimes.add((Date) data.get("element2").get("start")); + startTimes.add((Date) data.get("element3").get("start")); + List endTimes = new ArrayList<>(); + endTimes.add((Date) data.get("element1").get("end")); + endTimes.add((Date) data.get("element2").get("end")); + endTimes.add((Date) data.get("element3").get("end")); + + // Assert + assertTrue("Expected no errors", + data.getErrors() == null || data.getErrors().size() == 0); + for (Date start : startTimes) { + for (Date end : endTimes) { + assertTrue( + "The elements were not processed sequentially. Times were: " + + startTimes.get(0) + " => " + endTimes.get(0) + "\n" + + startTimes.get(1) + " => " + endTimes.get(1) + "\n" + + startTimes.get(2) + " => " + endTimes.get(2), + start.before(end)); + } } } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java index c28bf4438..f8e1c968b 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderTests.java @@ -42,6 +42,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -233,6 +234,35 @@ public void PipelineBuilder_BuildFromConfiguration_MandatoryParameterWrongType() builder.buildFromConfiguration(opts); } + /** + * When an element fails to build, the elements built before it are + * closed, and the failure reported is the original one. + */ + @Test + public void PipelineBuilder_BuildFromConfiguration_SecondElementFails_FirstClosed() throws Exception { + RecordingElementBuilder.BUILT.clear(); + PipelineOptions opts = new PipelineOptions(); + ElementOptions first = new ElementOptions(); + first.builderName = "RecordingElementBuilder"; + opts.elements.add(first); + // The mandatory 'multiple' parameter is not set, so this one fails. + ElementOptions second = new ElementOptions(); + second.builderName = "MultiplyByElementBuilder"; + opts.elements.add(second); + + maxErrors = 1; + + try { + builder.buildFromConfiguration(opts); + fail("Expected the second element to fail to build"); + } catch (PipelineConfigurationException e) { + assertEquals(0, e.getSuppressed().length); + } + assertEquals(1, RecordingElementBuilder.BUILT.size()); + assertTrue("The first element should have been closed", + RecordingElementBuilder.BUILT.get(0).isClosed()); + } + @Test(expected = PipelineConfigurationException.class) public void PipelineBuilder_BuildFromConfiguration_NoBuilder() throws Exception { // Create the configuration object. @@ -416,26 +446,27 @@ public void PipelineBuilder_BuildFromConfiguration_ParallelElements() throws Exc options.elements.add(elOpts); // Pass the configuration to the builder to create the pipeline. - Pipeline pipeline = builder.buildFromConfiguration(options); - // Get the elements - ListSplitterElement splitterElement = pipeline.getElement(ListSplitterElement.class); - MultiplyByElement multiplyByElement = pipeline.getElement(MultiplyByElement.class); - - // Create, populate and process flow data. - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence(splitterElement.evidenceKeys.get(0), "1,2,abc") - .addEvidence(multiplyByElement.evidenceKeys.get(0), 25) - .process(); + try (Pipeline pipeline = builder.buildFromConfiguration(options)) { + // Get the elements + ListSplitterElement splitterElement = pipeline.getElement(ListSplitterElement.class); + MultiplyByElement multiplyByElement = pipeline.getElement(MultiplyByElement.class); + + // Create, populate and process flow data. + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence(splitterElement.evidenceKeys.get(0), "1,2,abc") + .addEvidence(multiplyByElement.evidenceKeys.get(0), 25) + .process(); - // Get the results and verify them. - ListSplitterElementData splitterData = flowData.getFromElement(splitterElement); - TestElementData multiplyByData = flowData.getFromElement(multiplyByElement); + // Get the results and verify them. + ListSplitterElementData splitterData = flowData.getFromElement(splitterElement); + TestElementData multiplyByData = flowData.getFromElement(multiplyByElement); - assertEquals("1", splitterData.getResult().get(0)); - assertEquals("2", splitterData.getResult().get(1)); - assertEquals("abc", splitterData.getResult().get(2)); - assertEquals(75, multiplyByData.getResult()); + assertEquals("1", splitterData.getResult().get(0)); + assertEquals("2", splitterData.getResult().get(1)); + assertEquals("abc", splitterData.getResult().get(2)); + assertEquals(75, multiplyByData.getResult()); + } } } @@ -443,58 +474,60 @@ private void VerifyListSplitterElementPipeline( PipelineOptions options, SplitOption splitOn) throws Exception { // Pass the configuration to the builder to create the pipeline. - Pipeline pipeline = builder.buildFromConfiguration(options); + try (Pipeline pipeline = builder.buildFromConfiguration(options)) { - ListSplitterElement element = pipeline.getElement(ListSplitterElement.class); - // Check we've got the expected number of evidence keys. - assertEquals(1, element.evidenceKeys.size()); + ListSplitterElement element = pipeline.getElement(ListSplitterElement.class); + // Check we've got the expected number of evidence keys. + assertEquals(1, element.evidenceKeys.size()); - // Create, populate and process flow data. - try (FlowData flowData = pipeline.createFlowData()) { - flowData.addEvidence(element.evidenceKeys.get(0), "123,456|789,0") - .process(); + // Create, populate and process flow data. + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(element.evidenceKeys.get(0), "123,456|789,0") + .process(); - // Get the result and verify it. - ListSplitterElementData elementData = flowData.getFromElement(element); - switch (splitOn) { - case Comma: - assertEquals("123", elementData.getResult().get(0)); - assertEquals("456|789", elementData.getResult().get(1)); - assertEquals("0", elementData.getResult().get(2)); - break; - case Pipe: - assertEquals("123,456", elementData.getResult().get(0)); - assertEquals("789,0", elementData.getResult().get(1)); - break; - case CommaMaxLengthThree: - assertEquals("123", elementData.getResult().get(0)); - assertEquals("456", elementData.getResult().get(1)); - assertEquals("|78", elementData.getResult().get(2)); - assertEquals("9", elementData.getResult().get(3)); - assertEquals("0", elementData.getResult().get(4)); - break; - default: - break; + // Get the result and verify it. + ListSplitterElementData elementData = flowData.getFromElement(element); + switch (splitOn) { + case Comma: + assertEquals("123", elementData.getResult().get(0)); + assertEquals("456|789", elementData.getResult().get(1)); + assertEquals("0", elementData.getResult().get(2)); + break; + case Pipe: + assertEquals("123,456", elementData.getResult().get(0)); + assertEquals("789,0", elementData.getResult().get(1)); + break; + case CommaMaxLengthThree: + assertEquals("123", elementData.getResult().get(0)); + assertEquals("456", elementData.getResult().get(1)); + assertEquals("|78", elementData.getResult().get(2)); + assertEquals("9", elementData.getResult().get(3)); + assertEquals("0", elementData.getResult().get(4)); + break; + default: + break; + } } } } private void VerifyMultiplyByElementPipeline(PipelineOptions options) throws Exception { // Pass the configuration to the builder to create the pipeline. - Pipeline pipeline = builder.buildFromConfiguration(options); - MultiplyByElement element = pipeline.getElement(MultiplyByElement.class); + try (Pipeline pipeline = builder.buildFromConfiguration(options)) { + MultiplyByElement element = pipeline.getElement(MultiplyByElement.class); - // Check we've got the expected number of evidence keys. - assertEquals(1, element.evidenceKeys.size()); + // Check we've got the expected number of evidence keys. + assertEquals(1, element.evidenceKeys.size()); - // Create, populate and process flow data. - try (FlowData flowData = pipeline.createFlowData()) { - flowData.addEvidence(element.evidenceKeys.get(0), 5) - .process(); + // Create, populate and process flow data. + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(element.evidenceKeys.get(0), 5) + .process(); - // Get the result and verify it. - TestElementData elementData = flowData.getFromElement(element); - assertEquals(40, elementData.getResult()); + // Get the result and verify it. + TestElementData elementData = flowData.getFromElement(element); + assertEquals(40, elementData.getResult()); + } } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java index c688da1ee..e7d0f1d7b 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineTests.java @@ -102,30 +102,31 @@ public TestElementData create(FlowData flowData) { new FlowElement[]{element1, element2}); // Don't create the flow data via the pipeline as we just want // to test Process. - FlowData data = new FlowDataDefault( + try (FlowData data = new FlowDataDefault( mock(Logger.class), pipeline, - new EvidenceDefault(mock(Logger.class))); - - // Act - data.process(); - - // Assert - assertTrue("Expected no errors", - data.getErrors() == null || data.getErrors().size() == 0); - // Check that the resulting data has the expected values - assertTrue("data from element 1 is missing in the result", - data.getDataKeys().contains("element1")); - assertTrue("data from element 2 is missing in the result", data.getDataKeys().contains("element2")); - assertEquals("done", data.get("element1").get("key").toString()); - assertEquals("done", data.get("element2").get("key").toString()); - // Check that element 1 was called before element 2. - verify(element2, times(1)).process(argThat(new ArgumentMatcher() { - @Override - public boolean matches(FlowData flowData) { - return flowData.getDataKeys().contains("element1"); - } - })); + new EvidenceDefault(mock(Logger.class)))) { + + // Act + data.process(); + + // Assert + assertTrue("Expected no errors", + data.getErrors() == null || data.getErrors().size() == 0); + // Check that the resulting data has the expected values + assertTrue("data from element 1 is missing in the result", + data.getDataKeys().contains("element1")); + assertTrue("data from element 2 is missing in the result", data.getDataKeys().contains("element2")); + assertEquals("done", data.get("element1").get("key").toString()); + assertEquals("done", data.get("element2").get("key").toString()); + // Check that element 1 was called before element 2. + verify(element2, times(1)).process(argThat(new ArgumentMatcher() { + @Override + public boolean matches(FlowData flowData) { + return flowData.getDataKeys().contains("element1"); + } + })); + } } @Test diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java index c891b7181..1abdcd94a 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/integration/PipelineIntegrationTests.java @@ -54,11 +54,10 @@ public static void setup() { @Test public void PipelineIntegration_SingleElement() throws Exception { MultiplyByFiveElement fiveElement = new MultiplyByFiveElement(mock(Logger.class)); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(fiveElement) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(fiveElement.evidenceKeys.get(0), 2); flowData.process(); @@ -71,12 +70,11 @@ public void PipelineIntegration_SingleElement() throws Exception { public void PipelineIntegration_MultipleElements() throws Exception { MultiplyByFiveElement fiveElement = new MultiplyByFiveElement(mock(Logger.class)); MultiplyByTenElement tenElement = new MultiplyByTenElement(mock(Logger.class)); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(fiveElement) .addFlowElement(tenElement) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(fiveElement.evidenceKeys.get(0), 2); flowData.process(); @@ -91,11 +89,10 @@ public void PipelineIntegration_MultipleElements() throws Exception { public void PipelineIntegration_MultipleElementsParallel() throws Exception { MultiplyByFiveElement fiveElement = new MultiplyByFiveElement(mock(Logger.class)); MultiplyByTenElement tenElement = new MultiplyByTenElement(mock(Logger.class)); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElementsParallel(new FlowElement[]{fiveElement, tenElement}) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(fiveElement.evidenceKeys.get(0), 2); flowData.process(); @@ -113,19 +110,20 @@ public void PipelineIntegration_StopFlag() throws Exception { FlowElement testElement = mock(FlowElement.class); when(testElement.getElementDataKey()).thenReturn("test"); when(testElement.getProperties()).thenReturn(Collections.emptyList()); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(stopElement) .addFlowElement(testElement) - .build(); + .build()) { - // Create and process flow data - try (FlowData flowData = pipeline.createFlowData()) { - flowData.process(); + // Create and process flow data + try (FlowData flowData = pipeline.createFlowData()) { + flowData.process(); - // Check that the stop flag is set - assertTrue(flowData.isStopped()); - // Check that the second element was never processed - verify(testElement, never()).process(any(FlowData.class)); + // Check that the stop flag is set + assertTrue(flowData.isStopped()); + // Check that the second element was never processed + verify(testElement, never()).process(any(FlowData.class)); + } } } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java new file mode 100644 index 000000000..6ab3d4221 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RecordingElementBuilder.java @@ -0,0 +1,47 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.testclasses.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.core.flowelements.FlowElement; + +import java.util.ArrayList; +import java.util.List; + +/** + * Builds list splitter elements and keeps each one, so that a test can check + * what happened to the elements a pipeline builder made. + */ +@ElementBuilder +public class RecordingElementBuilder { + + @SuppressWarnings("rawtypes") + public static final List BUILT = new ArrayList<>(); + + @SuppressWarnings("rawtypes") + public FlowElement build() { + FlowElement element = new ListSplitterElement(",", 100000); + BUILT.add(element); + return element; + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 819bcee46..5b7c1b325 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -105,33 +105,36 @@ public void run() throws Exception { loggerFactory) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the engines when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(cloudRequestEngine) .addFlowElement(ageElement) - .build(); + .setAutoCloseElements(true) + .build()) { - String dob = "18/12/1992"; + String dob = "18/12/1992"; - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence("cookie.date-of-birth", dob) - .process(); - - String starSign = - flowData.getFromElement(ageElement).getStarSign(); - if (starSign == null) { - // The service answered but offers no star sign - // product, which is what the 51Degrees cloud says, - // because star signs are only served by the example - // service. The connection, the property and evidence - // key negotiation and the request itself all worked. - System.out.println("The cloud service at " + endpoint() - + " does not offer the star sign product, so no " - + "star sign is available for a date of birth of " - + dob + "."); - } else { - System.out.println("With a date of birth of " + dob - + ", your star sign is " + starSign + "."); + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence("cookie.date-of-birth", dob) + .process(); + + String starSign = + flowData.getFromElement(ageElement).getStarSign(); + if (starSign == null) { + // The service answered but offers no star sign + // product, which is what the 51Degrees cloud says, + // because star signs are only served by the example + // service. The connection, the property and evidence + // key negotiation and the request itself all worked. + System.out.println("The cloud service at " + endpoint() + + " does not offer the star sign product, so no " + + "star sign is available for a date of birth of " + + dob + "."); + } else { + System.out.println("With a date of birth of " + dob + + ", your star sign is " + starSign + "."); + } } } //! [usage] diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java index 8406ac0aa..76d7e4473 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java @@ -45,22 +45,25 @@ public void run() throws Exception { new SimpleFlowElementBuilder(loggerFactory) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the element when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(starSignElement) - .build(); - Calendar dob = Calendar.getInstance(); - dob.set(1992, Calendar.DECEMBER, 18); + .setAutoCloseElements(true) + .build()) { + Calendar dob = Calendar.getInstance(); + dob.set(1992, Calendar.DECEMBER, 18); - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence("date-of-birth", dob.getTime()) - .process(); - - System.out.println("With a date of birth of " + - new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + - ", your star sign is " + - flowData.getFromElement(starSignElement).getStarSign() + - "."); + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence("date-of-birth", dob.getTime()) + .process(); + + System.out.println("With a date of birth of " + + new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + + ", your star sign is " + + flowData.getFromElement(starSignElement).getStarSign() + + "."); + } } //! [usage] } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java index 594cdbd5b..8d803d244 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java @@ -47,22 +47,26 @@ public void run() throws Exception { Main.class.getClassLoader().getResource("starsigns.csv").getPath(), false); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the engine, and with it the data file, + // when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(starSignEngine) - .build(); - Calendar dob = Calendar.getInstance(); - dob.set(1992, Calendar.DECEMBER, 18); + .setAutoCloseElements(true) + .build()) { + Calendar dob = Calendar.getInstance(); + dob.set(1992, Calendar.DECEMBER, 18); - try (FlowData flowData = pipeline.createFlowData()) { - flowData - .addEvidence("date-of-birth", dob.getTime()) - .process(); - - System.out.println("With a date of birth of " + - new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + - ", your star sign is " + - flowData.getFromElement(starSignEngine).getStarSign() + - "."); + try (FlowData flowData = pipeline.createFlowData()) { + flowData + .addEvidence("date-of-birth", dob.getTime()) + .process(); + + System.out.println("With a date of birth of " + + new SimpleDateFormat("yyyy/MM/dd").format(dob.getTime()) + + ", your star sign is " + + flowData.getFromElement(starSignEngine).getStarSign() + + "."); + } } //! [usage] } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java index 407759b4d..f6da01434 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java @@ -88,26 +88,30 @@ public void run() throws Exception { new File(Objects.requireNonNull(getClass().getClassLoader(). getResource("51d.xml")).getFile())); - // Build a new Pipeline from the configuration. - Pipeline pipeline = new PipelineBuilder() + // Build a new Pipeline from the configuration. Closing the + // pipeline closes the ShareUsageElement it created, which sends + // any usage data still queued. + try (Pipeline pipeline = new PipelineBuilder() .addService(http) - .buildFromConfiguration(options); - - // Alternatively, the commented code below shows how to - // configure the ShareUsageElement in code, rather than - // using a configuration file. - //ShareUsageElement usageElement = new ShareUsageBuilder(loggerFactory, http) - // .setSharePercentage(0.01) - // .setMinimumEntriesPerMessage(2500) - // .build(); - //Pipeline pipeline = new PipelineBuilder() - // .addFlowElement(usageElement) - // .build(); + .buildFromConfiguration(options)) { - System.out.println( - "Pipeline created with share usage element. Evidence processed " + - "with this pipeline will now be periodically shared with " + - "51Degrees using the specified configuration."); + // Alternatively, the commented code below shows how to + // configure the ShareUsageElement in code, rather than + // using a configuration file. + //ShareUsageElement usageElement = new ShareUsageBuilder(loggerFactory, http) + // .setSharePercentage(0.01) + // .setMinimumEntriesPerMessage(2500) + // .build(); + //Pipeline pipeline = new PipelineBuilder() + // .addFlowElement(usageElement) + // .setAutoCloseElements(true) + // .build(); + + System.out.println( + "Pipeline created with share usage element. Evidence processed " + + "with this pipeline will now be periodically shared with " + + "51Degrees using the specified configuration."); + } } } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java index 2c3b0ac4f..830f8d8ef 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ComponentMetaData.java @@ -50,9 +50,10 @@ public interface ComponentMetaData extends Closeable { /** * List of the properties which come under the umbrella of this component. + * The caller closes the collection, which can hold native memory. * @return list of properties */ - Iterable getProperties(); + CloseableIterable getProperties(); /** * Get the property from the component which has the name provided. Null is diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java index 2d6357f34..da0c127dc 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaData.java @@ -91,9 +91,10 @@ public interface FiftyOneAspectPropertyMetaData /** * Get the values which relate to the property i.e. the values which can be * returned for this property. + * The caller closes the collection, which can hold the engine's data set. * @return values which relate to the property */ - Iterable getValues(); + CloseableIterable getValues(); /** * Get the default value for the property. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java index cd80a737f..a460d8443 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/FiftyOneAspectPropertyMetaDataDefault.java @@ -130,8 +130,8 @@ public ComponentMetaData getComponent() { } @Override - public Iterable getValues() { - return () -> { + public CloseableIterable getValues() { + return new CloseableIterableDefault(() -> { Iterator innerValues = values.iterator(); return new Iterator() { @@ -145,7 +145,7 @@ public ValueMetaData next() { return innerValues.next(); } }; - }; + }); } @Override diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java index bf0ddfabc..94ea3d982 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ProfileMetaData.java @@ -39,16 +39,18 @@ public interface ProfileMetaData extends Closeable { /** * Get the values which are defined in the profile (for some Engines * multiple profiles are required to build a full set of results). + * The caller closes the collection, which can hold the engine's data set. * @return values defined in the profile */ - Iterable getValues(); + CloseableIterable getValues(); /** * Gets the values associated with the profile and the property name. * @param propertyName to get the values for + * The caller closes the collection, which can hold native memory. * @return values matching the property */ - Iterable getValues(String propertyName); + CloseableIterable getValues(String propertyName); /** * If there is a value for the profile with the property name and value then diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java index 66a433a25..cd25d63ac 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java @@ -33,6 +33,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import fiftyone.pipeline.engines.fiftyone.data.SetHeadersData; import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -66,6 +67,13 @@ public SetHeadersElementTests() { when(internalLogger.getLogger(anyString())).thenReturn(mock(Logger.class)); loggerFactory = new TestLoggerFactory(internalLogger); } + + @AfterEach + public void cleanUp() throws Exception { + if (pipeline != null) { + pipeline.close(); + } + } /** * Check that SetHeaderElement handles both 'String' and @@ -84,14 +92,15 @@ public void setHeadersElement(boolean valueIsAPV) throws Exception { new HashMap(); propertyNameValues.put("SetHeaderBrowserAccept-CH", value); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); - assertEquals("UA-Platform", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); + assertEquals("UA-Platform", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); + } } /** @@ -104,12 +113,13 @@ public void setHeadersElement_InvalidPropertyValues(Object sourcePropertyValue) Map propertyNameValues = new HashMap(); propertyNameValues.put("SetHeaderBrowserAccept-CH", sourcePropertyValue); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + } } /** @@ -139,12 +149,13 @@ public void setHeadersElement_APV_InvalidPropertyValues( Map propertyNameValues = new HashMap(); propertyNameValues.put("SetHeaderBrowserAccept-CH", value); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(0, typedOutput.getResponseHeaderDictionary().size()); + } } /** @@ -167,10 +178,11 @@ public void setHeadersElement_InvalidPropertyNames( Map propertyNameValues = new HashMap(); propertyNameValues.put(sourcePropertyName, "TEST"); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - assertThrows(AggregateException.class, () -> { - data.process(); - }); + try (FlowData data = pipeline.createFlowData()) { + assertThrows(AggregateException.class, () -> { + data.process(); + }); + } } /** @@ -184,17 +196,18 @@ public void setHeadersElement_MultipleProperties() throws Exception { propertyNameValues.put("SetHeaderHardwareCritical-CH", "Sec-CH-UA-Model,Sec-CH-UA-Mobile"); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(2, typedOutput.getResponseHeaderDictionary().size()); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Critical-CH")); - assertEquals("Sec-CH-UA", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); - assertEquals("Sec-CH-UA-Model,Sec-CH-UA-Mobile", - typedOutput.getResponseHeaderDictionary().get("Critical-CH")); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(2, typedOutput.getResponseHeaderDictionary().size()); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Critical-CH")); + assertEquals("Sec-CH-UA", typedOutput.getResponseHeaderDictionary().get("Accept-CH")); + assertEquals("Sec-CH-UA-Model,Sec-CH-UA-Mobile", + typedOutput.getResponseHeaderDictionary().get("Critical-CH")); + } } /** @@ -207,17 +220,18 @@ public void setHeadersElement_MultipleProperties_SameHeader() throws Exception { propertyNameValues.put("SetHeaderBrowserAccept-CH", "Sec-CH-UA"); propertyNameValues.put("SetHeaderHardwareAccept-CH", "Sec-CH-UA-Model,Sec-CH-UA-Mobile"); createPipeline(propertyNameValues); - FlowData data = pipeline.createFlowData(); - data.process(); + try (FlowData data = pipeline.createFlowData()) { + data.process(); - // Verify the output - SetHeadersData typedOutput = getFromFlowData(data); - assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); - assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); - String acceptCHValue = typedOutput.getResponseHeaderDictionary().get("Accept-CH"); - assertTrue(acceptCHValue.contains("Sec-CH-UA")); - assertTrue(acceptCHValue.contains("Sec-CH-UA-Model,Sec-CH-UA-Mobile")); - assertTrue(acceptCHValue.matches("^Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+$")); + // Verify the output + SetHeadersData typedOutput = getFromFlowData(data); + assertEquals(1, typedOutput.getResponseHeaderDictionary().size()); + assertTrue(typedOutput.getResponseHeaderDictionary().containsKey("Accept-CH")); + String acceptCHValue = typedOutput.getResponseHeaderDictionary().get("Accept-CH"); + assertTrue(acceptCHValue.contains("Sec-CH-UA")); + assertTrue(acceptCHValue.contains("Sec-CH-UA-Model,Sec-CH-UA-Mobile")); + assertTrue(acceptCHValue.matches("^Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+,Sec-CH-[a-zA-Z-]+$")); + } } private void createPipeline( diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index cf699e55b..176a2a156 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -32,6 +32,7 @@ import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Disabled; @@ -185,6 +186,18 @@ public void Init() throws Exception { tracker = new TestTracker(); } + /** + * The element runs its own sending thread, which closing it stops. + * Closing an element a test has already closed does nothing more. + */ + @AfterEach + public void cleanUp() throws Exception { + if (shareUsageElement != null) { + shareUsageElement.close(); + } + pipeline.close(); + } + private void createShareUsage( double sharePercentage, int minimumEntriesPerMessage, @@ -250,40 +263,41 @@ public void ShareUsageElement_SingleEvent_ClientIPAndHeader() throws Exception { evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + FIFTYONE_COOKIE_PREFIX + "Profile", "123456"); evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + "RemoveMe", "123456"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, devices.getLength(), "One device expected"); + NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "One device expected"); - NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(2, headers.getLength(), "2 headers expected"); - NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("5.6.7.8",header1.item(0).getTextContent()); - NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("2001::",header2.item(0).getTextContent()); + NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(2, headers.getLength(), "2 headers expected"); + NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("5.6.7.8",header1.item(0).getTextContent()); + NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("2001::",header2.item(0).getTextContent()); - NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, cookies.getLength(), "1 cookie expected"); - assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); - assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); + NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, cookies.getLength(), "1 cookie expected"); + assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); + assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); - NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, clientIp.getLength(), "1 client IP expected"); - assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, clientIp.getLength(), "1 client IP expected"); + assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + } } /** @@ -321,43 +335,44 @@ public void ShareUsageElement_SingleEvent_ClientIPAndHeader_HttpClient() throws evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + FIFTYONE_COOKIE_PREFIX + "Profile", "123456"); evidenceData.put(EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + "RemoveMe", "123456"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] zip = ((TestHttpClient)shareUsageElement.httpClient).getBuffer(); - // unzip the buffer - copy(new GZIPInputStream(new ByteArrayInputStream(zip)), baos); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] zip = ((TestHttpClient)shareUsageElement.httpClient).getBuffer(); + // unzip the buffer + copy(new GZIPInputStream(new ByteArrayInputStream(zip)), baos); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, devices.getLength(), "One device expected"); + NodeList devices = (NodeList) xpath.compile("//Devices/Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "One device expected"); - NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(2, headers.getLength(), "2 headers expected"); - NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("5.6.7.8",header1.item(0).getTextContent()); - NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals("2001::",header2.item(0).getTextContent()); + NodeList headers = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(2, headers.getLength(), "2 headers expected"); + NodeList header1 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='x-forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("5.6.7.8",header1.item(0).getTextContent()); + NodeList header2 = (NodeList) xpath.compile(EVIDENCE_HTTPHEADER_PREFIX+"[@Name='forwarded-for']").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals("2001::",header2.item(0).getTextContent()); - NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, cookies.getLength(), "1 cookie expected"); - assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); - assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); + NodeList cookies = (NodeList) xpath.compile(EVIDENCE_COOKIE_PREFIX).evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, cookies.getLength(), "1 cookie expected"); + assertEquals("51d_Profile", cookies.item(0).getAttributes().getNamedItem("Name").getTextContent()); + assertEquals("123456", cookies.item(0).getFirstChild().getTextContent()); - NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(1, clientIp.getLength(), "1 client IP expected"); - assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + NodeList clientIp = (NodeList) xpath.compile("ClientIP").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(1, clientIp.getLength(), "1 client IP expected"); + assertEquals("1.2.3.4", clientIp.item(0).getTextContent()); + } } @Test @@ -373,14 +388,15 @@ public void ShareUsageElement_TwoEvents_FirstEvent() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); + // Act + shareUsageElement.process(data); - // sending of data never gets triggered - assertNull(shareUsageElement.getSendDataFuture()); + // sending of data never gets triggered + assertNull(shareUsageElement.getSendDataFuture()); + } } @Test @@ -396,30 +412,31 @@ public void ShareUsageElement_TwoEvents_SecondEvent() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - shareUsageElement.process(data); + // Act + shareUsageElement.process(data); + shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - // Verify that there is only one devices element - NodeList devices = (NodeList) xpath.compile("//Devices").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, devices.getLength(), "1 devices node expected"); + // Verify that there is only one devices element + NodeList devices = (NodeList) xpath.compile("//Devices").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "1 devices node expected"); - // Make sure there are 2 'Device' nodes - NodeList device = (NodeList) xpath.compile("//Device").evaluate(devices.item(0), XPathConstants.NODESET); - assertEquals(2, device.getLength(), "2 device nodes expected"); + // Make sure there are 2 'Device' nodes + NodeList device = (NodeList) xpath.compile("//Device").evaluate(devices.item(0), XPathConstants.NODESET); + assertEquals(2, device.getLength(), "2 device nodes expected"); + } } @Test @@ -440,33 +457,34 @@ public void ShareUsageElement_RestrictedHeaders() throws Exception { evidenceData.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "forwarded-for", "2001::"); evidenceData.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "user-agent", useragent); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); - - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); - - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); - - // Verify that there is only one clientIP element - NodeList clientIP = (NodeList) xpath.compile("//ClientIP").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, clientIP.getLength(), "1 ClientIP node expected"); - - // Make sure there is a header node - NodeList device = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, device.getLength(), "1 user-agent header expected"); - // Make sure no x-forwarded-for - NodeList xf = (NodeList) xpath.compile("//header[@Name='x-forwarded-for']").evaluate(doc, XPathConstants.NODESET); - assertEquals(0, xf.getLength(), "0 x-forwarded-for header expected"); - // Make sure no forwarded-for - NodeList f = (NodeList) xpath.compile("//header[@Name='forwarded-for']").evaluate(doc, XPathConstants.NODESET); - assertEquals(0, f.getLength(), "0 forwarded-for header expected"); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); + + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); + + // Verify that there is only one clientIP element + NodeList clientIP = (NodeList) xpath.compile("//ClientIP").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, clientIP.getLength(), "1 ClientIP node expected"); + + // Make sure there is a header node + NodeList device = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, device.getLength(), "1 user-agent header expected"); + // Make sure no x-forwarded-for + NodeList xf = (NodeList) xpath.compile("//header[@Name='x-forwarded-for']").evaluate(doc, XPathConstants.NODESET); + assertEquals(0, xf.getLength(), "0 x-forwarded-for header expected"); + // Make sure no forwarded-for + NodeList f = (NodeList) xpath.compile("//header[@Name='forwarded-for']").evaluate(doc, XPathConstants.NODESET); + assertEquals(0, f.getLength(), "0 forwarded-for header expected"); + } } @Test @@ -483,32 +501,33 @@ public void ShareUsageElement_LowPercentage() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - int requiredEvents = 0; - while (connector.getBaos().size()==0 && requiredEvents <= 1000000) { - shareUsageElement.process(data); - requiredEvents++; + // Act + int requiredEvents = 0; + while (connector.getBaos().size()==0 && requiredEvents <= 1000000) { + shareUsageElement.process(data); + requiredEvents++; + } + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + logger.info("Required events was {}", requiredEvents); + + // Assert + // On average, the number of required events should be around + // 100,000. However, as it's chance based it can vary + // significantly. We only want to catch any gross errors so just + // make sure the value is of the expected order of magnitude. + assertTrue(requiredEvents > 10000, + "Expected the number of required events to be at least " + + "10,000, but was actually '" + requiredEvents + "'"); + assertTrue(requiredEvents < 1000000, + "Expected the number of required events to be less than " + + "1,000,000, but was actually '" + requiredEvents + "'"); } - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); - - logger.info("Required events was {}", requiredEvents); - - // Assert - // On average, the number of required events should be around - // 100,000. However, as it's chance based it can vary - // significantly. We only want to catch any gross errors so just - // make sure the value is of the expected order of magnitude. - assertTrue(requiredEvents > 10000, - "Expected the number of required events to be at least " + - "10,000, but was actually '" + requiredEvents + "'"); - assertTrue(requiredEvents < 1000000, - "Expected the number of required events to be less than " + - "1,000,000, but was actually '" + requiredEvents + "'"); } @Test public void ShareUsageElement_SendOnCleanup() throws Exception { @@ -524,33 +543,34 @@ public void ShareUsageElement_SendOnCleanup() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); + // Act + shareUsageElement.process(data); - // No data should be being sending yet. - assertNull(shareUsageElement.getSendDataFuture()); + // No data should be being sending yet. + assertNull(shareUsageElement.getSendDataFuture()); - // Dispose of the element. - shareUsageElement.close(); + // Dispose of the element. + shareUsageElement.close(); - // Assert - // data should have been sent - assertNotNull(shareUsageElement.getSendDataFuture()); - assertTrue(shareUsageElement.isClosed()); + // Assert + // data should have been sent + assertNotNull(shareUsageElement.getSendDataFuture()); + assertTrue(shareUsageElement.isClosed()); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - // Check that the expected values are populated. - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "1 device expected"); + // Check that the expected values are populated. + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "1 device expected"); + } } @Test public void ShareUsageElement_CancelOnServerError() throws Exception { @@ -568,17 +588,18 @@ public void ShareUsageElement_CancelOnServerError() throws Exception { ((TestConnector)shareUsageElement.dataUploader).setResponseCode(500); Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - // Assert - assertFalse(shareUsageElement.isCanceled()); - assertEquals(0, shareUsageElement.evidenceCollection.size()); + // Assert + assertFalse(shareUsageElement.isCanceled()); + assertEquals(0, shareUsageElement.evidenceCollection.size()); + } } @Test @@ -598,15 +619,16 @@ public void ShareUsageElement_IgnoreOnEvidence() throws Exception { evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); evidenceData.put("header.User-Agent", "Azure Traffic Manager Endpoint Monitor"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Check that the consumer task did not start. - assertNull(shareUsageElement.getSendDataFuture()); + // Act + shareUsageElement.process(data); + // Check that the consumer task did not start. + assertNull(shareUsageElement.getSendDataFuture()); - assertEquals(0, connector.getBaos().size()); + assertEquals(0, connector.getBaos().size()); + } } @Test @Disabled // we now throw exceptions for invalid data @@ -672,28 +694,29 @@ public void ShareUsageElement_BadSchema() throws Exception { // (0x0018) - Cancel control character evidenceData.put(EVIDENCE_HEADER_USERAGENT_KEY, ""); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); - - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); - - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); - - // Check that the expected values are populated. - NodeList f = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "1 user-agent header expected"); - assertEquals("", f.item(0).getTextContent()); - assertEquals("true", f.item(0).getAttributes().getNamedItem("replaced").getTextContent()); - NodeList b = (NodeList) xpath.compile("//BadSchema").evaluate(doc, XPathConstants.NODESET); - assertEquals(0, b.getLength()); + // Act + shareUsageElement.process(data); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); + + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); + + // Check that the expected values are populated. + NodeList f = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "1 user-agent header expected"); + assertEquals("", f.item(0).getTextContent()); + assertEquals("true", f.item(0).getAttributes().getNamedItem("replaced").getTextContent()); + NodeList b = (NodeList) xpath.compile("//BadSchema").evaluate(doc, XPathConstants.NODESET); + assertEquals(0, b.getLength()); + } } /** @@ -715,27 +738,28 @@ public void ShareUsageElement_SessionIdAndSequence_None() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - sequenceElement.process(data); - shareUsageElement.process(data); + // Act + sequenceElement.process(data); + shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "Expecting 1 session ID"); - NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); - assertEquals("1", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "Expecting 1 session ID"); + NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); + assertEquals("1", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + } } /** @@ -760,28 +784,29 @@ public void ShareUsageElement_SessionIdAndSequence_Existing() throws Exception { evidenceData.put(Constants.EVIDENCE_SESSIONID, "abcdefg-hijklmn-opqrst-uvwxyz"); evidenceData.put(Constants.EVIDENCE_SEQUENCE, 2); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // Act - sequenceElement.process(data); - shareUsageElement.process(data); + // Act + sequenceElement.process(data); + shareUsageElement.process(data); - // Wait for the consumer task to finish. - assertNotNull(shareUsageElement.getSendDataFuture()); - shareUsageElement.getSendDataFuture().get(); + // Wait for the consumer task to finish. + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); - ByteArrayOutputStream baos = connector.getBaos(); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); - assertEquals(1, f.getLength(), "Expecting 1 session ID"); - assertEquals("abcdefg-hijklmn-opqrst-uvwxyz", f.item(0).getTextContent(), "Expecting session ID to be ..."); - NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); - assertEquals("3", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + NodeList f = (NodeList) xpath.compile("//SessionId").evaluate(doc, XPathConstants.NODESET); + assertEquals(1, f.getLength(), "Expecting 1 session ID"); + assertEquals("abcdefg-hijklmn-opqrst-uvwxyz", f.item(0).getTextContent(), "Expecting session ID to be ..."); + NodeList ff = (NodeList) xpath.compile("//Sequence").evaluate(doc, XPathConstants.NODESET); + assertEquals("3", ff.item(0).getTextContent(), "Expecting Sequence ID 1"); + } } /** @@ -808,28 +833,29 @@ public void CheckForOffByOneError() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // collect multiple documents - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // Act - for (int i=0; i < count; i++) { - shareUsageElement.process(data); - } - // Wait for the consumer task to finish. - shareUsageElement.close(); + // collect multiple documents + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // Act + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + // Wait for the consumer task to finish. + shareUsageElement.close(); - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(count, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(count, f.getLength(), "Expecting devices"); + } } /** * A consumer that has finished its drain loop with fewer entries left than @@ -890,54 +916,55 @@ protected void sendUsageData() { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // collect multiple documents - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // collect multiple documents + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // close in a finally as well, so that an assertion failure before the - // Act step below still releases the lingering consumer. - try { - // Send one full batch and wait for the consumer to drain it. - for (int i = 0; i < minimumEntriesPerMessage; i++) { - shareUsageElement.process(data); - } - assertTrue(consumerHasDrainedQueue.await(LINGER_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), - "The consumer should have drained the first batch"); + // close in a finally as well, so that an assertion failure before the + // Act step below still releases the lingering consumer. + try { + // Send one full batch and wait for the consumer to drain it. + for (int i = 0; i < minimumEntriesPerMessage; i++) { + shareUsageElement.process(data); + } + assertTrue(consumerHasDrainedQueue.await(LINGER_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), + "The consumer should have drained the first batch"); - // Queue too few entries to start a consumer of their own, while the - // previous consumer's task has not completed yet. - for (int i = 0; i < remainder; i++) { - shareUsageElement.process(data); - } - assertEquals(remainder, shareUsageElement.evidenceCollection.size()); - assertTrue(shareUsageElement.isRunning(), - "The first consumer should still be reported as running"); + // Queue too few entries to start a consumer of their own, while the + // previous consumer's task has not completed yet. + for (int i = 0; i < remainder; i++) { + shareUsageElement.process(data); + } + assertEquals(remainder, shareUsageElement.evidenceCollection.size()); + assertTrue(shareUsageElement.isRunning(), + "The first consumer should still be reported as running"); - // Act - close must flush the remainder. - shareUsageElement.close(); - assertEquals(0, shareUsageElement.evidenceCollection.size(), - "close should leave nothing queued"); - } finally { - // Swallowed so that a failure above is still the reported one - the - // close in the Act step is the one under test. - try { + // Act - close must flush the remainder. shareUsageElement.close(); - } catch (Exception exception) { - logger.warn("Exception closing the element under test", exception); + assertEquals(0, shareUsageElement.evidenceCollection.size(), + "close should leave nothing queued"); + } finally { + // Swallowed so that a failure above is still the reported one - the + // close in the Act step is the one under test. + try { + shareUsageElement.close(); + } catch (Exception exception) { + logger.warn("Exception closing the element under test", exception); + } } - } - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - assertEquals(minimumEntriesPerMessage + remainder, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + assertEquals(minimumEntriesPerMessage + remainder, f.getLength(), "Expecting devices"); + } } @Test @@ -962,29 +989,30 @@ public void CheckForDrainQueueEvenIfError() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // collect multiple documents - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // Act - for (int i=0; i < count; i++) { - shareUsageElement.process(data); - } - // Wait for the consumer task to finish. - shareUsageElement.close(); + // collect multiple documents + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // Act + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + // Wait for the consumer task to finish. + shareUsageElement.close(); - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - // this shows that sending is ignoring the failures - assertEquals(count, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + // this shows that sending is ignoring the failures + assertEquals(count, f.getLength(), "Expecting devices"); + } } @Test public void CheckForContinueAndResumeWhenQueueFull() throws Exception { @@ -1003,48 +1031,49 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - FlowData data = pipeline.createFlowData(); - data.addEvidence(evidenceData); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); - // the output document will collect multiple documents, - // so we need to wrap them in a root element - connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); + // the output document will collect multiple documents, + // so we need to wrap them in a root element + connector.getBaos().write("".getBytes(StandardCharsets.UTF_8)); - // Overwhelm the queue - int count = 55; - for (int i=0; i < count; i++) { - shareUsageElement.process(data); - } - long amountOfLostData; - assertTrue((amountOfLostData = shareUsageElement.lostData) > 0); - // wait for queue to drain - logger.info("Sleeping"); - Thread.sleep(5000); - - // send a new element which should reset the flag - shareUsageElement.process(data); - assertEquals(amountOfLostData, shareUsageElement.lostData); - - // overwhelm the queue again - for (int i=0; i < count; i++) { + // Overwhelm the queue + int count = 55; + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + long amountOfLostData; + assertTrue((amountOfLostData = shareUsageElement.lostData) > 0); + // wait for queue to drain + logger.info("Sleeping"); + Thread.sleep(5000); + + // send a new element which should reset the flag shareUsageElement.process(data); - } - assertTrue(amountOfLostData < shareUsageElement.lostData); + assertEquals(amountOfLostData, shareUsageElement.lostData); + + // overwhelm the queue again + for (int i=0; i < count; i++) { + shareUsageElement.process(data); + } + assertTrue(amountOfLostData < shareUsageElement.lostData); - // Wait for the consumer task to finish. - shareUsageElement.close(); + // Wait for the consumer task to finish. + shareUsageElement.close(); - // get the data output - ByteArrayOutputStream baos = connector.getBaos(); - baos.write("".getBytes(StandardCharsets.UTF_8)); - byte[] output = baos.toByteArray(); + // get the data output + ByteArrayOutputStream baos = connector.getBaos(); + baos.write("".getBytes(StandardCharsets.UTF_8)); + byte[] output = baos.toByteArray(); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(output)); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(output)); - NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); - // the count is what we put in minus what we have recorded as lost - assertEquals(2 * count + 1 - shareUsageElement.lostData, f.getLength(), "Expecting devices"); + NodeList f = (NodeList) xpath.compile("//Device").evaluate(doc, XPathConstants.NODESET); + // the count is what we put in minus what we have recorded as lost + assertEquals(2 * count + 1 - shareUsageElement.lostData, f.getLength(), "Expecting devices"); + } } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java index be67bd7e4..9583850a7 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java @@ -30,10 +30,12 @@ import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.services.DataUploader; import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; import org.json.JSONObject; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -95,43 +97,50 @@ public void Initialise() throws Exception { .setProcessCost(0) .build(); + // Closing the element sends what is still queued, so it is given an + // uploader that sends nothing anywhere. ShareUsageElement shareUsage = - new ShareUsageBuilder(loggerFactory) + new ShareUsageBuilder(loggerFactory, mock(DataUploader.class)) .build(); pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(engine) .addFlowElement(shareUsage) + .setAutoCloseElements(true) .build(); } + @AfterEach + public void cleanUp() throws Exception { + pipeline.close(); + } + + // The FlowData are all processed before any is closed, so that closing + // them is not part of the time measured. They are closed together in a + // finally block rather than one by one with try-with-resources. @Test - public void ShareUsageOverhead_SingleEvidence() { + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void ShareUsageOverhead_SingleEvidence() throws Exception { int iterations = 10_000; double maxOverheadPerCall = 0.1; List data = new ArrayList<>(); - for (int i = 0; i < iterations; i++) { - FlowData flowData = pipeline.createFlowData(); - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); - data.add(flowData); - } + long start; + long end; + try { + for (int i = 0; i < iterations; i++) { + FlowData flowData = pipeline.createFlowData(); + data.add(flowData); + flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); + } - long start = System.currentTimeMillis(); - for (FlowData entry : data) { - entry.process(); - } - long end = System.currentTimeMillis(); - - // It is not efficient to use try-with-resources - // in this scenario, so loop through and perform - // manual close on each FlowData - data.forEach((f) -> { - try { - f.close(); - } catch (Exception e) { - e.printStackTrace(); + start = System.currentTimeMillis(); + for (FlowData entry : data) { + entry.process(); } - }); + end = System.currentTimeMillis(); + } finally { + closeAll(data); + } double msOverheadPerCall = (double)(end - start) / iterations; overheadMillis.put("ShareUsageOverhead_SingleEvidence_ms", msOverheadPerCall); @@ -142,36 +151,34 @@ public void ShareUsageOverhead_SingleEvidence() { msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCall); } + // See ShareUsageOverhead_SingleEvidence for why the FlowData are closed + // in a finally block. @Test - public void ShareUsageOverhead_ThousandEvidence() { + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void ShareUsageOverhead_ThousandEvidence() throws Exception { int iterations = 1_000; int evidenceCount = 1_000; int maxOverheadPerCallMillis = 10; List data = new ArrayList<>(); - for (int i = 0; i < iterations; i++) { - FlowData flowData = pipeline.createFlowData(); - for (int j = 0; j < evidenceCount; j++) { - flowData.addEvidence("header." + j, j); + long start; + long end; + try { + for (int i = 0; i < iterations; i++) { + FlowData flowData = pipeline.createFlowData(); + data.add(flowData); + for (int j = 0; j < evidenceCount; j++) { + flowData.addEvidence("header." + j, j); + } + flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); } - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); - data.add(flowData); - } - long start = System.currentTimeMillis(); - for (FlowData entry : data) { - entry.process(); - } - long end = System.currentTimeMillis(); - - // It is not efficient to use try-with-resources - // in this scenario, so loop through and perform - // manual close on each FlowData - data.forEach((f) -> { - try { - f.close(); - } catch (Exception e) { - e.printStackTrace(); + start = System.currentTimeMillis(); + for (FlowData entry : data) { + entry.process(); } - }); + end = System.currentTimeMillis(); + } finally { + closeAll(data); + } double msOverheadPerCall = (double)(end - start) / iterations; overheadMillis.put("ShareUsageOverhead_HundredEvidence_ms", msOverheadPerCall); @@ -181,4 +188,10 @@ public void ShareUsageOverhead_ThousandEvidence() { "Pipeline with share usage overhead per Process call was " + msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCallMillis); } + + private static void closeAll(List data) throws Exception { + for (FlowData flowData : data) { + flowData.close(); + } + } } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java index be3bbaada..5aed919b3 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/CookieTests.java @@ -138,16 +138,15 @@ public void TestJavaScriptCookies( JavaScriptBuilderElement jsElement = new JavaScriptBuilderElementBuilder(loggerFactory) .setEnableCookies(enableInConfig) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) - .addFlowElement(cookieElement) - .addFlowElement(sequenceElement) - .addFlowElement(jsonElement) - .addFlowElement(jsElement) - .build(); - // Act String javaScript = null; - try (FlowData flowData = pipeline.createFlowData()) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(cookieElement) + .addFlowElement(sequenceElement) + .addFlowElement(jsonElement) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence( Constants.EVIDENCE_ENABLE_COOKIES, diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index a94604104..2178a5f66 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -72,12 +72,12 @@ private String render(String contextRoot) throws Exception { builder.setContextRoot(contextRoot); } JavaScriptBuilderElement jsElement = builder.build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) .addFlowElement(jsElement) .build(); - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence("query.user-agent", "iPhone"); flowData.addEvidence("query.id.usage", "non-marketing"); flowData.process(); @@ -164,18 +164,19 @@ private List renderWithHosts(String... hosts) throws Exception { new JavaScriptBuilderElementBuilder(factory) .setEndpoint("/json") .build(); - Pipeline pipeline = new PipelineBuilder(factory) + List scripts = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(factory) .addFlowElement(new SequenceElementBuilder(factory).build()) .addFlowElement(new JsonBuilderElementBuilder(factory).build()) .addFlowElement(jsElement) - .build(); - List scripts = new ArrayList<>(); - for (String host : hosts) { - try (FlowData flowData = pipeline.createFlowData()) { - flowData.addEvidence(Constants.EVIDENCE_HOST_KEY, host); - flowData.process(); - scripts.add( - flowData.getFromElement(jsElement).getJavaScript()); + .build()) { + for (String host : hosts) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(Constants.EVIDENCE_HOST_KEY, host); + flowData.process(); + scripts.add( + flowData.getFromElement(jsElement).getJavaScript()); + } } } return scripts; diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index 9c9d9f935..965526329 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -460,13 +460,15 @@ public void JsonBuilder_LazyLoading() throws Exception { .build(); SequenceElement sequenceElement = new SequenceElementBuilder(loggerFactory) .build(); - Pipeline pipeline = new PipelineBuilder(loggerFactory) + // The pipeline closes the elements, the lazy loading engine's + // threads included. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) .addFlowElement(sequenceElement) .addFlowElement(engine) .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) .build(); - - try (FlowData flowData = pipeline.createFlowData()) { + FlowData flowData = pipeline.createFlowData()) { flowData.process(); diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java index 351b3b5da..309b7669b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java @@ -127,15 +127,24 @@ public boolean preHandle( HttpServletResponse response, Object handler) throws Exception { resultService.process(request); - - // Set the Client Hints request headers - uachService.setResponseHeaders(request, response); - - if (fiftyOneJsService.serveJS(request, response) == false && - fiftyOneJsService.serveJson(request, response) == false) { - return super.preHandle(request, response, handler); + + boolean proceed = false; + try { + // Set the Client Hints request headers + uachService.setResponseHeaders(request, response); + + proceed = fiftyOneJsService.serveJS(request, response) == false && + fiftyOneJsService.serveJson(request, response) == false && + super.preHandle(request, response, handler); + return proceed; + } finally { + // Spring calls afterCompletion, which closes the FlowData, only + // when preHandle returns true. In every other case, serving the + // JavaScript or JSON included, the FlowData is closed here. + if (proceed == false) { + flowDataProvider.getFlowData(request).close(); + } } - return false; } @Override diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java index 2b9562ada..077a88b46 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.web.mvc; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -123,4 +124,18 @@ public void FiftyOneInterceptor_AfterCompletion() throws Exception { verify(flowData, times(1)) .close(); } + + /** + * Check that the FlowData is closed when preHandle serves the JavaScript + * itself, as Spring does not call afterCompletion when preHandle returns + * false. + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() + throws Exception { + assertFalse(interceptor.preHandle(request, response, handler)); + verify(flowData, times(1)) + .close(); + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java index 1834f5a56..098da0926 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java @@ -122,27 +122,30 @@ public void doFilter( // Populate the request properties and store against the // HttpContext. resultService.process((HttpServletRequest)request); - - // Get FlowData from the request. - FlowData flowData = flowDataProviderCore.getFlowData((HttpServletRequest)request); - - // Set UACH response headers. - uachServiceCore.setResponseHeaders( - flowData, (HttpServletResponse)response); - - // If 51Degrees JavaScript or JSON is being requested then serve it. - // Otherwise continue down the filter Pipeline. - if (jsService.serveJS((HttpServletRequest)request, (HttpServletResponse) response) == false && - jsService.serveJson((HttpServletRequest)request, (HttpServletResponse) response) == false) { - chain.doFilter(request, response);//sends request to next resource - } - // The rest of the filters in the chain have now been called. So it is - // time to dispose of the FlowData instance. try { - flowDataProviderCore.getFlowData((HttpServletRequest) request).close(); - } catch (Exception e) { - throw new ServletException("FlowData could not be disposed of.", e); + // Get FlowData from the request. + FlowData flowData = flowDataProviderCore.getFlowData((HttpServletRequest)request); + + // Set UACH response headers. + uachServiceCore.setResponseHeaders( + flowData, (HttpServletResponse)response); + + // If 51Degrees JavaScript or JSON is being requested then serve it. + // Otherwise continue down the filter Pipeline. + if (jsService.serveJS((HttpServletRequest)request, (HttpServletResponse) response) == false && + jsService.serveJson((HttpServletRequest)request, (HttpServletResponse) response) == false) { + chain.doFilter(request, response);//sends request to next resource + } + } finally { + // The rest of the filters in the chain have now been called, or + // one of them failed. Either way it is time to dispose of the + // FlowData instance. + try { + flowDataProviderCore.getFlowData((HttpServletRequest) request).close(); + } catch (Exception e) { + throw new ServletException("FlowData could not be disposed of.", e); + } } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java index 9bc21609a..289c90ff9 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java @@ -71,14 +71,29 @@ public Default( this.pipeline = pipeline; } + /** + * The FlowData is kept in the request, and whatever handles the end + * of the request closes it. If processing fails it never reaches the + * request, so it is closed here instead. + */ @Override + @SuppressWarnings("PMD.FlowDataInTryWithResources") public void process(HttpServletRequest request) { // Create the flowData FlowData flowData = pipeline.createFlowData(); - // Extract the required pieces of evidence from the request - evidenceService.addEvidenceFromRequest(flowData, request); - // Start processing the data - flowData.process(); + try { + // Extract the required pieces of evidence from the request + evidenceService.addEvidenceFromRequest(flowData, request); + // Start processing the data + flowData.process(); + } catch (RuntimeException | Error e) { + try { + flowData.close(); + } catch (Exception closeException) { + e.addSuppressed(closeException); + } + throw e; + } // Store the FlowData in the request request.setAttribute(HTTPCONTEXT_FLOWDATA, flowData); } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java index e4362e3ae..e0f7b7508 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java @@ -25,6 +25,7 @@ import fiftyone.pipeline.core.data.ElementData; import fiftyone.pipeline.core.data.FlowData; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,7 +42,9 @@ import static fiftyone.pipeline.engines.fiftyone.data.SetHeadersData.*; import static fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement.*; import static fiftyone.pipeline.web.Constants.*; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -54,6 +57,7 @@ public class PipelineFilterTests { private FilterChain chain; private FlowData flowData; private ElementData elementData; + private PipelineFilter filter; @Before public void init() throws IOException, ServletException { @@ -102,13 +106,23 @@ public void init() throws IOException, ServletException { }).when(request).setAttribute(eq(HTTPCONTEXT_FLOWDATA), any(Object.class)); } + /** + * Close the pipeline each filter built. + */ + @After + public void cleanUp() { + if (filter != null) { + filter.destroy(); + } + } + /** * Check that the filter can be initialised. * @throws ServletException */ @Test public void PipelineFilter_Initialise() throws ServletException { - PipelineFilter filter = new PipelineFilter(); + filter = new PipelineFilter(); filter.init(config); } @@ -121,7 +135,7 @@ public void PipelineFilter_Initialise() throws ServletException { */ @Test public void PipelineFilter_Process() throws ServletException, IOException { - PipelineFilter filter = new PipelineFilter(); + filter = new PipelineFilter(); filter.init(config); filter.doFilter(request, response, chain); @@ -142,10 +156,31 @@ public void PipelineFilter_Process() throws ServletException, IOException { */ @Test public void PipelineFilter_CloseFlowData() throws Exception { - PipelineFilter filter = new PipelineFilter(); + filter = new PipelineFilter(); filter.init(config); filter.doFilter(request, response, chain); verify(flowData, times(1)).close(); } + + /** + * Check that the filter disposes of the FlowData when a later filter in + * the chain fails. + * @throws Exception + */ + @Test + public void PipelineFilter_CloseFlowData_ChainFails() throws Exception { + doThrow(new ServletException("chain failed")) + .when(chain).doFilter(any(ServletRequest.class), any(ServletResponse.class)); + filter = new PipelineFilter(); + + filter.init(config); + try { + filter.doFilter(request, response, chain); + fail("Expected the exception from the chain"); + } catch (ServletException e) { + assertEquals("chain failed", e.getMessage()); + } + verify(flowData, times(1)).close(); + } } diff --git a/pipeline/pom.xml b/pipeline/pom.xml index a3c3b5afc..abcd6d8f6 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -95,6 +95,7 @@ 2.6 3.7 0.7.0 + 3.26.0 true false @@ -351,6 +352,40 @@ + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + + + ${maven.multiModuleProjectDirectory}/ci/pmd-resource-ruleset.xml + + true + true + true + false + + **/interop/swig/*.java + + + + + check-resources-closed + process-test-classes + + check + + + +