From f607449cb7d64b9f0e8269a1c6db534827ad9569 Mon Sep 17 00:00:00 2001 From: DanasFi Date: Mon, 14 Apr 2025 03:35:33 -0400 Subject: [PATCH 1/6] Added metric for task queue total size. --- crates/meilisearch/db.snapshot | Bin 0 -> 19430 bytes crates/meilisearch/src/metrics.rs | 5 +++++ crates/meilisearch/src/routes/metrics.rs | 1 + 3 files changed, 6 insertions(+) create mode 100644 crates/meilisearch/db.snapshot diff --git a/crates/meilisearch/db.snapshot b/crates/meilisearch/db.snapshot new file mode 100644 index 0000000000000000000000000000000000000000..e09c484c59b583e56416552c6306f7c20bd185d9 GIT binary patch literal 19430 zcmb@t1ymee(=Lh=AV35QnxMfU4DJa|fCP65!QI^h1P=~_ySuvtcXtWy?lyggZodUYRx&l!{F&z%A=p9!Yzk}xOrzs6xs-FIt>4vY^&+S5 z-Tj86!X^z6({UV;pWpaag6GJsxSbs@-U%M`E|K*eFYkb@LZ8^qQG(DB_iYmuaf3ZN zOXR9!s2qp6_o-4>6#sBQVo&26+!HfRh$|!GMYGXR%(tRK=0>NuconJU2md>*U%3w+ z@KKC_j~SWC`Qui<{%T#OKvKpsj8BOUR(72XD4*puTNzNPu(anq=OHHi4Ls>7ClcH#OMusut2t zOEz9@y$2Q=BD}t*=Hgd56>2g)9E#3(Z?ni|i$3`5A>F+WgO5pBV5LJAAb2PtAQ*=i zsa`daZrPcsj?CjQIHD=W>&2))GzP|-1^c%pJsi*NO0Gj*o-Na)WUXb_$h*g!6(@ub z#_HHJnVc6fPdnK#$n{^hhlk&%tvv6yCY<-O-m}LX{1br?|$xiudmOT<5Xf+s1)@q891;yQ8$tfwa$pYhY z)=PHNdVu*2u!Ms}iE9n{C(65G*OX&-w#M7DEx`x1B&kYMg+z0$y<8Q=IWwo_9Db>h>L{G|e;sy=Ld zR0_*Fn)E`$z-%O1CE*BimFL-?iUVKgg4>O%`l}M(#THzNrY1jlsJh9~O}RR(8C^UL z5CWG^-1i(_TQd?eRD*YO6>WkKPALxq>omo#GjNvG+{V?ErW}tG3~^lUG)Ma9l_ytS|#xbgfWy{ zqEXOCZBw@#>-H(6QlcwScGaN z8N5?!>gNdT_-SvaRC0|mM-{NE16&KdCd_9gW0zEDf1DYsLl$qBZeA?j*lq!9hpu_X zB$@|9xH>SZ(C;=i!tEjdHo-Pz<%c1Et2ftkaLYH|*-yBXENdp3&xDWNe-<%HLxym_ zYWTh}^yF?yd_fn5wEP8yIccD;D`8DdRgukp6yyqpCe+*qR3ECN7BT9jcXNW~Vdt1h z>u$*w(VPLX${1ff)BB8MjlmC<-J0Q6fn>pDO}4GcoXgQ0E((~_x;Kqc*_VNnMaQnj zMJ2I?SS_me+J%T}f#@;hP6+P(xvEF%(fZ4R_k$hT;ZSXVfPL;R;+AP>`2ip<+9S5XF_c*0WP3aOhPR-$ye6-p(Us;c1o8F+7-299SXkIaW zW&O_Rh%ls?7&p*`JQ8TmwR-uG&N{=tu0YvR>}ZbteL@la!6$|B({_r#r6wSx%Sa5z z_#lX+Y0Q}8)h**wChzHmn>nT68_Vfb-i&kPfB@?rSipLWAN?*?P1c-IHSoZfxU}xa zhR5Zm_g-{nFDtI5ZzScS`jaN_XSH?5_?gtKxZWIjoA1Z6m(>MqRg`k()N^SPPcy-2 z7=&Kq#tcIIc#jlWyn+=^u?;I`=N(%$RPN1O8qTO&U;h!cvqV@l8LK> zNO_CuQF5Es@d|oj9yVo7BTHank0zxFs+|+uCL;m`UPw9WB#ZWI@l|KmI=^nM396T3 zV+h;+@}Hk>p7=uqKg*k)o_+h};25waTvgo+MG^@{%+zd!yb zFE{ukz`Br0@zgd1T+FxVwdKhSYBXXBGE9y65iLT)kz2XDM{sX(9V0u;r0mFsK+b|l zTav%Rxm!SXGirk=BzUH;+Btt8e5NUkW{(<8cywi#r8Giiby>pl!YQrRla#5Iz_37_ zkrDAu(btYKD3SlO2hZIIEmn?Ufx*X&5=MRvG;bY+pH9{#gac`rK6jx7*uAlVv@gA9 z0@87ZBeA71zfuJTlJ9u;$Sf5hw2&Ejx1X>cbOc7#spq$2645UvxiWRX7)AruLn$;a z4G@rQ>2?w~=iMVM=>$VY;JIUdWoA-y#(dS04W&>JJpQoPl*pve9l?NPJ*L1B)4H&p zy{7Cvu5csH8PjUfu3$&!r6h&AsXR7`LL@ojXbnpw$+MuCpa1rKDv}RwtN7FB&Sqoi zNW(Uyj0K6B6>YgtxM02Ebs-w>mzG+xz>Fthyg3~|l{n1l<7-N6AMI_PwaARlV zc4PN}x|QUzWupja$+MDC575A^3%CTx9@~}rC8;w^U+r1T=X`N!b+!DJxtbthV0EzA zt8d+3`B9%qPBq-nsp9z^;hmUAubwdZZU?j6)BZ>dqu!ka$6h0&%{*M1!tYoNQf(Ji z-3hAoMb9V}ew0jaIAvMZC9$x7iguJNd@qe@QJVJV?Y$!#%PO6eV5-OQ0J?8FBeIi5 z2IKN3vr?1;VZjV`*}^6Y>|EpLhONQ%7cn%20A16wYJL${^5bcnA^^KxoH#hOOizXkQEBuyt-HUjGbkRkS9GWfe-&W$mnV(s)MrVa-#2{qFwMvm=G<%**{+ zl6G(tj_Qu;cT3$}>$Fk2kBJppDRV8>>BZai*;}F~zSQ5^WVd2sPLT&aGr%vp=lu3J zWJXndV(q59xs|P@qc>G!E{g65|sX^}N%$^qv|YnW`JPKRKl~ zkz3SW!NQlCdCEHo5KO(hc7(cRXy zZ|HIu3?^(freidGs?4PQf+g?^_I~I|mO74OeH@*Prgh)(kNSMth*)JiTf|d?_gUB)L5TgrHKv1A3N{1uPDnGRn*N|9b(-|nenKjwev0!p zd|NQAjf%9!5XIJRx#9ceHcoFC^svD)N-E1j()V|?5{)}t?c3@Y9J4IQED0{`T(d?S z@-JGsDR7=P%yCAyh9fS5ap-b3CN>iB;jq%QjSEelaM{w-_ZVjn2=)eg)AyGj>=Np` z`C?+nGn(?Na(lon(0jKc?!BjxjXg8y&3;1w?r*n*Io&!YG*R|zJ!NpjL!T~3lq5*v zx0skvHCBSq`qc9Di7H;YEc_VdeQ_mh!@R-g>ajjqfNqRv%4*Sat zu46*)$=AKGXc0g0iJ5~l zxzeK)wO5F&%6s|kBd}ZwoZ4kEa!SEEQflmNHXX+GT@?`>R)yx`qy>iMZF!*uM)pktwsaOLW2T+y9evfhn+{Osv0_uty!fkwVL{Ky~!KFfMw6qMi5qa-jYbyROHO- z$IjOX|2JNm3z)U`lxZj5xK*CS^;+**Q<`anr%g|&sElKk)cL|<9)&pb7{qSYr|at4 zQVDOjUsY_!BplVLCtjl!uff@d#+w6`1;21c znxssPJBn?ufqC7<5lFd)lZK*hQtI<^V5R@ zaY|n!3SLT3#t~$a+6jx8DOJ(ooqxv&)~3+(^o}COJ9*{DpqAa+Th07|J!eY@VOo)C zPj<775u-_?wGgT<=Vad0wUo+Qzih7x4TiqhS++pofDtY>qO@?a5_@?*_h7@yVo7v0o88>wce3!Jjr@8H0HVSq}a$Z5IOh<)Ohzl zx6=0~JS#w$M>rH!(6+<%t_&o9m1tUDotw`xKH?n1(7|-BEll(N$&ah#7th`nK-Cus zS=$D`v7q!B798*$*4MrWeI9_3KAu@37R)fr?JBBO}fgA#0orJSQHV zHWKnLv=3zZT65rE=?pb@FLN#iCGhM_eYopO|yc&@%)`mz5k0+o?j|Xc(58n`1y@{pJ=%Z_QWBt46Khk zF)6_~hAs6;Ss=o6h}(_E*ePPa{`R&mlwA-0U20EymMYt|S2unVvE^;WN~7=d#F_!> z?JqFG3Qe<0u(=XS{H@N4_Ad?h6DF8J5A*+o%3>P`;n!UpxP?#sB7@ zKi0pl`>)iOzA_4+@Li>c$*#=MK@J*_q`~jb?ZwqEJ0XYmkeKVyBx8TlD8cSTkf(Md zItv@UT7r#d;GrjzRYg0gc(@-{m2hgVzVC?KOz+KCUY zk5n=rZ~YaDrtx=g>U_Uj5SSSg8$8;b+z6@3KHZPp>&wDoh>1{Dkud%t8$t1T{hPOc zzCN{J2fXVmQj)@`uPTbI3XDy>l6mJN;ejrBdCC$Jdeqc{65^j;4?R8H7sw)iTVH&p z-YN?X{Ne6y4>ZK;w)%z%P3dAH)Kw&`e;8oA#|w@X)0t6RV$tMCpkY0(SWHSa;OgGp2zs_e6t(EvNh@p5cJTLTlK4N=9bJ-28L| zqnp=NsVH!jh4`1Z9vneNC4j@5ANy|`hw$GCX;DbpDQi?{C{`g(Z#r7kkHj&#={x}E zF`*_mqKQbVk-G;euT8+Y2w+q`8oLcTwYd#Xd(Ko148vIXxee`mmK+TXW2)?b_MxYK z7{>b0@DV?ro2cxEJ+KyNARJ*p{{1i0?u1dS4~p#gKl{*A+m-!-7jloCWdGHPqlO5y zp_NRsJ!ft>_+R&XZDuQH*@ss_+ae6!Qyfz>@p( za#`gs8so+<-q7T`WEAx(OJ!YLDIBCt9RCT3_Y#EafVWx!LOT+{a~v79Pn$QZU|c9k z+zD;y15JL>o~j{8)5~O?#l4lx3*_mPbb65!L4|^Znf^8xxaP?^T3BA5W>8hV;-9Yp z_Tr>B<%_>oI9*vc(*_J8 zk7S6!`{HcmR}}6)YFSQtJ6OW^$nGfc{*_TC4gV2U^Qj4W2zxS%j^{LHkgA1Nabw0n zn!N?P$a;VLYWez8W?$J}Ls~MUhC22I>L#_v04aI$BEX_5BHCOdNfvA0tzlPCN zhCi&?bP%{0ls^k_SG(OT8D9by>{<_o3U|PX$(3Rikm7h8t8n1qercs}0Q6s}DNaI` z9WMLwF9F8r7-(zZ=_~poV<3p%us|QM*m1u20X=0=*U&hf1j%`$k8}2O9n)x&|L&btQ&KN_V3>Lu0`w;ZdqK}z7cTZm=Z-${MM<4an5`TiGL?YlZECxcUoQ`rB) zq{n#xWf&hT?Ypqx_g%WYaVMee+9QbT$<0W+AH}%_sh-1vm@YD8-AWB%?vka&Cu-Dzn) zO^5zisLX5HJphO#Hwu$Cz@3?K5f{LJ-r=7=1OT&P7?+yE7yRhV2LnXC+z~F+w=(}q zEs&4#i&1y?oM&P7w3uu21ho*(hi#hOm4!dFn|j9ar)ZzMdPeZ4&I5>ykYNfS&Xux` zyN(w6*cH9B5nJT`5hNdzfAEm@Qcy$w>NT?_48ITP{*tjQngRWgW-QlK2D>jW@Z7p_ zKtFQh+gBkW8O!XTV0SGo{#&=Vpj$VM#qBGn!80x2IL*x{luf`wJTc<+UCLi+kc?>j zzNeuLVZfZhtKsj2M>9nquBg_Y@&ZR8u8?I{K+XaH$X<_Wl>)E+QlfCNX7e6kxs5~3 zFY#mj@~a+vx57QV+3;YNM*7L4uPjC;9947)udHISZ;{wBS{y9N_cCMZM3$LcS7&zi zKl@C5zZzU2t7$g)+S-?mjC{W~_4?U3{LdrtE4`#(;KVbM*M_NL)}v*#x0uC`{VsEq z{|ovhCOSq=#n9+Ow(h*Ney!u%Zn|Te_jkabg3WK~?}UuEifJv^o*@;_H07$^dvsSb zF00W)JRSy>_*GdUz;%^NLDwJsUIXs;*Ydjn#$^kY$!nmwLP;?M0A!8waDbNW+vY5V zHDDEUF8>gizSw;O-D%l4Jk?}8=f7A5hRIXJ&C_Q7O~EbRVHZO5_204>hAGdk5O}z6 za{Dzmocfw0r7t+>ABxt+`aup(SUM;omwBmpxv@I zfyz&iUlRIH6*BHb)oOZHrZEey_^QKE_}PzhUSYLadHdQ^+iDx8yvz?eIrnXM!OhQe zu0NX_dHENjKwbwp2@riQL z;%l|}%)Y4<{h2;5om8Ds4yJ65|4Qayc>6m~GfP>;F6Q!9=JGr}YOj}HV0 z9o^z{&n3V;X!$*odveE-A1*uc?4(A76ry}D@(9P*w2X!!fX=CZZ&QFv`km;%D-B>v z!!yYk4YUv*F!CRj)N#Dj-j{>Jz5DiTyw_CbP<$zVL7D~TZ{!6($KHgo-PB@d(!6dm zZSX&>b_ot(LCKCHeJ36G@5V0&c{MQyguVGoA5f$k-=ePZ0VTjHPsH4e4r{X8>xLX})F`Q5CzSA7>RnZd6QWJ5if0tahbMV3kB zUB{cP)AcqWYFTnKBzbxvx09p9&)<}9cE5@mCyV4>I~=|hiDJIJo#E+z#Xv#lR;BYa z-Bt=5#~e%@b?!YH{DkG^opcCika(1NFGB)58&wRmHm6%?Q`uK~d&#$D??|FG`gj1%m`cmYCy!rt2Pi6O*Y~=Hw z6$JGahF++Mzf@POF^g2Ni>&$D3Br=r&CaEcYN4}1jRpu(d!vB1B_C7gJG036#9#KC zSmODzkTJ)NlGAs?{}jFT{;PD&?>u5Dq$9teD#2WKFQ_XJJvPn!FED{3yS{{8gA3(8 zuze)wo%FXUirM0cx*Ua)6q*7>bOm#iQ;JDF+q;<82 zlj{b5Wk=T^BViyUDc$Klrfi-u}CJm z`LlAJ(gi_1rC~ZvU;C}R&QVGqt2gA5|0C!4o6RSi34_a~OR>t4&m#Ck=z@vx>*%aS zY#q*9x6kO6Y6b#5xmOQ|eu zc9s;0jEt5$PABA}zUA1wE*9RI#M7K*Thsa8Qk>u+HVpmhawC`Wo>iwogN4d*19(qg zx^|#ghf~F#BS9R~G|^TcZJ=(AMMD_8JGu7-^@(Z5yhyVhjV=Fv)G%$|MG`k!?W0a@_x(QyF+~?Gnh<=i)1^mfFT7Z{SV-OOH-25T{ zwkDNo4)FPlJwU<9_y}Mox@);Cevs_Cr{IVAoAq7iF95pK<5&_P;fSJ;!HWRDO(D5l zRzBJ(A}L|W!o4vEmS*I&cH*0+tPV-u#Oi(ao9UvDdk)ge_K?8>=a$Lyp}`j5u%6LW zG9>fhvE;Af{Uw*OAi%FnBKucq_8@g(DowVoS~}e;V`HPf(#F~EKlAiR#LzMEKKx8T zs`%N2oGq;Zrs31?;eHqOs#3IYzx1lCLho&ol^;8v%5`|3f~l%IcETL3M17Z0M0*Lc zt5wLto?JoJM0uuRex|YJ?&@u=K%UKT^QT6*fE1Fi?t0^&`%6q*^oEhG4Kx(%;SzPZ zz4M+FQoprCBo8pK^Vw2?W|3#*%_heRjg?X@m&WU#Hqr$}P#U!{z?baRr}up+?f)d5 zL_6uTPSNd})v|V7T#!6|1q_^+6kn8h7wkkBqZWQSdq=C%*tMIUQJT*1ww_)kB4aP3 zq+Q=&&$8s;oO8SZ#=H0oOlsv^iq;yC|9Z!{)fw8SB)?iCT=+x_Pm_X%OKYr$(^6f- zFu=Q}1^eOI*Cl%u1zZJ1Ys3&%C(*C0f@@~dFNqS?x+dU{>Z-;Y&=6E2$)h7P1?q6I zRGuS9DZy_vliHd0%LLU~EZAG0*=M=eR+eRb@8Si7qyo!H1$@i(+DPD;+*Wb%pLpt$Jfq9s!63;;6nO5^j*97Cx z{KRe6mZVb*r?iseBsAKqPzg|ypZbB|W#@IC~22WOz-zX zU63rI4P-zOJiPjxXCs4qR_wO3!7|O@Je;S*Z@)Cn=3x}3+Sic#s@Nn~Q+P{5QTA)X zB(|FOoVi4xQEm0>e8Y{R^Zb$aHmTxVukxbZ#v#dkI^;fk;oXJu%lh}t%aag+^V5TF zhL)>T(JkJ)1j!f+$v-JAryJN2@t@K%T=S7c`_xfJk2$TQUUTb4TzD0OR=*TtrduL0 zm#8&-GPl{}muS?nDVCvZT~KJ>ab&t#li#@+B_?yj7Kdh)s8B;jNf z#}r$tvsiI@1rp<4Vb+tzbui_5DZ`h9)|13S+C*{$V=RtZafRp3H3CKNg?9<%`W&`? z9=Vwg--v46ZqT+^_P&%Ph`rM#?Sy7E&5_hbgcKuKUpm8SXrmh{n<|uYIGK7$?=Ag6 zKshbKo~r?O(g#?^W2wl&VLLzt)nk+8N+s>&unEzvx2SVFLm?q(exTZLwFxy1U@JhS zl>ZvduKbh9RFPRvnnixJ)J!*#xmARUyC@qLtRuVyr%z&TBP}bhi8cj_lDn&FT03)S z8FARgzgmq=5$7q}OjvY3)ce4O-K~QyS*hIn8B0-4!^e&T?@j*igt_u=dA?QS+d$`3 z)BaCxZy41}O_ef-csIlD`Rsan1116&9WIqqZWm6GFJnaA>*?Mv4_jlLyG0nF*y$ap_S_uY;y%^yJ=PUefQ?_`aP8~j4?5Lj=Oi68Hu=42$LIF74#5L75 zsGR2gX=;%RwUzk{jbbr~~{78}>Fhk3*@{lo&!Xj0q|leU?a8+uG`i>vPkFAnyCo%Y;@L(w}G3FQM4l zTXgS2((=%QJIsGE>BjGRz7pwH@_oqWS2UKgnpas1dF?N(XKfH0BGC#$HH3Mw?rOMJ zv!ApPh1;+}DY!f>hCuu>b8RFNH{hZEO=gMgx_3EURs4<@;GN}-C{f)a(o%A zPC)03a_?V=q5DBGoZKf;Y8q#<3}Q)@aB*o9cW9~nEv+G}i?GS_+YOrnWSZ ztER*jQPw4Xk6=2{CiZMzDHO6ZXnKSC#GA~Lu)rq`&He|{B~09pLf!fp^7$Cpj8z&X zZD*wCB{cQ9hHa;eJ{5{FU!fHnTG>0+RDh9x#&+g*BZ!ehe->A!u4J^v8+%@X9_>AB zI@yPYZZw{~8$*>*CfnI}Qqt8S@p-WCa*ltYJ@e-5{Yng$Jk~~Pgxi>KM`yBMc3H?? z4^JyuI_)m1(YhIU1Hp#zs4)^lDljCxMuP1I|8SYwkY6mdYg-Yrp)~d+&2KE+m0*fg z4sYd#XV~*>9<|<8?G`}{ zKay68ens}Wu;<6`FnHP7<;D7Of%|W(yL43BwDtPE$&pSk1&xQh`%zp=`!vJr6OXRg zOI+73Qu>i3G~C!!@{m6NK>_^zip2-i4(|$U`JUezpFnf^XD125M-m@x?~G z>l&bCN2cT#BZ~^sN&T$XWW0vm$c zku#p|(9tD8xQ_X9d7}&5~x0xEk+C4{tYea*6 zC%V~YML`;REj-!)v?1crouVcAkO#oRx45q^%x3j$Z8503tDIRVJzOL_Xz`j(%dK)| zsS}Vs4RIPg-6zC5i7y@k<2(hVKHJg_`SIiXzqwl9-bN1{lI$Ph>%ezrX z!%^L_2WFnyNkGDE0|Rhx=didP3p&>U?*|qdTJxM+a#db175aE3Bb5;^GU5vG#aqqCwrMl z-zg`Zb#eF&ZlNV^IH^U_Stg-f#xY~84?@G3gVf+=BP|pQTmvbr)G`dVnQ^;UaAWC& zrloS>TvR}{?R3lKTCB%Zt@TD#jmOsf6_M@rgI+P3lM1MYoFEQprw&5ma|SP4l5{orf82-cN9*_UzD}(BjmadOn|yljo+tSHS=0)ok#QlfIG653fEY1XsAhO!> zxjqbMeXn&fDY*DeeYG~}lq+I>b{7gPJ+H9eC~is5`%-JhoS}#Ht4>#~hvQ4exBkUu zH#5G-?kRO1hFK{Vonx&9XwvANL3Ry^L&koT72Z1Wga%wRR7NaQo3!Ph_z1 z#hB_hJaM1`AEMUql?3Xy)R6BFdPWvoYIj4s`J3*Zdpw2F<$bH2fDM|ayXD04_ZKW# znF%zXOVk^lQGk&{Uto^F8(GXb_0Fa-E@ew4wUcI8gvT`(<7sgo3#ogEL>_)!=(BdiO3qX@enz-%NB?NypV?|wf^cxrYIP8@U8$X=70%TiJJ&o= zP=FbKHMBA9X7}(bb{C_?R*8B!{})#TJ($h{mFD^;6_!aD?-@<;ign|*adzjR(0opN z?Kk8t(bgM=MpTWu{M@#D52!MY*4#Hfe0KmffgLFz68jt5RAlpN)$CeCOQW_154;0O zm2|(eDHh*R?NyEBPb(fsoFSPWYRxD8{Cqd7W+c~9{QQg6h@G<9@^FIORZQ&5>t^xM zc2WvbvlDt9H#@(W;YZ+jMm%OA&ijqIO+(nmmXdA7(JCqzMB!gz-+Of z343te3#3y^AL;{VJ7&E2-h8&y=<4g!)36s98uBW{Q@aHzRu)4)sNd51#jsJE=}F*3 zUGZ39Pi zY}+7r*<{c9W=j1E? z+sm_M7iN3)HC+Frnd|k3qc{aUJ3sEY9cH9zeRe@Um-2*QQyJ%418t^x-6%z~r`|GC z$ITP-743K@nskSp#!M+l*1XECPcd3?{X_=AgDCQpt%P@@W4zAIP680Npg8+!IuIO%zzqX-%< z({etI7)=S{B0)wX7rQJN@nSV*YttQ)x;EUrxwFvLu;pPR$hn-q>1o%FUk-C(DQWk5 zdC8BX`qv!cQGC~>qvFcBz1s)~g|1zIrS)|chWqEwE8b?F^5I-t!-zMFK2(M^Zu@aQ zd^R)j><_yUouX0JN6Npje+dmHz{2CCp~ec-2>UB2e6S*y!gWNkAI9LT13MYrfLkda}hQD{%hra zxi-J7*vvF>5xNDEfWCkqHoT?qFbl~`-)22AJgs*VH3I?uyVxX;?OL~ulHs~*3lh~< z2a=5~wa4Xw3#a8ZLsB;KQwdz1(B*-l2I0s@1)!HjgjhigC$a)mc($w&HFR%(C3Zu} z<;>B}Zk+ih7+Uk>j>M#bRy`wmXilu}rT?uwAp{o0UOg?jlAOThCuxuo{BUwswugrw zloFP-@@ej><^E&KvorTia92a5Td(|SZ^xxj8^mF4uib@UtGkI81~aG$FXUT^(fK9p zcJQI`t>Aj&cO0j*jjot1sXaASHDY7FjGaRLG$$=b(YVLt{aBQ(00gKjnGa|gydsU| zZQw1LXKLAc9jX=s#IELtlSWcdPl}yUN`5H_$B~MO$`{#!_FwDfl zH)6)mLk_8Y7E%cl0wM{S2oV7XiR`{78x$n7OAntL6{~=5BVytpS-21^QL@evvCkk` z7JTYp5##q{tm)zSBVreZna}X^%*!+LGd}2#W#V{ry!x{A^T}XI4bWfh(RRqF_JEkM zTyf2q@$l!pAZN57$tGIB;czcG5){s=dcu(r*e1$4rVf(D#W%vh&wCj%;U!`$N;W7$ zHu#>bDlJ@f^uLV&A!Nc^#8`yPcT`NuFjE913y+^i9D?Ttorr)OB1WR-At!SCkZT^P6WcNXFoSPkhH0(7qS((a5*&aFZ z^*Op3uWlj7Yt7nDobJtO&YpQKsKidbJ}IYj4*P)TAJw=XS90z^xNJ9Ra|=n~bRVpG zIc)yS)cay@4;j$f@EKWL3bLPeJV%DWe6c~%iIi;M9B zSjMGn`0d>@*?lC){DwNHkA@aJK3>t+Sn#dr{;UDMu6_=tu}Ms~X?MKz_zyV{8B;xR zZ}PoBobOm7WUOi7_@iRzhMCSF*(V|L{vuzpKjNE|&i9}U%2~! zc&-m*b{XND>EZPwVm2dUNrssiAX#pFBQ|^^9Q-_VsBVi$q=&yS%;dnQ_7e$A4<|Lu zj1iCQBe)M+gr*9*Mu|OHpo>CaEa+R2Gb9tZGF%AAR=nA@MQg`MN(%xuA1eT3@(RW) z%XMz^&-pp+%$&D+G3hm3v@XeB9_9T!A*?)3Bo7`}3a0pDA66h))Zb6#Cj#~t0Ym4) z`#(&_{~9xqVdkvizb5*(IsLmbBY|Xj@u|OwKMO6g)2y{}{%1yz%gR=8lH7v1DLr)DB$VP&m7qEWHNn(JC_oz( zJA-Z`VvEoy)G(6)G)4%LrNYmH4*?N`O!$cy3zJcdia`bPGz3eSOn5|W&@ht;KMy5j z;=72k0NG?l_BR^$EWrZ3CsxpVwm|BpBf4pf@G{2;q8W*lpt9${5;qY`7jaGwD8>#v4JS@ z|Cx5B;-m8QH3N)L=8D?kv890zPmmUZFYp@l^ou!WMHPKuTk$c zlJ*7`hq1prjLHeEy*D8$uwQaMk*whw=AGpzyGS*Ppc6~S`iAx45e%WG_Z&243L2vW z$kAp3(>^*}?9IeD~h2d^*0QcFzPJzbUJ)t0auJ~E6 zM0Wt=ir&SvbLDE;ZOBY1$tsMmjCHq&1HM6n$O4!v%hZ3=1t>m1g)TfQ_Vm9SsJ94e zMmTunA0#n^0vgWg3y>Q3e+0DuWcYWP(B)Lmmu4OS%H)TZ8Y^T+xJFi(X;86+3OEcn zGCtLZj()ueN&ENO7>a1s>+@Omtb`OTg1V}^Dt^c^YfcL53B00 zuqAwiUMS2$eI3!Gge6iR_2;*QAJrL(!YS{1XWWjg5+X$Xnw|SeYL?tLSo}ZoIDB}n zK)kOjj45E8z-1IX*^Dp=b=^0SVxv9QCu(s z{~GSZjlH?%B$#rTptLe?${dFAcR};EJO4J$;|=)Y$8TeOM1QETQXlbkF8gD><%iXq zNfX8U;maly5*~I!V6}t!tN3S5;S~;MnEzm%da|37ll}uPTc>-?_3p=#cK%xR1zmh7 zjXfvNhGpS?K>Ni1ABoA$4)V{q3JT%pR^oY00~aACLv{T(2ehKVxqfw5WWAHZsIV~F zd*g(6dH*AEctXW027TG`nE;^$(dkZi#~r2xFJzq?JVbCiPFCarz7AmW?%1hM555yR zD?1o}Q5j?YhqS=_C%OVP`bf&EJF?C{t`8p|X}BYKeBl`CSyH4Y_p2xjW3*61D)B-l zd_+{giKu=TQS}r7hdrYAE|hNwli8((Z>ENGjfz3B+Wx-`jQ~FnJ0wbojAG<(fTK6e z+!T-Gx_|!O2!KLW^Lq2~x*Lq|BJj_0Ho2H^yM1(#yp)#znY+qoc2OP|5?sGHA+-bS zFP|Kxrvyin#XVlJssu8`)qd#kq#y>UNXxa-vF4Q+|a2Z+(Z)yr4w}X%p;_5x> zlf(cA^>sUq@yX9>84484krY|^kK{}0-wCD^Sbk7b&0GH_3ks#9V!Vc#!iJf?hM9ktOJZYyBMdVU z@To&Z0@K3NBPp0Cq3%JpKU3W1A(^+Mb|CtK;?6_E5%5cy5^cX?!{UMEZbuJsA>j3B z{q)l?tTp<=02JK>Cz-(0(hBaHig8;p!FOQAzK7rI1|Ura_|>X_ytHK379e@Z zy*`G2_k(tcLj-I6#QJJ3Obep2))zg!1WB@3b{JVvnsy|9flMbny$&;-hkOvo_=f@i zySF&_&ALa{`J`nzN_;Oa*`c1mr0)86T@mcw)iYhy|NYwEzvVBD&fo$8m;%XX_L+!hVzzM@-; z7ahwcJZM6)wwm&s6fk*aW0~gJna^xc#|ba?fz53&d2C{|EAxrtYI28NK*ft3icqo4 z7w7?7tC-yu0a`dD-&3C9t{$B5cP9rg5;XAH+wc37k?G!bg1_9QF9$8*ED{q1HnTe; z|A98>T7n0gd{Dak@zMnea8_laYSExGns}&m$)m1I$qh~JWFJSn*s!jPn+=3DKipqw z1`|zRcdl@=@i_t9`dLUKx3>$%qPQ-ib;hDDjTs-SGJ+a2W}!!%MLVJ2j-pz{_O{Iq zwklN_XOuhz=0}E;1-nGBtB5uG#^{jf9&y}alwRAMsp-OU)|4LR}s^n45C-dp!d zz5H(K$m4E0XHjdd7b~J?$D1w`7a)-|P5>DZ%SEsQK;>pruStYS1bVN_%MSq$FZn~X zUTMAZV7;uG*(pn&SsU$77ZuS8T=?(oS;hpS@$w%hVpcM075E=gnpB#)G<(nb!-bt1 zXL^-ok@5409(gec?O)Uwcn|HkS-KdOv zx<)ADfCdSF@&$hd0ZKseMZ#sSb+c!~0Gap4IbIZVfrlW#H0a$w3B_`nFkbhoUnp~CB;1>+!-V#mQZ4mgPoBS2 zQ(;<-RS11qX(hs}D~|sEM_~P5@y~juc+fr65xD+%Wn9b6IRPb(W5}i1CqvguT`mOn ztk|xw1|X*;PrDVO72J$zkVt!t5CJ}KOKbD7V>0Yx;1Zb67K5G<5@?Y27(IwT2KA3~ z{^~L6g(^5Scl4Mw@)uEkr#JQ=5|c8Gp-Kw!NlKgwC<1@zeu!Zv(qBQY9F>`q`Qa8O z7i8Zgd5Ead1;UOyQ92EhdVyfHkRuTM|7>q`P{#MWN#G#`;9vrHln6dV1g!p0JNv*! z`sF2_wmzP~D)v3MF)ZYhzGhK+-{%}P&>##G{FFJ-ND@zy5g8sEpxXP>*C%i5=EWnY(k z`ktHar-yEOCqW|+_A7%<2RW@Yt=;)|QE=Jj)#v}-Xx~?Oc_4Q z{?`ukHgr9GQ2$-saanEs9dDO@-#q&kn|00X>G2Uhfu&ow-wO5#G?mji(ye(9bAagI z`kLA>!&XD|J`zXZ0RxEUHNl&XY5ePmt)H)T$M3pY`R=6K z{mq-!Y@YFG*WA_ecJFtEO27Y6a;fS;alQSC@AJ|o9~HSZJq&ZqEU+mX+!udn$r z|84bkyNW!Ef3Egg3Z>$~Hb2&HumAg#_4vD;Dc{%L|Nm{h|L*(if2AQNIGl2`|s~Xp*fgrQ(eOOP@QZ~3 E0M0#UP5=M^ literal 0 HcmV?d00001 diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index a48554e6c..5d44bf2f5 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -68,4 +68,9 @@ lazy_static! { "Meilisearch Task Queue Latency in Seconds", ) .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_TOTAL_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_task_queue_total_size", + "Meilisearch Task Queue Total Size", + )) + .expect("Can't create a metric"); } diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index 6e93284c2..b9cd7c8eb 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -169,6 +169,7 @@ pub async fn get_metrics( .map(|task| (OffsetDateTime::now_utc() - task.enqueued_at).as_seconds_f64()) .unwrap_or(0.0); crate::metrics::MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS.set(task_queue_latency_seconds); + crate::metrics::MEILISEARCH_TASK_QUEUE_TOTAL_SIZE.set(index_scheduler.size()? as i64); let encoder = TextEncoder::new(); let mut buffer = vec![]; From 425ef1b2053b6fcee2d0f8cd9deba101c3cc40a5 Mon Sep 17 00:00:00 2001 From: DanasFi Date: Mon, 14 Apr 2025 03:47:30 -0400 Subject: [PATCH 2/6] Added task queue used size metric --- crates/meilisearch/src/metrics.rs | 5 +++++ crates/meilisearch/src/routes/metrics.rs | 1 + 2 files changed, 6 insertions(+) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 5d44bf2f5..87d14b711 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -73,4 +73,9 @@ lazy_static! { "Meilisearch Task Queue Total Size", )) .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_USED_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_task_queue_used_size", + "Meilisearch Task Queue Used Size" + )) + .expect("Can't create a metric"); } diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index b9cd7c8eb..c7bd08935 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -170,6 +170,7 @@ pub async fn get_metrics( .unwrap_or(0.0); crate::metrics::MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS.set(task_queue_latency_seconds); crate::metrics::MEILISEARCH_TASK_QUEUE_TOTAL_SIZE.set(index_scheduler.size()? as i64); + crate::metrics::MEILISEARCH_TASK_QUEUE_USED_SIZE.set(index_scheduler.used_size()? as i64); let encoder = TextEncoder::new(); let mut buffer = vec![]; From 3d93efc6aa7fc53d3d2c4892544a9c4b271bf300 Mon Sep 17 00:00:00 2001 From: DanasFi Date: Mon, 14 Apr 2025 18:00:06 -0400 Subject: [PATCH 3/6] Added metric to check task queue size until stop --- crates/index-scheduler/src/lib.rs | 11 ++++++++++- crates/meilisearch/src/metrics.rs | 6 ++++++ crates/meilisearch/src/routes/metrics.rs | 3 +++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 7fcc42593..b4cda9bea 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -74,6 +74,8 @@ use crate::utils::clamp_to_page_size; pub(crate) type BEI128 = I128; +const TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT: u64 = 40; + #[derive(Debug)] pub struct IndexSchedulerOptions { /// The path to the version file of Meilisearch. @@ -425,6 +427,12 @@ impl IndexScheduler { Ok(self.env.non_free_pages_size()?) } + /// Return the max size of task allowed until the task queue stop receiving. + pub fn remaining_size_until_task_queu_stop(&self) -> Result { + Ok((self.env.info().map_size as u64 * TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT / 100) + .saturating_sub(self.used_size()?)) + } + /// Return the index corresponding to the name. /// /// * If the index wasn't opened before, the index will be opened. @@ -627,7 +635,8 @@ impl IndexScheduler { ) -> Result { // if the task doesn't delete or cancel anything and 40% of the task queue is full, we must refuse to enqueue the incoming task if !matches!(&kind, KindWithContent::TaskDeletion { tasks, .. } | KindWithContent::TaskCancelation { tasks, .. } if !tasks.is_empty()) - && (self.env.non_free_pages_size()? * 100) / self.env.info().map_size as u64 > 40 + && (self.env.non_free_pages_size()? * 100) / self.env.info().map_size as u64 + > TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT { return Err(Error::NoSpaceLeftInTaskQueue); } diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 87d14b711..4ef7803e5 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -78,4 +78,10 @@ lazy_static! { "Meilisearch Task Queue Used Size" )) .expect("Can't create a metric"); + pub static ref MEILISEARCH_TASK_QUEUE_SIZE_UNTIL_STOP_REGISTERING: IntGauge = + register_int_gauge!(opts!( + "meilisearch_task_queue_size_until_stop_registering", + "Meilisearch Task Queue Size Until Stop Registering", + )) + .expect("Can't create a metric"); } diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index c7bd08935..9bb1e3df9 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -172,6 +172,9 @@ pub async fn get_metrics( crate::metrics::MEILISEARCH_TASK_QUEUE_TOTAL_SIZE.set(index_scheduler.size()? as i64); crate::metrics::MEILISEARCH_TASK_QUEUE_USED_SIZE.set(index_scheduler.used_size()? as i64); + crate::metrics::MEILISEARCH_TASK_QUEUE_SIZE_UNTIL_STOP_REGISTERING + .set(index_scheduler.remaining_size_until_task_queu_stop()? as i64); + let encoder = TextEncoder::new(); let mut buffer = vec![]; encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); From eeb33b913c915416803228b6dfc8aee0f7b88fcb Mon Sep 17 00:00:00 2001 From: DanasFi Date: Mon, 14 Apr 2025 18:01:06 -0400 Subject: [PATCH 4/6] Corrected metric for task queue total size --- crates/index-scheduler/src/lib.rs | 5 +++++ crates/meilisearch/src/metrics.rs | 6 +++--- crates/meilisearch/src/routes/metrics.rs | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index b4cda9bea..7b1eea021 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -427,6 +427,11 @@ impl IndexScheduler { Ok(self.env.non_free_pages_size()?) } + /// Return the maximum possible database size + pub fn max_size(&self) -> Result { + Ok(self.env.info().map_size as u64) + } + /// Return the max size of task allowed until the task queue stop receiving. pub fn remaining_size_until_task_queu_stop(&self) -> Result { Ok((self.env.info().map_size as u64 * TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT / 100) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 4ef7803e5..29c1aeae8 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -68,9 +68,9 @@ lazy_static! { "Meilisearch Task Queue Latency in Seconds", ) .expect("Can't create a metric"); - pub static ref MEILISEARCH_TASK_QUEUE_TOTAL_SIZE: IntGauge = register_int_gauge!(opts!( - "meilisearch_task_queue_total_size", - "Meilisearch Task Queue Total Size", + pub static ref MEILISEARCH_TASK_QUEUE_MAX_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_task_queue_max_size", + "Meilisearch Task Queue Max Size", )) .expect("Can't create a metric"); pub static ref MEILISEARCH_TASK_QUEUE_USED_SIZE: IntGauge = register_int_gauge!(opts!( diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index 9bb1e3df9..250363ac3 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -169,7 +169,7 @@ pub async fn get_metrics( .map(|task| (OffsetDateTime::now_utc() - task.enqueued_at).as_seconds_f64()) .unwrap_or(0.0); crate::metrics::MEILISEARCH_TASK_QUEUE_LATENCY_SECONDS.set(task_queue_latency_seconds); - crate::metrics::MEILISEARCH_TASK_QUEUE_TOTAL_SIZE.set(index_scheduler.size()? as i64); + crate::metrics::MEILISEARCH_TASK_QUEUE_MAX_SIZE.set(index_scheduler.max_size()? as i64); crate::metrics::MEILISEARCH_TASK_QUEUE_USED_SIZE.set(index_scheduler.used_size()? as i64); crate::metrics::MEILISEARCH_TASK_QUEUE_SIZE_UNTIL_STOP_REGISTERING From fd0623c085602e849804588b01042ca5a7f2a158 Mon Sep 17 00:00:00 2001 From: DanasFi Date: Wed, 16 Apr 2025 10:30:28 -0400 Subject: [PATCH 5/6] Fix typo in function to get size until task queue stops --- crates/index-scheduler/src/lib.rs | 2 +- crates/meilisearch/db.snapshot | Bin 19430 -> 0 bytes crates/meilisearch/src/routes/metrics.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 crates/meilisearch/db.snapshot diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 7b1eea021..a5afdcbf9 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -433,7 +433,7 @@ impl IndexScheduler { } /// Return the max size of task allowed until the task queue stop receiving. - pub fn remaining_size_until_task_queu_stop(&self) -> Result { + pub fn remaining_size_until_task_queue_stop(&self) -> Result { Ok((self.env.info().map_size as u64 * TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT / 100) .saturating_sub(self.used_size()?)) } diff --git a/crates/meilisearch/db.snapshot b/crates/meilisearch/db.snapshot deleted file mode 100644 index e09c484c59b583e56416552c6306f7c20bd185d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19430 zcmb@t1ymee(=Lh=AV35QnxMfU4DJa|fCP65!QI^h1P=~_ySuvtcXtWy?lyggZodUYRx&l!{F&z%A=p9!Yzk}xOrzs6xs-FIt>4vY^&+S5 z-Tj86!X^z6({UV;pWpaag6GJsxSbs@-U%M`E|K*eFYkb@LZ8^qQG(DB_iYmuaf3ZN zOXR9!s2qp6_o-4>6#sBQVo&26+!HfRh$|!GMYGXR%(tRK=0>NuconJU2md>*U%3w+ z@KKC_j~SWC`Qui<{%T#OKvKpsj8BOUR(72XD4*puTNzNPu(anq=OHHi4Ls>7ClcH#OMusut2t zOEz9@y$2Q=BD}t*=Hgd56>2g)9E#3(Z?ni|i$3`5A>F+WgO5pBV5LJAAb2PtAQ*=i zsa`daZrPcsj?CjQIHD=W>&2))GzP|-1^c%pJsi*NO0Gj*o-Na)WUXb_$h*g!6(@ub z#_HHJnVc6fPdnK#$n{^hhlk&%tvv6yCY<-O-m}LX{1br?|$xiudmOT<5Xf+s1)@q891;yQ8$tfwa$pYhY z)=PHNdVu*2u!Ms}iE9n{C(65G*OX&-w#M7DEx`x1B&kYMg+z0$y<8Q=IWwo_9Db>h>L{G|e;sy=Ld zR0_*Fn)E`$z-%O1CE*BimFL-?iUVKgg4>O%`l}M(#THzNrY1jlsJh9~O}RR(8C^UL z5CWG^-1i(_TQd?eRD*YO6>WkKPALxq>omo#GjNvG+{V?ErW}tG3~^lUG)Ma9l_ytS|#xbgfWy{ zqEXOCZBw@#>-H(6QlcwScGaN z8N5?!>gNdT_-SvaRC0|mM-{NE16&KdCd_9gW0zEDf1DYsLl$qBZeA?j*lq!9hpu_X zB$@|9xH>SZ(C;=i!tEjdHo-Pz<%c1Et2ftkaLYH|*-yBXENdp3&xDWNe-<%HLxym_ zYWTh}^yF?yd_fn5wEP8yIccD;D`8DdRgukp6yyqpCe+*qR3ECN7BT9jcXNW~Vdt1h z>u$*w(VPLX${1ff)BB8MjlmC<-J0Q6fn>pDO}4GcoXgQ0E((~_x;Kqc*_VNnMaQnj zMJ2I?SS_me+J%T}f#@;hP6+P(xvEF%(fZ4R_k$hT;ZSXVfPL;R;+AP>`2ip<+9S5XF_c*0WP3aOhPR-$ye6-p(Us;c1o8F+7-299SXkIaW zW&O_Rh%ls?7&p*`JQ8TmwR-uG&N{=tu0YvR>}ZbteL@la!6$|B({_r#r6wSx%Sa5z z_#lX+Y0Q}8)h**wChzHmn>nT68_Vfb-i&kPfB@?rSipLWAN?*?P1c-IHSoZfxU}xa zhR5Zm_g-{nFDtI5ZzScS`jaN_XSH?5_?gtKxZWIjoA1Z6m(>MqRg`k()N^SPPcy-2 z7=&Kq#tcIIc#jlWyn+=^u?;I`=N(%$RPN1O8qTO&U;h!cvqV@l8LK> zNO_CuQF5Es@d|oj9yVo7BTHank0zxFs+|+uCL;m`UPw9WB#ZWI@l|KmI=^nM396T3 zV+h;+@}Hk>p7=uqKg*k)o_+h};25waTvgo+MG^@{%+zd!yb zFE{ukz`Br0@zgd1T+FxVwdKhSYBXXBGE9y65iLT)kz2XDM{sX(9V0u;r0mFsK+b|l zTav%Rxm!SXGirk=BzUH;+Btt8e5NUkW{(<8cywi#r8Giiby>pl!YQrRla#5Iz_37_ zkrDAu(btYKD3SlO2hZIIEmn?Ufx*X&5=MRvG;bY+pH9{#gac`rK6jx7*uAlVv@gA9 z0@87ZBeA71zfuJTlJ9u;$Sf5hw2&Ejx1X>cbOc7#spq$2645UvxiWRX7)AruLn$;a z4G@rQ>2?w~=iMVM=>$VY;JIUdWoA-y#(dS04W&>JJpQoPl*pve9l?NPJ*L1B)4H&p zy{7Cvu5csH8PjUfu3$&!r6h&AsXR7`LL@ojXbnpw$+MuCpa1rKDv}RwtN7FB&Sqoi zNW(Uyj0K6B6>YgtxM02Ebs-w>mzG+xz>Fthyg3~|l{n1l<7-N6AMI_PwaARlV zc4PN}x|QUzWupja$+MDC575A^3%CTx9@~}rC8;w^U+r1T=X`N!b+!DJxtbthV0EzA zt8d+3`B9%qPBq-nsp9z^;hmUAubwdZZU?j6)BZ>dqu!ka$6h0&%{*M1!tYoNQf(Ji z-3hAoMb9V}ew0jaIAvMZC9$x7iguJNd@qe@QJVJV?Y$!#%PO6eV5-OQ0J?8FBeIi5 z2IKN3vr?1;VZjV`*}^6Y>|EpLhONQ%7cn%20A16wYJL${^5bcnA^^KxoH#hOOizXkQEBuyt-HUjGbkRkS9GWfe-&W$mnV(s)MrVa-#2{qFwMvm=G<%**{+ zl6G(tj_Qu;cT3$}>$Fk2kBJppDRV8>>BZai*;}F~zSQ5^WVd2sPLT&aGr%vp=lu3J zWJXndV(q59xs|P@qc>G!E{g65|sX^}N%$^qv|YnW`JPKRKl~ zkz3SW!NQlCdCEHo5KO(hc7(cRXy zZ|HIu3?^(freidGs?4PQf+g?^_I~I|mO74OeH@*Prgh)(kNSMth*)JiTf|d?_gUB)L5TgrHKv1A3N{1uPDnGRn*N|9b(-|nenKjwev0!p zd|NQAjf%9!5XIJRx#9ceHcoFC^svD)N-E1j()V|?5{)}t?c3@Y9J4IQED0{`T(d?S z@-JGsDR7=P%yCAyh9fS5ap-b3CN>iB;jq%QjSEelaM{w-_ZVjn2=)eg)AyGj>=Np` z`C?+nGn(?Na(lon(0jKc?!BjxjXg8y&3;1w?r*n*Io&!YG*R|zJ!NpjL!T~3lq5*v zx0skvHCBSq`qc9Di7H;YEc_VdeQ_mh!@R-g>ajjqfNqRv%4*Sat zu46*)$=AKGXc0g0iJ5~l zxzeK)wO5F&%6s|kBd}ZwoZ4kEa!SEEQflmNHXX+GT@?`>R)yx`qy>iMZF!*uM)pktwsaOLW2T+y9evfhn+{Osv0_uty!fkwVL{Ky~!KFfMw6qMi5qa-jYbyROHO- z$IjOX|2JNm3z)U`lxZj5xK*CS^;+**Q<`anr%g|&sElKk)cL|<9)&pb7{qSYr|at4 zQVDOjUsY_!BplVLCtjl!uff@d#+w6`1;21c znxssPJBn?ufqC7<5lFd)lZK*hQtI<^V5R@ zaY|n!3SLT3#t~$a+6jx8DOJ(ooqxv&)~3+(^o}COJ9*{DpqAa+Th07|J!eY@VOo)C zPj<775u-_?wGgT<=Vad0wUo+Qzih7x4TiqhS++pofDtY>qO@?a5_@?*_h7@yVo7v0o88>wce3!Jjr@8H0HVSq}a$Z5IOh<)Ohzl zx6=0~JS#w$M>rH!(6+<%t_&o9m1tUDotw`xKH?n1(7|-BEll(N$&ah#7th`nK-Cus zS=$D`v7q!B798*$*4MrWeI9_3KAu@37R)fr?JBBO}fgA#0orJSQHV zHWKnLv=3zZT65rE=?pb@FLN#iCGhM_eYopO|yc&@%)`mz5k0+o?j|Xc(58n`1y@{pJ=%Z_QWBt46Khk zF)6_~hAs6;Ss=o6h}(_E*ePPa{`R&mlwA-0U20EymMYt|S2unVvE^;WN~7=d#F_!> z?JqFG3Qe<0u(=XS{H@N4_Ad?h6DF8J5A*+o%3>P`;n!UpxP?#sB7@ zKi0pl`>)iOzA_4+@Li>c$*#=MK@J*_q`~jb?ZwqEJ0XYmkeKVyBx8TlD8cSTkf(Md zItv@UT7r#d;GrjzRYg0gc(@-{m2hgVzVC?KOz+KCUY zk5n=rZ~YaDrtx=g>U_Uj5SSSg8$8;b+z6@3KHZPp>&wDoh>1{Dkud%t8$t1T{hPOc zzCN{J2fXVmQj)@`uPTbI3XDy>l6mJN;ejrBdCC$Jdeqc{65^j;4?R8H7sw)iTVH&p z-YN?X{Ne6y4>ZK;w)%z%P3dAH)Kw&`e;8oA#|w@X)0t6RV$tMCpkY0(SWHSa;OgGp2zs_e6t(EvNh@p5cJTLTlK4N=9bJ-28L| zqnp=NsVH!jh4`1Z9vneNC4j@5ANy|`hw$GCX;DbpDQi?{C{`g(Z#r7kkHj&#={x}E zF`*_mqKQbVk-G;euT8+Y2w+q`8oLcTwYd#Xd(Ko148vIXxee`mmK+TXW2)?b_MxYK z7{>b0@DV?ro2cxEJ+KyNARJ*p{{1i0?u1dS4~p#gKl{*A+m-!-7jloCWdGHPqlO5y zp_NRsJ!ft>_+R&XZDuQH*@ss_+ae6!Qyfz>@p( za#`gs8so+<-q7T`WEAx(OJ!YLDIBCt9RCT3_Y#EafVWx!LOT+{a~v79Pn$QZU|c9k z+zD;y15JL>o~j{8)5~O?#l4lx3*_mPbb65!L4|^Znf^8xxaP?^T3BA5W>8hV;-9Yp z_Tr>B<%_>oI9*vc(*_J8 zk7S6!`{HcmR}}6)YFSQtJ6OW^$nGfc{*_TC4gV2U^Qj4W2zxS%j^{LHkgA1Nabw0n zn!N?P$a;VLYWez8W?$J}Ls~MUhC22I>L#_v04aI$BEX_5BHCOdNfvA0tzlPCN zhCi&?bP%{0ls^k_SG(OT8D9by>{<_o3U|PX$(3Rikm7h8t8n1qercs}0Q6s}DNaI` z9WMLwF9F8r7-(zZ=_~poV<3p%us|QM*m1u20X=0=*U&hf1j%`$k8}2O9n)x&|L&btQ&KN_V3>Lu0`w;ZdqK}z7cTZm=Z-${MM<4an5`TiGL?YlZECxcUoQ`rB) zq{n#xWf&hT?Ypqx_g%WYaVMee+9QbT$<0W+AH}%_sh-1vm@YD8-AWB%?vka&Cu-Dzn) zO^5zisLX5HJphO#Hwu$Cz@3?K5f{LJ-r=7=1OT&P7?+yE7yRhV2LnXC+z~F+w=(}q zEs&4#i&1y?oM&P7w3uu21ho*(hi#hOm4!dFn|j9ar)ZzMdPeZ4&I5>ykYNfS&Xux` zyN(w6*cH9B5nJT`5hNdzfAEm@Qcy$w>NT?_48ITP{*tjQngRWgW-QlK2D>jW@Z7p_ zKtFQh+gBkW8O!XTV0SGo{#&=Vpj$VM#qBGn!80x2IL*x{luf`wJTc<+UCLi+kc?>j zzNeuLVZfZhtKsj2M>9nquBg_Y@&ZR8u8?I{K+XaH$X<_Wl>)E+QlfCNX7e6kxs5~3 zFY#mj@~a+vx57QV+3;YNM*7L4uPjC;9947)udHISZ;{wBS{y9N_cCMZM3$LcS7&zi zKl@C5zZzU2t7$g)+S-?mjC{W~_4?U3{LdrtE4`#(;KVbM*M_NL)}v*#x0uC`{VsEq z{|ovhCOSq=#n9+Ow(h*Ney!u%Zn|Te_jkabg3WK~?}UuEifJv^o*@;_H07$^dvsSb zF00W)JRSy>_*GdUz;%^NLDwJsUIXs;*Ydjn#$^kY$!nmwLP;?M0A!8waDbNW+vY5V zHDDEUF8>gizSw;O-D%l4Jk?}8=f7A5hRIXJ&C_Q7O~EbRVHZO5_204>hAGdk5O}z6 za{Dzmocfw0r7t+>ABxt+`aup(SUM;omwBmpxv@I zfyz&iUlRIH6*BHb)oOZHrZEey_^QKE_}PzhUSYLadHdQ^+iDx8yvz?eIrnXM!OhQe zu0NX_dHENjKwbwp2@riQL z;%l|}%)Y4<{h2;5om8Ds4yJ65|4Qayc>6m~GfP>;F6Q!9=JGr}YOj}HV0 z9o^z{&n3V;X!$*odveE-A1*uc?4(A76ry}D@(9P*w2X!!fX=CZZ&QFv`km;%D-B>v z!!yYk4YUv*F!CRj)N#Dj-j{>Jz5DiTyw_CbP<$zVL7D~TZ{!6($KHgo-PB@d(!6dm zZSX&>b_ot(LCKCHeJ36G@5V0&c{MQyguVGoA5f$k-=ePZ0VTjHPsH4e4r{X8>xLX})F`Q5CzSA7>RnZd6QWJ5if0tahbMV3kB zUB{cP)AcqWYFTnKBzbxvx09p9&)<}9cE5@mCyV4>I~=|hiDJIJo#E+z#Xv#lR;BYa z-Bt=5#~e%@b?!YH{DkG^opcCika(1NFGB)58&wRmHm6%?Q`uK~d&#$D??|FG`gj1%m`cmYCy!rt2Pi6O*Y~=Hw z6$JGahF++Mzf@POF^g2Ni>&$D3Br=r&CaEcYN4}1jRpu(d!vB1B_C7gJG036#9#KC zSmODzkTJ)NlGAs?{}jFT{;PD&?>u5Dq$9teD#2WKFQ_XJJvPn!FED{3yS{{8gA3(8 zuze)wo%FXUirM0cx*Ua)6q*7>bOm#iQ;JDF+q;<82 zlj{b5Wk=T^BViyUDc$Klrfi-u}CJm z`LlAJ(gi_1rC~ZvU;C}R&QVGqt2gA5|0C!4o6RSi34_a~OR>t4&m#Ck=z@vx>*%aS zY#q*9x6kO6Y6b#5xmOQ|eu zc9s;0jEt5$PABA}zUA1wE*9RI#M7K*Thsa8Qk>u+HVpmhawC`Wo>iwogN4d*19(qg zx^|#ghf~F#BS9R~G|^TcZJ=(AMMD_8JGu7-^@(Z5yhyVhjV=Fv)G%$|MG`k!?W0a@_x(QyF+~?Gnh<=i)1^mfFT7Z{SV-OOH-25T{ zwkDNo4)FPlJwU<9_y}Mox@);Cevs_Cr{IVAoAq7iF95pK<5&_P;fSJ;!HWRDO(D5l zRzBJ(A}L|W!o4vEmS*I&cH*0+tPV-u#Oi(ao9UvDdk)ge_K?8>=a$Lyp}`j5u%6LW zG9>fhvE;Af{Uw*OAi%FnBKucq_8@g(DowVoS~}e;V`HPf(#F~EKlAiR#LzMEKKx8T zs`%N2oGq;Zrs31?;eHqOs#3IYzx1lCLho&ol^;8v%5`|3f~l%IcETL3M17Z0M0*Lc zt5wLto?JoJM0uuRex|YJ?&@u=K%UKT^QT6*fE1Fi?t0^&`%6q*^oEhG4Kx(%;SzPZ zz4M+FQoprCBo8pK^Vw2?W|3#*%_heRjg?X@m&WU#Hqr$}P#U!{z?baRr}up+?f)d5 zL_6uTPSNd})v|V7T#!6|1q_^+6kn8h7wkkBqZWQSdq=C%*tMIUQJT*1ww_)kB4aP3 zq+Q=&&$8s;oO8SZ#=H0oOlsv^iq;yC|9Z!{)fw8SB)?iCT=+x_Pm_X%OKYr$(^6f- zFu=Q}1^eOI*Cl%u1zZJ1Ys3&%C(*C0f@@~dFNqS?x+dU{>Z-;Y&=6E2$)h7P1?q6I zRGuS9DZy_vliHd0%LLU~EZAG0*=M=eR+eRb@8Si7qyo!H1$@i(+DPD;+*Wb%pLpt$Jfq9s!63;;6nO5^j*97Cx z{KRe6mZVb*r?iseBsAKqPzg|ypZbB|W#@IC~22WOz-zX zU63rI4P-zOJiPjxXCs4qR_wO3!7|O@Je;S*Z@)Cn=3x}3+Sic#s@Nn~Q+P{5QTA)X zB(|FOoVi4xQEm0>e8Y{R^Zb$aHmTxVukxbZ#v#dkI^;fk;oXJu%lh}t%aag+^V5TF zhL)>T(JkJ)1j!f+$v-JAryJN2@t@K%T=S7c`_xfJk2$TQUUTb4TzD0OR=*TtrduL0 zm#8&-GPl{}muS?nDVCvZT~KJ>ab&t#li#@+B_?yj7Kdh)s8B;jNf z#}r$tvsiI@1rp<4Vb+tzbui_5DZ`h9)|13S+C*{$V=RtZafRp3H3CKNg?9<%`W&`? z9=Vwg--v46ZqT+^_P&%Ph`rM#?Sy7E&5_hbgcKuKUpm8SXrmh{n<|uYIGK7$?=Ag6 zKshbKo~r?O(g#?^W2wl&VLLzt)nk+8N+s>&unEzvx2SVFLm?q(exTZLwFxy1U@JhS zl>ZvduKbh9RFPRvnnixJ)J!*#xmARUyC@qLtRuVyr%z&TBP}bhi8cj_lDn&FT03)S z8FARgzgmq=5$7q}OjvY3)ce4O-K~QyS*hIn8B0-4!^e&T?@j*igt_u=dA?QS+d$`3 z)BaCxZy41}O_ef-csIlD`Rsan1116&9WIqqZWm6GFJnaA>*?Mv4_jlLyG0nF*y$ap_S_uY;y%^yJ=PUefQ?_`aP8~j4?5Lj=Oi68Hu=42$LIF74#5L75 zsGR2gX=;%RwUzk{jbbr~~{78}>Fhk3*@{lo&!Xj0q|leU?a8+uG`i>vPkFAnyCo%Y;@L(w}G3FQM4l zTXgS2((=%QJIsGE>BjGRz7pwH@_oqWS2UKgnpas1dF?N(XKfH0BGC#$HH3Mw?rOMJ zv!ApPh1;+}DY!f>hCuu>b8RFNH{hZEO=gMgx_3EURs4<@;GN}-C{f)a(o%A zPC)03a_?V=q5DBGoZKf;Y8q#<3}Q)@aB*o9cW9~nEv+G}i?GS_+YOrnWSZ ztER*jQPw4Xk6=2{CiZMzDHO6ZXnKSC#GA~Lu)rq`&He|{B~09pLf!fp^7$Cpj8z&X zZD*wCB{cQ9hHa;eJ{5{FU!fHnTG>0+RDh9x#&+g*BZ!ehe->A!u4J^v8+%@X9_>AB zI@yPYZZw{~8$*>*CfnI}Qqt8S@p-WCa*ltYJ@e-5{Yng$Jk~~Pgxi>KM`yBMc3H?? z4^JyuI_)m1(YhIU1Hp#zs4)^lDljCxMuP1I|8SYwkY6mdYg-Yrp)~d+&2KE+m0*fg z4sYd#XV~*>9<|<8?G`}{ zKay68ens}Wu;<6`FnHP7<;D7Of%|W(yL43BwDtPE$&pSk1&xQh`%zp=`!vJr6OXRg zOI+73Qu>i3G~C!!@{m6NK>_^zip2-i4(|$U`JUezpFnf^XD125M-m@x?~G z>l&bCN2cT#BZ~^sN&T$XWW0vm$c zku#p|(9tD8xQ_X9d7}&5~x0xEk+C4{tYea*6 zC%V~YML`;REj-!)v?1crouVcAkO#oRx45q^%x3j$Z8503tDIRVJzOL_Xz`j(%dK)| zsS}Vs4RIPg-6zC5i7y@k<2(hVKHJg_`SIiXzqwl9-bN1{lI$Ph>%ezrX z!%^L_2WFnyNkGDE0|Rhx=didP3p&>U?*|qdTJxM+a#db175aE3Bb5;^GU5vG#aqqCwrMl z-zg`Zb#eF&ZlNV^IH^U_Stg-f#xY~84?@G3gVf+=BP|pQTmvbr)G`dVnQ^;UaAWC& zrloS>TvR}{?R3lKTCB%Zt@TD#jmOsf6_M@rgI+P3lM1MYoFEQprw&5ma|SP4l5{orf82-cN9*_UzD}(BjmadOn|yljo+tSHS=0)ok#QlfIG653fEY1XsAhO!> zxjqbMeXn&fDY*DeeYG~}lq+I>b{7gPJ+H9eC~is5`%-JhoS}#Ht4>#~hvQ4exBkUu zH#5G-?kRO1hFK{Vonx&9XwvANL3Ry^L&koT72Z1Wga%wRR7NaQo3!Ph_z1 z#hB_hJaM1`AEMUql?3Xy)R6BFdPWvoYIj4s`J3*Zdpw2F<$bH2fDM|ayXD04_ZKW# znF%zXOVk^lQGk&{Uto^F8(GXb_0Fa-E@ew4wUcI8gvT`(<7sgo3#ogEL>_)!=(BdiO3qX@enz-%NB?NypV?|wf^cxrYIP8@U8$X=70%TiJJ&o= zP=FbKHMBA9X7}(bb{C_?R*8B!{})#TJ($h{mFD^;6_!aD?-@<;ign|*adzjR(0opN z?Kk8t(bgM=MpTWu{M@#D52!MY*4#Hfe0KmffgLFz68jt5RAlpN)$CeCOQW_154;0O zm2|(eDHh*R?NyEBPb(fsoFSPWYRxD8{Cqd7W+c~9{QQg6h@G<9@^FIORZQ&5>t^xM zc2WvbvlDt9H#@(W;YZ+jMm%OA&ijqIO+(nmmXdA7(JCqzMB!gz-+Of z343te3#3y^AL;{VJ7&E2-h8&y=<4g!)36s98uBW{Q@aHzRu)4)sNd51#jsJE=}F*3 zUGZ39Pi zY}+7r*<{c9W=j1E? z+sm_M7iN3)HC+Frnd|k3qc{aUJ3sEY9cH9zeRe@Um-2*QQyJ%418t^x-6%z~r`|GC z$ITP-743K@nskSp#!M+l*1XECPcd3?{X_=AgDCQpt%P@@W4zAIP680Npg8+!IuIO%zzqX-%< z({etI7)=S{B0)wX7rQJN@nSV*YttQ)x;EUrxwFvLu;pPR$hn-q>1o%FUk-C(DQWk5 zdC8BX`qv!cQGC~>qvFcBz1s)~g|1zIrS)|chWqEwE8b?F^5I-t!-zMFK2(M^Zu@aQ zd^R)j><_yUouX0JN6Npje+dmHz{2CCp~ec-2>UB2e6S*y!gWNkAI9LT13MYrfLkda}hQD{%hra zxi-J7*vvF>5xNDEfWCkqHoT?qFbl~`-)22AJgs*VH3I?uyVxX;?OL~ulHs~*3lh~< z2a=5~wa4Xw3#a8ZLsB;KQwdz1(B*-l2I0s@1)!HjgjhigC$a)mc($w&HFR%(C3Zu} z<;>B}Zk+ih7+Uk>j>M#bRy`wmXilu}rT?uwAp{o0UOg?jlAOThCuxuo{BUwswugrw zloFP-@@ej><^E&KvorTia92a5Td(|SZ^xxj8^mF4uib@UtGkI81~aG$FXUT^(fK9p zcJQI`t>Aj&cO0j*jjot1sXaASHDY7FjGaRLG$$=b(YVLt{aBQ(00gKjnGa|gydsU| zZQw1LXKLAc9jX=s#IELtlSWcdPl}yUN`5H_$B~MO$`{#!_FwDfl zH)6)mLk_8Y7E%cl0wM{S2oV7XiR`{78x$n7OAntL6{~=5BVytpS-21^QL@evvCkk` z7JTYp5##q{tm)zSBVreZna}X^%*!+LGd}2#W#V{ry!x{A^T}XI4bWfh(RRqF_JEkM zTyf2q@$l!pAZN57$tGIB;czcG5){s=dcu(r*e1$4rVf(D#W%vh&wCj%;U!`$N;W7$ zHu#>bDlJ@f^uLV&A!Nc^#8`yPcT`NuFjE913y+^i9D?Ttorr)OB1WR-At!SCkZT^P6WcNXFoSPkhH0(7qS((a5*&aFZ z^*Op3uWlj7Yt7nDobJtO&YpQKsKidbJ}IYj4*P)TAJw=XS90z^xNJ9Ra|=n~bRVpG zIc)yS)cay@4;j$f@EKWL3bLPeJV%DWe6c~%iIi;M9B zSjMGn`0d>@*?lC){DwNHkA@aJK3>t+Sn#dr{;UDMu6_=tu}Ms~X?MKz_zyV{8B;xR zZ}PoBobOm7WUOi7_@iRzhMCSF*(V|L{vuzpKjNE|&i9}U%2~! zc&-m*b{XND>EZPwVm2dUNrssiAX#pFBQ|^^9Q-_VsBVi$q=&yS%;dnQ_7e$A4<|Lu zj1iCQBe)M+gr*9*Mu|OHpo>CaEa+R2Gb9tZGF%AAR=nA@MQg`MN(%xuA1eT3@(RW) z%XMz^&-pp+%$&D+G3hm3v@XeB9_9T!A*?)3Bo7`}3a0pDA66h))Zb6#Cj#~t0Ym4) z`#(&_{~9xqVdkvizb5*(IsLmbBY|Xj@u|OwKMO6g)2y{}{%1yz%gR=8lH7v1DLr)DB$VP&m7qEWHNn(JC_oz( zJA-Z`VvEoy)G(6)G)4%LrNYmH4*?N`O!$cy3zJcdia`bPGz3eSOn5|W&@ht;KMy5j z;=72k0NG?l_BR^$EWrZ3CsxpVwm|BpBf4pf@G{2;q8W*lpt9${5;qY`7jaGwD8>#v4JS@ z|Cx5B;-m8QH3N)L=8D?kv890zPmmUZFYp@l^ou!WMHPKuTk$c zlJ*7`hq1prjLHeEy*D8$uwQaMk*whw=AGpzyGS*Ppc6~S`iAx45e%WG_Z&243L2vW z$kAp3(>^*}?9IeD~h2d^*0QcFzPJzbUJ)t0auJ~E6 zM0Wt=ir&SvbLDE;ZOBY1$tsMmjCHq&1HM6n$O4!v%hZ3=1t>m1g)TfQ_Vm9SsJ94e zMmTunA0#n^0vgWg3y>Q3e+0DuWcYWP(B)Lmmu4OS%H)TZ8Y^T+xJFi(X;86+3OEcn zGCtLZj()ueN&ENO7>a1s>+@Omtb`OTg1V}^Dt^c^YfcL53B00 zuqAwiUMS2$eI3!Gge6iR_2;*QAJrL(!YS{1XWWjg5+X$Xnw|SeYL?tLSo}ZoIDB}n zK)kOjj45E8z-1IX*^Dp=b=^0SVxv9QCu(s z{~GSZjlH?%B$#rTptLe?${dFAcR};EJO4J$;|=)Y$8TeOM1QETQXlbkF8gD><%iXq zNfX8U;maly5*~I!V6}t!tN3S5;S~;MnEzm%da|37ll}uPTc>-?_3p=#cK%xR1zmh7 zjXfvNhGpS?K>Ni1ABoA$4)V{q3JT%pR^oY00~aACLv{T(2ehKVxqfw5WWAHZsIV~F zd*g(6dH*AEctXW027TG`nE;^$(dkZi#~r2xFJzq?JVbCiPFCarz7AmW?%1hM555yR zD?1o}Q5j?YhqS=_C%OVP`bf&EJF?C{t`8p|X}BYKeBl`CSyH4Y_p2xjW3*61D)B-l zd_+{giKu=TQS}r7hdrYAE|hNwli8((Z>ENGjfz3B+Wx-`jQ~FnJ0wbojAG<(fTK6e z+!T-Gx_|!O2!KLW^Lq2~x*Lq|BJj_0Ho2H^yM1(#yp)#znY+qoc2OP|5?sGHA+-bS zFP|Kxrvyin#XVlJssu8`)qd#kq#y>UNXxa-vF4Q+|a2Z+(Z)yr4w}X%p;_5x> zlf(cA^>sUq@yX9>84484krY|^kK{}0-wCD^Sbk7b&0GH_3ks#9V!Vc#!iJf?hM9ktOJZYyBMdVU z@To&Z0@K3NBPp0Cq3%JpKU3W1A(^+Mb|CtK;?6_E5%5cy5^cX?!{UMEZbuJsA>j3B z{q)l?tTp<=02JK>Cz-(0(hBaHig8;p!FOQAzK7rI1|Ura_|>X_ytHK379e@Z zy*`G2_k(tcLj-I6#QJJ3Obep2))zg!1WB@3b{JVvnsy|9flMbny$&;-hkOvo_=f@i zySF&_&ALa{`J`nzN_;Oa*`c1mr0)86T@mcw)iYhy|NYwEzvVBD&fo$8m;%XX_L+!hVzzM@-; z7ahwcJZM6)wwm&s6fk*aW0~gJna^xc#|ba?fz53&d2C{|EAxrtYI28NK*ft3icqo4 z7w7?7tC-yu0a`dD-&3C9t{$B5cP9rg5;XAH+wc37k?G!bg1_9QF9$8*ED{q1HnTe; z|A98>T7n0gd{Dak@zMnea8_laYSExGns}&m$)m1I$qh~JWFJSn*s!jPn+=3DKipqw z1`|zRcdl@=@i_t9`dLUKx3>$%qPQ-ib;hDDjTs-SGJ+a2W}!!%MLVJ2j-pz{_O{Iq zwklN_XOuhz=0}E;1-nGBtB5uG#^{jf9&y}alwRAMsp-OU)|4LR}s^n45C-dp!d zz5H(K$m4E0XHjdd7b~J?$D1w`7a)-|P5>DZ%SEsQK;>pruStYS1bVN_%MSq$FZn~X zUTMAZV7;uG*(pn&SsU$77ZuS8T=?(oS;hpS@$w%hVpcM075E=gnpB#)G<(nb!-bt1 zXL^-ok@5409(gec?O)Uwcn|HkS-KdOv zx<)ADfCdSF@&$hd0ZKseMZ#sSb+c!~0Gap4IbIZVfrlW#H0a$w3B_`nFkbhoUnp~CB;1>+!-V#mQZ4mgPoBS2 zQ(;<-RS11qX(hs}D~|sEM_~P5@y~juc+fr65xD+%Wn9b6IRPb(W5}i1CqvguT`mOn ztk|xw1|X*;PrDVO72J$zkVt!t5CJ}KOKbD7V>0Yx;1Zb67K5G<5@?Y27(IwT2KA3~ z{^~L6g(^5Scl4Mw@)uEkr#JQ=5|c8Gp-Kw!NlKgwC<1@zeu!Zv(qBQY9F>`q`Qa8O z7i8Zgd5Ead1;UOyQ92EhdVyfHkRuTM|7>q`P{#MWN#G#`;9vrHln6dV1g!p0JNv*! z`sF2_wmzP~D)v3MF)ZYhzGhK+-{%}P&>##G{FFJ-ND@zy5g8sEpxXP>*C%i5=EWnY(k z`ktHar-yEOCqW|+_A7%<2RW@Yt=;)|QE=Jj)#v}-Xx~?Oc_4Q z{?`ukHgr9GQ2$-saanEs9dDO@-#q&kn|00X>G2Uhfu&ow-wO5#G?mji(ye(9bAagI z`kLA>!&XD|J`zXZ0RxEUHNl&XY5ePmt)H)T$M3pY`R=6K z{mq-!Y@YFG*WA_ecJFtEO27Y6a;fS;alQSC@AJ|o9~HSZJq&ZqEU+mX+!udn$r z|84bkyNW!Ef3Egg3Z>$~Hb2&HumAg#_4vD;Dc{%L|Nm{h|L*(if2AQNIGl2`|s~Xp*fgrQ(eOOP@QZ~3 E0M0#UP5=M^ diff --git a/crates/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs index 250363ac3..31abb93a7 100644 --- a/crates/meilisearch/src/routes/metrics.rs +++ b/crates/meilisearch/src/routes/metrics.rs @@ -173,7 +173,7 @@ pub async fn get_metrics( crate::metrics::MEILISEARCH_TASK_QUEUE_USED_SIZE.set(index_scheduler.used_size()? as i64); crate::metrics::MEILISEARCH_TASK_QUEUE_SIZE_UNTIL_STOP_REGISTERING - .set(index_scheduler.remaining_size_until_task_queu_stop()? as i64); + .set(index_scheduler.remaining_size_until_task_queue_stop()? as i64); let encoder = TextEncoder::new(); let mut buffer = vec![]; From 2984be880f6a452f186964f3f347154c34302edd Mon Sep 17 00:00:00 2001 From: DanasFi Date: Wed, 16 Apr 2025 19:19:02 -0400 Subject: [PATCH 6/6] Add task queue metrics to grafana dashboard --- assets/grafana-dashboard.json | 294 ++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) diff --git a/assets/grafana-dashboard.json b/assets/grafana-dashboard.json index 027afcbba..4c1afd2c7 100644 --- a/assets/grafana-dashboard.json +++ b/assets/grafana-dashboard.json @@ -1501,6 +1501,300 @@ "title": "Task queue latency", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_used_size{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue used size in bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_size_until_stop_registering{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue available size until it stop receiving tasks.", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 29, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_task_queue_max_size{instance=\"$instance\", job=\"$job\"}", + "interval": "", + "legendFormat": "{{value}} ", + "range": true, + "refId": "A" + } + ], + "title": "Task queue maximum possible size", + "type": "stat" + }, { "collapsed": true, "datasource": {