From 93a4fcd32c2b58a116498ac9eb31752bf07bae71 Mon Sep 17 00:00:00 2001 From: Lorina Poland Date: Mon, 6 Nov 2023 13:51:18 -0800 Subject: [PATCH] Rewrite compaction documentation, adding UCS Adds some SAI/Vector doc fixes too. patch by Lorina Poland, Branimir Lambov; reviewed by Mick Semb Wever for CASSANDRA-18232 --- doc/modules/ROOT/pages/index.adoc | 10 +- .../assets/images/unified/sharding.png | Bin 0 -> 40953 bytes .../images/unified/shards_graph_lambda_0.svg | 2180 ++++++++++++++++ .../unified/shards_graph_lambda_0_33.svg | 2203 ++++++++++++++++ .../unified/shards_graph_lambda_0_5.svg | 2238 +++++++++++++++++ .../images/unified/shards_graph_lambda_1.svg | 1943 ++++++++++++++ .../cassandra/assets/images/unified/ucs.png | Bin 0 -> 46473 bytes doc/modules/cassandra/nav.adoc | 32 +- .../pages/architecture/overview.adoc | 99 +- .../pages/architecture/storage-engine.adoc | 219 +- .../sai/_user-defined-type-match.adoc | 2 +- .../cql/indexing/sai/sai-query.adoc | 2 + .../managing/operating/compaction/index.adoc | 341 +-- .../managing/operating/compaction/lcs.adoc | 142 +- .../operating/compaction/overview.adoc | 235 ++ .../managing/operating/compaction/stcs.adoc | 118 +- .../operating/compaction/tombstones.adoc | 129 + .../managing/operating/compaction/twcs.adoc | 153 +- .../managing/operating/compaction/ucs.adoc | 626 +++++ .../pages/managing/operating/index.adoc | 23 +- .../pages/managing/operating/logging.adoc | 5 + doc/modules/cassandra/pages/new/index.adoc | 37 +- .../cql-commands/compact-subproperties.adoc | 211 +- .../pages/reference/vector-data-type.adoc | 4 + .../partials/compaction-strategies.adoc | 4 +- .../partials/default-compaction-strategy.adoc | 5 + .../partials/sai/sai-only-select.adoc | 4 + .../cassandra/partials/ucs-recommend.adoc | 5 + 28 files changed, 10191 insertions(+), 779 deletions(-) create mode 100644 doc/modules/cassandra/assets/images/unified/sharding.png create mode 100644 doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0.svg create mode 100644 doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_33.svg create mode 100644 doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_5.svg create mode 100644 doc/modules/cassandra/assets/images/unified/shards_graph_lambda_1.svg create mode 100644 doc/modules/cassandra/assets/images/unified/ucs.png create mode 100644 doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc create mode 100644 doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc create mode 100644 doc/modules/cassandra/pages/managing/operating/compaction/ucs.adoc create mode 100644 doc/modules/cassandra/pages/managing/operating/logging.adoc create mode 100644 doc/modules/cassandra/partials/default-compaction-strategy.adoc create mode 100644 doc/modules/cassandra/partials/sai/sai-only-select.adoc create mode 100644 doc/modules/cassandra/partials/ucs-recommend.adoc diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index 2533479169..02b3f949ea 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -16,18 +16,24 @@ If you would like to contribute to this documentation, you are welcome to do so | xref:cassandra:new/index.adoc[What's new in 5.0] | What's new in Cassandra 5.0 +| xref:cassandra:overview/index.adoc[Overview] | Cassandra's features + | xref:cassandra:architecture/index.adoc[Architecture] | Cassandra's big picture | xref:cassandra:developing/data-modeling/index.adoc[Data modeling] | Hint: it's not relational | xref:cassandra:developing/cql/index.adoc[Cassandra Query Language (CQL)] | CQL reference documentation -| xref:cassandra:getting-started/configuration/index.adoc[Configuration] | Cassandra's handles and knobs +| xref:cassandra:getting-started/configuring.adoc[Configuring] | Cassandra's handles and knobs -| xref:cassandra:managing/operating/index.adoc[Operation] | The operator's corner +| xref:cassandra:managing/operating/index.adoc[Operating] | The operator's corner | xref:cassandra:managing/tools/index.adoc[Tools] | cqlsh, nodetool, and others +| xref:cassandra:vector-search/overview.adoc[Vector Search] | Vector Search + +| xref:cassandra:reference/index.adoc[Reference] | Reference + | xref:cassandra:troubleshooting/index.adoc[Troubleshooting] | What to look for when you have a problem | xref:cassandra:overview/faq/index.adoc[FAQ] | Frequently asked questions diff --git a/doc/modules/cassandra/assets/images/unified/sharding.png b/doc/modules/cassandra/assets/images/unified/sharding.png new file mode 100644 index 0000000000000000000000000000000000000000..9376d9a5cfdfdf10727a2996579011fb680428f1 GIT binary patch literal 40953 zcmbrl1yq$y`z{U}kW>%^q`PBF2+~M*cWneVA>CanB8_y1v~+hEfOJX6rn|fOKltkV z`JMkcYn^p|uC-x1&&)hC_uO;c*L}?pq@*DE5RDKG4i4_2w3L_%92^2C92|%Qf4h}8bLR3^qT2z!=$-&mt!U_rpClwSMhpZMoiS4&{mlyaBPnb88dxOOF|DXMu!*DWz=w8O`IY=f*Czd_K47pBr*ke16Oe*_oBzL=L=xfscxjf;|eIH5=S z)$xRvmaf!{YR0dO6#JW`8>^ohd-L*Nwp3wPT+*o9%L+$$kIze_A}Rfp8e1i2K|6T%0T1vCw&H5Z6lmO zFq?nW<#lAHINB@Yk?=q=%^4_Rx;# z_^qTNO6Z$-Lu@ZN4lKk4a&|HQC*;qih}hrtQV=eq)Qh$#kw1YK7q!R2#1=`;OjROP zLx~l^Rbn~jaKdXq(SLiHfjsz(6TUe>@Ed&sVow`m*&{2^h#~Jb0`fP@`4=yc`rlP9 z2-vadp>nse?2Dg2n8C#S0$yNvA{>;tth7l__2l77@S&*qCnIIhPYpHJ&nW6auf?jf zJd|9OMe{ZD70sEG-sFA?8$2}!JE0^6D7>^4U(1pmdcKb5#C^%?@l+t__C1Ah4BPvs zd>9Bg;X&W5+LjH2pQ$I9_Cn=^OrIRVhqQOUaxUl60$U++!8f8NM+k&ke|0gsDx;cU zAbynR$MKzK{@PAXkB}8v^NnSje%tJqutmYe_Wjpxj!npd*aP3p7DD!&&mUcrTtD@B zPcA)3JMs7$#Y*Hsrk`EPphBJaQ<`3^oDU6R$VSiD{OjIo$yrew&H?3Cz%L4(1?G0mo1R}T7_-{euE@YZj6F2nySlUSzuY`kmgYJ zP|`lQuKa~)3QLQ8lU$Sdb@;g@>YH~pi26pwV!2`;zgq<9VwX~*M)5|KM9Fm(M?s^^ zXpm|0vguV!ReV&sR4!DURmQVBa_onBhj)jqHmo*`hvTy)>FI-M+hf0e+Ri4<)5{I* zHiSAspR%K}yR*}octB~PIZzg;D3rT9t}88`Bucbv;|JSv)H2$#Lf69|-Uhi9p)H}6 zR%aC*P}ey59;Pn79>-~MWI@g!aPdu(T8j+$=#}qpBd8_y<)LrWf@D=R@73}anvyLUGf$3m46bC z*1bml$(rAs(XF!allnDVbuw>3NzrZ&t6Hhjo_hMw;3&$_k74s+w@=bPvn@JalMZeV zr~FhI(J83bE;QK6WE zqg|!lRL)totVLShWGh~-V^Q7nxo5MmI;^^_VaVgFL=Wv$?u=<9t>{tS%1FtSk(Itx z?POZzjft6cys2yVuA(Q^mDm;U)d6k;(+j5X(x>)=c80uaFEDsiNU}(LoroP&T#b0u zc?BHDoJ{#Nc_M~RHc*crTw3C*;`_^1_EZ(7ZsCn9aK|L)>6Yl0Hdc9l@GQ7e#e5j? zBElm=T9Ii$J}uSYDVVI(+u$t*?t>8Jo`mC?gSo}o9nnM7LxvOgRm*v%eQOA6>9Ik* z@5++;RH0j;&W#`0H``Vij<5dRlRF5g4E`1#3O|6*2jWIPMO1z>hBS)gjpBmJisJL+ z6Fw!z23jmi7+Uee)JKl!Oc+FH+^8^YA1V>HH$+2tK_M7nMZqaLJ1v*DS z&l@bcO6qGg?5&5Vj<%@B*v3=CJH!IT$HjeLU9vGHsPUO(M%_L`w4up zpMT=eVMJntZrr&MoK}wRl178BpEiKzIn7dZR`kGO zwtG<1?#bsZ?vlMFqC>}XQ3e+VWK~)u!8f1P=$++X>n35aKbd{9iQ0S1zQh1wicU0_*xxk{28SUXs&W))D=VYM@rThHP-z3wuw z^PKMkAGYgt5;RFY;l#&mqW{z?c6(=%3g+&knNF2P^@Csr+PbxVw!T*nJKL?9(66}C zW#v2bNV70do{RU=xIH23^`XCg<)XNqHV4{zL*-fS>F$Yq%Cl3*5X0EZOg!1@;M)Vg z=|6+8hgrhsR{W!4eVt;FVj(lr2)0;*T^!X+1Cd#hz3S2HPU?>BagA3bGbL+r{C**| zq#T>&E-7^PpJhRJ7T(7%Esc+da85Mb}DX%?E^zXm??)mqs~b#4F{p8iHD{# zjyC8&qlVZg31PfH@rK4+$F8h28@)Y`z#Xk>>8e7SE(WJwFOkcs8jVjyRs#wYeAWs> z?C~WV#+(Hjp7?ASxC*=OLDPFQOWpM9e~Rwa+l?-4)?2!a_Z#Ht+cv-Q&RWSG?OTNu z`&jQ*?^@SG9Se7Ac6Lr%aJ@-fQ+D;5wp`A-1v;9EZVE529*8{H#gusXXx0oy<{NoD z-}mh}kRzf-sDr=XoAM%iz2I=98+Pn#NmxP3E40n!b)-F5DM?lIF}^kL_7=){#dG04 zlrW{WQ+IJ*j~Wp)4c2IV>fPnbbHuoYSs>I6*=iO%$~#Kuru#-G2}7CdZRzvP+b_9n zUb!k^xGTsbixTRUTk}yn*E$M1Em(7$SSwF_2(g0%w{H4c->n|)99SK0Gp92c_D6u| z#TF*C;N~=_97Pb}Vy5A+G{L>YMQZ__9sJc}}D1$FdSX z$rcRF;B>VbcU5;FK_(=>FeXzXBN=VW1PLw?`yD=-F6Zsx3P}o07wx18_hbsk1(}a*R3Z<@o_lT@gV`bR z2@M5>9~=mU{Lc?OwB_n&rXmt>@Q7H#e|`|dGjcMc{HGEMxnVIJE}^9vi9gk!hWAf# z3FH25hE=BFhZa^{8Q%8z->$HP{m>rv{QX80EhgcBjwx-)`M+Nv9Pl3c=k>n^QiPi! zV!TsCW%t*(urfrxvHv{)&={IhX1UAQ;}?HT2}LRM6~SKvfP=S0${*C?+d?7!d$jq3 zWf=dO0T_y3HAv08DAgG;{O{4KnQJ}xk72Nc+jIcijP>(^B>x^QU@nBeXY+q+E^-(a zq=M`63F`Hag;tZ1%jSxUvu2g-^Pngp-`$_0m?YP#k4Vo|!-)kM=~AA0BuPdDTTNFf z`ami>x#a{Il;|Pf>nl9hC@!p5Y|K)`L(;MAClIc}e_W1kR9JRWFlZc#btQ8vPAnL+ z$(Z!MLdXte1NoBjFg93`2p-2*vaWRDc+?+WRuJ}F;zlIq&FOkgm+d5<$tuHKm~C4I zcX?S<$JVq$u?ZacuT?FGd0|d>&d1=7UawB$gHj+|HhhcaUfWadNQJf3D5b-eZ<2K1B@KJSM*NLu z#E+f`7MG4fTr_#YJ1F;feE95VlCAl(J5Mk;T5(Q$XFI0qCnA2ndX>7q_uGnB&LO zZzg$UKTo3Lc7fLn3^)wsd~e;=(&}8(u4^a-GOp%k16Pv4+}&NpJ;64EGn@+-G>0d7 zt&KvdiUMzI23;dnE9CtB&?0xO=G;#4hb-!(2R@hHI>%cXv?}Vj7OlIS>XQV|(SKw! zL;Q|!jI=s1N396s7Ft^Nd+?cWnz9{K`ozOqI}Cc$z0oU|Tw{<%aNg{Qq#0h?e;B{l zraC^mXKEEQ(xBG;BH6P0k~l<}(zw6CmvIGFyH>`3mX5@JwRA$zUSiN#SG8^D<(nyh z&C_Z%<6>e6T?!#x$8208y4Z|9>F-j!Q(jlEAX~F`E*axA*PzXc50eNA=0OMm+doF} zCR5gv=#z3_+}~6c>Y{+LK$2LSYDzE&Mm3Vs4EiscU+s`yY>q4u+2k2Nx*H++s*14M zlifOCBTkr_7N%W$s0Gc1UZE*i&6bwzm<&CfjN#TWHGy38kgrE2S< zR|is;SdanBtrvXnEcXPpS$9>z+vX>sXW=ZJi-qeU?cQ;coyvPL#^tx}%V6!LMsc|j z7g#Zk&FAT5H|@<-m8qR=zWj@C7bUd_bzQA9CwHW!r;eC@zd*{S39yRYDXccudD}pK z*So2DIb{D?!+89t)>x@qUtb4H4=)|tb>|waEwE951L_^#4s6mZ&jW!bKf$HRh3n6s zt~%fq)8p*%Me~XOL240#_9+&~Eq_1D29no6eS>ra(y>V%t#wSMRlD&31K=h33HnTe zOLW2kv_5h6B++*=%XiGd3g7Q2bZX1}+~k^K_Y9UgJ6)q#1qyo}LjtJvrZy}y+PqtG z)apU-Z(|QV+s4ECW`oxKO@dT912LI9%L7qQFp`q5JJpBJ_`@6hojQr`A;4v6Wke3X z8Fz80d}rkWELN;JjBU7L2iO?^`YJoBmOdCr^SIuuFeo22`HdP1F|GKW(><;M^B>uE z#1s=O3R4zTDA(e{ro-(Ncy?}k-t1I3FA17)RujhkFsB)n$P#N$Q8hj^!^?=R?4^}( zve%@1m}{tX6zBo_oP0asBh09VYL(Nq6AvKabdGd#BS!qSE7})%j?25T;LdU;s!m2G zBs7u;-Oe3IxV~@gBA(K225koVh0orRfN|xtYe-T!x8VeH=;01idxS6^lH7j!*dn&zA&qlAtLG5HtJr5Y*><#4_&mgTI#oB2@5(*EO?tdB>t z!2CK=@^>u5%`)1lQ24)A)sY*Z&)#J)xgqSdHa|L3AZ)gE?#Ny;d#Rs}?lJDtYI=19 zZiTyUAtg$OwTOYmw?^6;lK(4wVr1tLw0&s}a~2sd16g#-Blj*E zqd6+tEBa)Y?*ad-QNPlHEI$%9*FCVm^S`lLOp>duNMG<=t?q>le1F%q>NNH~AnknI z^;4L)dBe<^&im-in;SG_qA+a0DBpeU0Un5>*?%4;m(1=m;_vN0&}GhKWHbEsjS|C6 zgeU>>quZM$P2_p?E#B2EPIzvx z9tZ?(-8I}#g`m>1$Lt!>?fvh)_1?8}Y-u2NU2Mj)>4bz+M;j&fI8)b~$aNo*|H^o0 zm2BDtg3fVkah1E&+M~|v!j!_-)Z)2_*@|y_0l9T$GQEwZv2N~==kqr6n&HUXMR>pK z=A2LzdL0<%+G`W)H3gn$r8CgiFxGAr>64#bMYL5#!$tLqP|Ww5;yen$1Ut+LW!3(8 zJ+kw~%uC<1FTMBI+&YN6Kicr%NG)GGbXt$pP{o*jT%BllI=awYw#Y?xv(GS*!Tvr5~VXhTc2and=mdRYX9+}6@`nmy2iCfu<4qKv%wEX~X zH&Y9xNdo~^)>(9#LdZNOOVSNspo*)#$mv659S!}x02K6 zD|!uj6lQe=I2t^-)rTz{HJWFZ%ba`pFtdxfV(;PVppjo6f0$oc3-?!eUX zNpnb^TLF$YcsryIVB~yAOG1{2PTF#3#i~gSx3k%mL`}Z}3V58FE*-sfj(Omr!MdmY zZn=hp!k&uuSn1pnCtAisggV~_5+;@HcL_D_K|1%GcbSWxZnq#s^DGh8$A4O(;k{uj z3;Bj)^G}>zXfQcD8hi)P9WEbPkp!fZfjNm(D7BYg^}RihI+sH+eH-_0&)` zt}dg4BRkKYRbLf3pJ%Q|KY{nYI@sp}bf-3<7o$hd6?yDTlFKyB-vmaSob9FtX2Sv? zFD8$zHb?ecP@e|Of%%&5xGPLI%3b@;&we6AcN*-itMSh4mhxXjmwtQ$n@NXHKNEES zoHDbiB`Y~)yt_tZ5>p!cJDj!hzbZ{+AsXAYf8%8& zLdUS?nf6V{{TIbObnWVQZGPD9{n9S`y1{#BC-e{Ph$6TVLGMj2S570V7~Gf~8`o-= zZ^3W4%oFRshlpFQavwu z1D%|FSxx3WB068}F!vRCic@Cx_h|bG`ozRzMsW$f(*x72XRCZlkJ|1&e(UUS)8Hw{ zKPDOE9GRX-u3^|q$(ddBOs>50YiWr*I@H~iP3B-iej;isTd++O(elD+Z}*#mP^tkP zj3@)#T<+5r^7vO;2#+)$I0X7wGH5<;0J%pD2VSabqYPWS0mcnn!)9!&Kh(WD>^2dJ zQa^Jj6XzeEju+Yz!gb1FDYH7J_O??g%LdhAsN~Ja1XQMQ$&?7MD%mV%TnE&x(yHiaIK5-J48T zX~FBOcpc(xd-)26`}tc-&jb=j8Uq^l?XQfq?!0NHS$EAqB~!(nfY0Y$UhqA+1}H?W zS<#=RslB<$J6rsrx%@3ib!Dg%S*B1m`QohN((@ngFq4+ZMM%TAI#UzD+1%26)${%& z*<`b~mg=FYo{({p^<24k9y6pyX83IT`J=9PepSwyGNZu>STea!t5iOPVTmP+2w)Mj zmd>^0`VM9P8Wm@`hjjvOk7kemTjXYqqDairK&I@lp4ZH5flRKS8j>pnJMWQ*X>*Q~_+WHx7|R9F^0b?M{x7|v5-v~GI?eO@U)qOoFuGEv^Z}CJ;$NDNVI?5? zB(IT{|5X;b2R1`LbrZ}LCJOo7uTo*Mx zbI|MNgY)ylM`}sn>H#{?`GP75?uT^0^J{IRhk53adae`PDQI|cuL^MLH@Mgt^o|1; z2$<$ku;CHI(fO}o@G9Bn$9t5#Y0e?x>OfV>WV4CG!ung05@Rxy z3s@ceA{w>piL&uyfoiCu-wqhZ2fva=&$C?)YQnF=aQ<#+#Qb*q6So&X(hrDPmPFv- z5s;i9C09<RlRKea81raPVv zI1aD6m%GIPv${rP-Ja6b;aXoDZx1i+;GEXWRN=9hYATOm)}DF9^GOWOAH!cY>3P&4 zZV&W&6&-})CYj+d8r*+duXXTgj^PQ-OE~{X&hakyi&G> z{uG{(1T6=qNQG41m9gxEM<8ij!<+ClA>3lMz@1meJs-M6a~BVjXQAA<#WPUuMpGLf zK^(AQF9)ZkW@$8RtR((H*rs&-Aru>ONUT7in8wdT{MLIC8IzP*^gTv7*p5=6PI#)< zHrp^0G<;dLikhzRiEgmo(M-CXKhe+zsR_LI*@sCF@)Q6qT4!(vi(rD*5f_*}H6K^nPnXGF}S)E(X* zL3MC(A-CSivl0>j>C?s=s`J01Vrtp|4*W} zF(aq3ftTo1^)~H@P_i`(tv^-dXAp@lguL0~RDN_()drEDWq%tdD_cM0Xl0rQ=Gl<) z@#Z6_t z%?Ebten!gU@b0FP&g@x$NSmVY*~&(XcY}3Tf@!jCV`aqe*=yBV<$AxwVFRIv(b&P> zx_rEwor<&dJ*Fs(wBdQAwOXiK<1~8O%M97LIuDk%VH5te8`u+yzjfI%qKFAeWqa+J-M=2f>G5KpjKo_Sspo82|JoHU3j z1|LxQ2}dCr5FOQox*g_FOFq6@4l5WmzdA*$v)cGPA^Y+XB9_$WHb=!~=7z`A6n=hZ zh^7MxTifa5CE9zDL8RaaI-cFZ`K{!a`E*lc6N^B!Iu4DbB2-#oJi zkXhd_+IDuf0jj%A(B%&Y>OkL~i{0H$$0X%9_bNjC00-j18T76vjhK!LEf@^-K9IM7 zRU~^|xF`LduM>ZD;Bmu#$M5;BGvPYjUGJnI88DMVfv_m#U&z`Z&0EvbRvJPA3A0!! z(g|Z6uK8A=Qe$PoxBLNEBzDApbC^zd98>Z^G8mXHiGNb{>+{orJm~@znkH3xnKy7C z3M8E>8%Fid>W?M|mcs$}Wo>C{{ImDC7kIAT&>8`gk(uAzV#U3ox?eh^+pAtfCUlEl z7Bh{_$DDu1GHP{z4X9is{?}2|vKUaPaQq|W?|mu_NE*>b(DR=)3 zm>2!U(TeOpq(iv1(Kcqwwn3(QKkWs$-=O=QuwNU|URV^BWH_Qz*eHQ7T4IJ=EGYaASeca}s0vz#m z1(CY{eauK+aL+>H!1@d$w7wKX+*S&VC_pR!38Y;43l&rOJKodKWgw<5A^BWb6v(9> z2UyLDFlBcPFoVR%13{I!1NPyajxUn*25jU4B)H>#bB}&!MA~2eT6YxZf4GmJxNJqe z42+NpY0lRyc*gz+PW$kPooa~+h}a*}+p;hPDQ>)fZRFyV)T!Kn86QnZfo^HsZ^yo_ zynCq9gjoC}_xcT-A05(^E5Cj)9t-8D;^)n-LuH_o7p!(CC-Ge+sf`h?Wy{Hc`M7p; z{y~00|J$=UE6AW&C>%%`)rr6IpFq(Bw$q0(6*FhE)uw0{NDlrNsQn+1j**jRhzbC= z|HO*_gZs7t6bgMHJl6Dtw>wYOY&n~M86fi1C{F}DRzsfDs^@gUFl%Rb`Ci*P)9u@(sq_bt4vtM z*bso@$fy1|&Ubp*Wy^hLy%eHGXx&F` zW{{H3r5YLlC(+lyaphTaF{6b(mA^NWri5Z-kgdGTZw6oM*cxvS4vRn8;_~>z;pcok zE7k>PYfR)S`vkx|ib*vuvJ@a}iYhM?0BrouT4prKs9ouy|0gzB%~tf)`Wi4?o$hi7YJh;OBn5T7MK7ZB zS4ued^ejAOe8VAe#gpveJuSjsA6t>xa6BP#L6Wb!r$%l0YX&Df>7|huM?N23yFCB` zgeQ{C_AI2qc>}5s9}0wrno^`#$NoJ^T&CFC-@K%9V2c{s6Q|((pgRKCh=iPW^!cJ#xpwKO90Kop6He7*Asn+BN)c;u;=7~6Z-R?hxXF?>cbIl*I;=m;pwnZ8T0M`IpN;Y}tiP?9@0qLVvmXtl?&d`r)CB5`-tTjzj2-dD5B;fT>1^?r> zDkAfRRaJftDv7A`WxL3@gj=Qr@c$v_%VNT|JR{kfX8FC#V#A?LYUaHOm|_53{2i4d z(Ln^U1h7gDHB$$8etg=O0krs$iHQF^=0YbH@c6m9A4K902Y+@!u#>Pk(2Ll1?z7%t zvvdiB@YTsqN$FIX;^igjbmZwnB6vg${Cvm>C(lu62u4KIKH&6xporTw=>pyQ>fSUq zZobK`w|@4Z-6IZkd}gW1vEHwJF(o^!k7x4$mI`2q+D4R0VZU^L&xMsJwE?1veunAJ zW{aslQ#LQciey_f2-+PNH`Q|ExbQWAUt16jx7ly*^Gb^!`=#FM!;LwIxXL>UtkL6r?E_MH-)l^;kp9E$fkXxXOy3yAS69=OS67Z~z6?s)VZb?V1gnaAwlwQyO zU@syH2mOhi?_m%eh!ndHNXdChue4QaeONGKgFgYYqW8~(Jf#Js3yjg8bsNCIjGD#t zzd>6kZK}5=BM1=oK4Ir;Gzb5(J3yWe86MupI$%?tpb-8M_@1eeTc27<%O6Y+#t-T!MN7G=$@n}^o53T zjI|vjMeD#P@HmN{EI6D^l|QKN28q?$QdZVjA2G{_8( zoarD*m7$Q6R~Svc=CPY6e*&bmFa51`Yp=!*)GKWFdf)K)`zUNJPVwq7d10PvB07=xjTy{g3o07V*uo|0bLxH?{++leoSGQ;HP1d z-}{tH6$RNJ5p}}jm46Qfst);6mOn^ze_@l_0LX(E!xBONRVW1%zA+^DUuxdd#;+G4 z&~4cz=t*~V5cs(&KpN`Mn%F(w0H*~idYxTA038g&-*(Lle1GHsY>{VF?%lpi^b{ zs6%*c2G4M|-`d_Y&($ZQGDo3V=N^+h(O)kABB-f%E$XM)dqJc#8`Acna+2V6MJ6Hf=3=8w{w3#Tu|ni^WcHZIIyITJ^EeZs{?9G(iij_jc0(IKz&z4#E`>=08Es`HoERXr zDx>SUDcD*}X2|~hmdu_3<`rZjcIZ-iiCrx;Jk1pMI(#CJV{Z1| zw5_xMvETKz<3Ja^C?z9D1IbU5M)e0#d$RjnL#^po`nmzKIcQa0I5OJN@Ph^4 zeh~$5(QOq=M+LE|!bUYQ$md|lx`FmiW1%CQfNH5D+`d6I(f;WYmkK2T=y(p$jzwtnAdvc30|OJ`a2{TB=#)n=`EuP}@En<*1w@Lq z?%N~me(q1luY?^O*$vW39GdRS??q%q+G`fjzP$M7;ArA41i(@re@U+^2LVz*^-|5$ zZgGkQ$V!0KOxfh`AS#%6nA4kd>kLFY!&1Mi>)x+O=ymKkoJ7@>96<6q*8ufYBJ%;{ zv+;Xzmnz+hf40Ghz_Dktrg$1UJw~EZ|5}{EpeeyQ5;0Vm5&fToBZ}6apB+^gIkNR8 zo1J>J?Hv0!;q1KZ=PKNjw`UFl)rvK=gP*W?H6g)`20oI#dGB1r2na)j zg+Ed7?zg4z3|d%AbkFIsLFSNyQewHc4K^UZWG4ZKMrz4RG5A2857BLr6r#(SfuzAB z6TMF##u9L>A}HSz!g4@M3la}`^%o5kZli}cCPaEB6JYrFDO4NUy+95mfYDWnu)*M9 z+>nPr4nw~`T4ZZ-TN?u6I|L#gKSkN=c0#keqK62Pz^JD|M{6wDzbJIUpx61XTK#SJ zG<0qTN#m)_MA=I&+uR@vCR4nZw0eNbd9%{e*mS_$KG*CCon)V@{kR|S9#xC#*=_Y% z%FR)P5OX>0syPA_LR$RfTTXD>jjI4Nx1MvLb9>*KXTIYHRAn-LyZb{Y2J5NnxI^4- z36fy|Vsh-4Ivf^*=p<6UmDkP+D?ioSmStj|4@L>lTW6-I^Bs5#$__4^Q<|#J|0m#oG^~) zYR88VI}jI5Q)sWRzw*Di!ju9|)3U9vRCAmXRqVkV>AtrNiu?!fs;sBEc9;5Jzti67 zarVsUBq(h<3|{XFPkL@C;}1frAY{3DT&6)T;2rVPR!NfAiRasAcHjm@?oy)l$`{F7 zny_Tf52*XawR2H7S7)6ubVCSxbIrOXO@}fCs(InpXRUX!?;jG$45SIHgn(P5j0k9>%u~;mVrb|C(1mSa;e%`I3&|R;WaPW!(8cdT4%61PP^UHE5Skj&mcYar^eI8myFt(H3(_zEae3@)o~Kr3DsPc7 zALTpMjR@^E9i{YiU-Ic0xK3yZZjG0MJ+xB+@$cZVR;6j!^%B{g-I#T=y13#~^I_dl zQt#s-IUIs#hJNb=n({@{iEitDenX&zvB}-p#-9FEJ_MujH6xFc&DbG%+Z(~7UXC2^ zt5b)KocQjJFd~l2_R9co_aY;V2;p56w+4};t3xAiOW9*9OhCYLV9gj9({t>5emLNJJEC1H zbh%EIM|spmmoAm3kZN*$egIF0^@x<8UNu+l@SZ5;71fIKgAWU>EV?x>&XBjh z;Qt!%GmByB0?kr`4uA2<3TTdEx?sP*e5Oc1Hh|y#9=i(CO#A{K$B|wahp1aMgYa|I>n&}pA&yJmZ z8~!O2e;XzR-VN$n)wpyI;)6j^%%C)43W|B?uM@1X(rNO_HO2DJ2|%X$=sd~#_S^XJ zR^~U!#RvJC5Y~ou%Av~I+XAPJ;eN&DJA=KvQM5462i&%E#`wh*-CIH}b}&rh7xQYS z#~Fpto*5YGyO5Awl<&I2$=0koMN&j5w5BgS9yqUb$M)lD zStBXmZH^UhVRSU?ygmpLb^=XVRqm!&$CjF87maBP_-E3wRMk7*@v*q)}l_dq(`DRtHEV2aUA4(F*tt zxwfzG&}Bxoz&^<(vR625sJP#pjHjAe&ny75c!a-uw7>A}6Onv#pB$M4^}TV#d(2i| z9~`?>0^3Ra_N3G|Aho2#7ufUR=BhoHO<~Dewfcb4^g&_11$htaMpmQyb?WGQUsCtw z$3x?r8^BdLA6!qp8b0C^CSV1J64&DVR!se zaq#%Vv5|qn$RkeSG`Q8Z_%H3|c0EoQu7E{oU5MkdJCBd1`FO&@BDL8VJw-7uw z7AHsMwf}NWo9C>?;pXRRtTWkO`gJYN?d;oD+qtI6$#_cyk=@F=FW>foq^uX~*(z{& z_~MwN6-VI(h2fJa?4TNx8k`Fc!Ai@Zw7U>G>dg&%jNAo zR|e7{$L+1a`SdeM@&RQCJ6+0m@d}F(+M%$>Iit8y3Fa5UDa#K@@mm1|YXNzrHwj+r z0xvKliGXewO_!UqF@w7Bf9&!WK7?P#)wgCAhD6E&(!4EdHlUE{R)!7#Eq3{#Jw3wc z<91x_`9KGW0<>IqGk}}0`>&-DvXq9oPgmrl0wn3v;{@PG8Yh!mzRIg?Gz3^9ieZlpB;#qa#oC5Ee;nbB160EB=d~{Z1`?-g&#kQ zB@;34;%JQz+>5bW(s~lM$UWy~oNEB_y{YyB!ewHEC9=1Fyf04KuJGl91%Ojb-Vp2; zV6s5$+i~G%k2`-BsER|52^dH7l|?;Hw-XEEYo@XvlDw0d+V|&*#7e2!xh~eNecSm*LdbUpj?0&=+zQ(|2JEOO1kLcv~JB z;Pg>f=4EPui)*SD{7$^?Q+h6&8T!4JE1QXHO^A2((RK12+|+tUwsiD0AClMJNS>mw zG2E1DvibM}zOOXaruymCM~Wf%;J8E%vrE_R^E5C-Fi-ho=p$d>pP-K0BW_n4^H0f+!cU< z+wWO{zI1C>K**0T5a)5tc<_)=YP>{$Hk}}SsWZYgskd)^{w&90B$pz2gpALnXuaoy zXpX9)!1EE&CEgiya1;50PVsA4OEahK9KWH{C3CbZb{{5%QJ7)u)nYn6%xAM7qv56t8KAlJ=f=$ z5*>HeUlXON9(Tt|aZ3I)9j7FH#?lahs?e}AY^|05GhdmCXRD}^{bX2EeEq6Qght=g zdGevc)YMdq_thIdzEs@1<6W)!dk!JKnQ+Z1ul2&qi_%*@YJnz14 z)bf%rbzv(07o}Yh9$9v;TJL@gjgg)H97_)Cez*5Rcbdy6%TCsjF=ES=(oO>2#HR5& zM&_;-S}gcivzrS zAM9k^_slGX`<+{(NO-L&P6>;%_Bken&?bCk<=)ube0#=4TC5$f3& zlD~^PaKyiK^mO&bfc2nPOq*Hi^eLwl#2u)zTmJg-5H~$8QFh{dHag1wQr7d~=02B^ z`^a?t$b{VVqZ zAuSqFodArk`A{n}0yav(Xleq&s}GWa2Vgf-&c$AVH~qo#Y2rw6G3QpIZy$rXUlIEj zuIUT#Dh;Hm`1PM<^k9qbHx9D31j}9Vso7({QQEc`Lj8!G^M9CJ4Ni+_iKaXvBLf^YiP>$>@)wzc^`mw=S zOET`9)r3c}`itag=lPWD5CPVZ3Fmk|atfdXk=w#f=2^y{A#svQVEEpW1}fc!wWrEl z#xQ)_@FENc(_|wp8ujn7P}4(@%4*&`s(_aIXYygUM}WcH`0(yOuMGn}1EybWDZmB% z{Yu2_1F=I#d52MZ>#f8x_1k-(;&cb)mo_9q7mCAX4NCGcO;M@w{AbIBdX;of?}UNC z0uvEfL@4h6hTLu^{#u z0bmUftCajD5bZn%^%mA{sm}lR3lW1xm_o|fv+fq9lH0rUoY7ni6bp5EHYkOA8w9dD z6`$b9;s>^cM7aJ$*R$o{(bGZb8T-p@9IOY@%G?%ta~?)ty}$)i<9QToD*JsMUQTqP zKr?ha1`SmV0;abG9h?SW*wdQO{kK#1m&~t{#4M!418^WUtfQcO6>5=amuULsKkExI zB+0%bZ5C(^|6cxgTCn6)@=O006ul3+VS}CnfbicdHBJNs2xOTtJGatd^-M}AL^*6NMv;}`l_)|0Pw zL5o1UrRhfFfec1c%xl(;4du4c9Gu_yUXp3W^e&3Nq=vmW0q1lvkJGX@CWOqZowWNy zpH)&g248#-^k8vRt$)=1!rHRDdy&4)-#LdVq^8?#ZMfs)jXymx6=dLMfp(>QIfsjU znY3DsV%`d;E}PkMuSK00TfT3YeLTGH#r$oL%JC{za-nBz6B-^jUY?x0ZylBT&mgV3 zT1>D>5RIQapx1*mjCl^nh|WmU*brURc zZ3(#*3&;nm2-&_BD|z<<5((XLTVfH{4}ReZTjX4_y#|K0{yJb>S(cJ^5x4?Gb1wjuYyy6Ep{|q zd&#CF$IGDe*lP0oc!k5Z#?#4}j?;TAI$Boc9MOuSW@BsyWyn$0n60JuiP_RzXy57I zb{`lpat+Yp9!A#qzNV!a;(C8f)T#&Ztbcq@KIk6=0?fKRYEaRzI6F+vz#`j{n~9dc zjKN1zK?CDIZav(B8=Q^}R&R<=Ze8wxNOst62-rFYi`F)})pY|epvxZZ$%y!_ zf>jqgP6WDOn5GwR=-l$nBkLD5W*oC!k5}WLBD#&rN)X8bK%P{wndytg^WOSj)p$#* zr=x}RKrPf^&}4B4x%UWgF)y3OmffOC%@r*Fw2t^?3V_xTTmBKJ?e53&&&8|)D@j%2 zy>7vRx2zqtfOb5^`o9xYrHtNxI3J0Z7ZaTiNnE<1v0nd0_x>-`EI?^-h4=n*FaBRR za75|$VkyLV428EnsoayF9t!}Z6zC8w_y!}OJ>R~d=ad5rNqJLMe5>HuveZpK#Xlh9 zY((OAe3v&|&Ka*nWjaB#m&fiq69VT;7Ign=h~W$tVKo#2->XcGLPP#Mb_4@08`iYF z&LeQPJXQN_*UjVwF(YINjR~NqptE2)DIg>43)qKyE{dYp?d#Mq2Nc?-y8o} zk2Zhm^^wb7eIV<0A=4@+Py^uhiO%o~u*43aSuwt>@t-btQuvcWvw^zO-qwrA0>(hF zLL%6bqnZ6zOos3`Zm&Qs1~4Adp%(A$K8acM$KHFSbwi7qH%ozSdgNo`k>R$LgvgW^I6cN_3&nrr!s1uYF^#& zM*}MOHCGTT^`T8Zo8T_>)~az^WCu0az2TfceEfD{{XfXrKd|i?B;Lp^JvNwYDUS1R zFa!}1L;;MHHt!q(%KeTID5fFSWtLqh%PI5c_>B7P#J+72r1QeWt(LC>K$(#Ehd9x{ zdq~_(D|1>tu$sN#`n8tU-y16qL(at-W6c$Rxw0MGdEGZ!bm* zbf|8TaJ>cEVpnnFC5bc9CXrk4(tM<79MGuSqi7V&)`s(-(p4VYhMtSDT40k*&iEWG z4}^mFpBlR+i}oi9P&>N17U;^;f*Oljg)(rJ_~^> z;5O65P(0b;1nd3AXsl+dNYLOfFH~+^tUqIP8d&ZYTEG%Lrzo@NiRww@?XoGZ zkMq1{6b~w4{BEeQ2yCH#p!_j-Ju(M!{u4&2_jGF*O7pGZoHym%Cb?mgga*_M6*CQzwC9_UKhw7Oq|ZFlV7y#A{j* zT>7HC>r`|-ou476p(A*JJ=KJ0Mb$*tpMb?DOo zheTH#^OVnh(SsA>!s8bTpf(iCVo>v9!vNHT5xjK=K^h0{**z=Ir)A$c-(3g8g#AG? z^B+9TdFZ7RvsuN{Z<1HdH1*GM^S07X-t*Xzuy{)!eQ@CKYX3}#EW|K-b%(8}`FQ@DShip)QS1Vh zzJHiCPH#h|T~09UL==uveGP-#IY>7o9|9jbY==NC!}qfrsf4~)>UA>7rSn6h$e4Fe zKOi^mGntD&sA7j)I5ZfsY!9?0Jj=NC1-$7kl_I)$eWyUghCKeA7<@h^PSr}OJ&0E6 zB{J@VMdMX&6MfHJu+d}Sb2d^@rVPo&Iwf!0+#2y!3eZ~u{B5xBVz3})D%!$JMDtx2 zAlY3^)tr)Ff^Dp5*^S_5>xwVvPre}k1InhgeO(5D45;vFnU{0SUo~(Rc6TSRQ!P_j zVxoN!pPd%J_`w!8NgfQgghjsrDCM73=It3|>#P@WunofM@Enj|IP_}CgE{$g&3lRN z6B`$YgA)tI#Au*+Yl`Sa?sAeNLi*>gZvi(QCf+EX8)>1PiuP8V;Tm^9SS0U~_$QSU z3kcBUn0zfQ7KwZTh&T#u>BK<l_!QD^AN4^N8VYIfnU>}+!o`^MVh zg&qY#l({&{`Zf>1LP5RC*n1@(hUgOwx z5Qu^_M_AR-7cLx@kRfS`cw`Z%@BnXiW;IvEu~I9b(x>aZ*kV}yX!gG+z{Th^%wJZn z3-e?$Gb=av6x8xV-O@)kKL?@S#DAakA&S=!l4KPYPdzW*Jk}_{ysFtur zlVqJuDR49@{}+h?tjAAoXJl+H#X!mR{1TQ0psRA8c^NSd;l8K`xL2*hAG)dKp}&&T zXF7XtBgv&sCz44T*)tUlTA=fO+PMwpswRHA)ig_$dVrRx>E@_mz)yB*rs?(D>bAm9hf-xYHVJs^D3n{|hI_G&Ux zkXiW5Zv8PW4AaMgpeo}?3Mx1Vwlbmn{TT=J8U9(rxc!VA8(r!)4Ib-b-#Q^fozU2_ zp7S&bVZAS$kflCeN(9RUWIOEui^U+RV>^K5kPI!olKDX_VrwNP2Y@8U9kA#l7Ug-A zNnie+&by-dY9eBQCT>KdCIhh4zue?5!A|4Eb6?LoH)10plrYbiOa18*&MUJ|=(Rrt zEc7`jYVVdSuT{axzN;R6rWJU=6A((kV|wmAcqBhv=b7b*fgi>~abtu)vB{^raxgTWb7M|}1%@&w z@OfgM6Jl~34)P|;>sIGKaA4Q;fSf}Jy1^z}+g^c8;a*@HIVp%m8bus{F*~7|0aPF%yFWG_3%l^Zu(UT$;iXqV%47 zuU_J#ZLjo0dmvq^rK$ZfCy;K9#}K!6+2f*5#c&ZgF)x_;6oJQTJyY_GZ00kIOl^me zA3_ZNSHCO0Zwz`i$6h7>M^XmX6&hA*K1gZD|3>F%-uv5YYOyt22 zuLaIS%6!lEK;-nRYZ_)Z#WGR3^|rI&a0^tuEE1`Jb;ck>A->=(u5vqT#adF#;BBz5 zvtrnel|OdhIF|VMxR_GvHNl|?4m~bH*apy+F2B$5Ti{MG%76>*V$D1puo3?&dlCH{ zKWQuM-~AWAex;4@Z}>PS_35uyQp~b~gGNEye;-3or}*zHY)0gOk-!U0jRIXiz^=*6 z=IhL~pF0p{faHe^PrKSAyv%=k)rKel(Z)xRaJ5Nk=?g220b7WOkH<%S|HVxXhb_qO zo`Mw@H~;GGq7e0Iq-)4^57_3u{l>(0O?DrEt2#!X^gn7DTwW7Ih#1_B^iOzfoJxNN zd7uG~{yq-qKa`~3iM-1aua+$@$IwjlXFu~z z`@P-ap|jlTzq1N^ryP;qdFn!&|F+c?dkC7&&7$2}UfwDN;hyah?Oh030>dYe1n5-S zLi6Tu2tjQ5D8SeA9y>8UL=Nv$!$VHz1=+DOOO+zrhJz734$r*>rrfg@EYWsQFnqh{ zhzaIfAY>><3D>~iS&-SFMuW_cZ~ew?ZJ3L9FPm^N#`(!|>Yo4Ak-r?hVfoqK(Tu;5 z2&gA!eQB2jDu)fyIwMyN`*@?j{|JpzGj^s-WZc4+knl` z6Ar6>3Y}W_!n(t8o1g7o6j5ToYAX(5T(2xoOYFTZG}YcdKuPg6i7E5HbniWSw6xP| zEE7o<3$9eP#&wm_WAcd+XXSLl%@|4;N>R-U?P~u=Z+jJ&MQ0b z?h$tSBZJ~^{7sfBF#(S)Cd*{Exb76+f@ZE~JH{vH`+XOvZVjm()1Dbe{Wci!qpQ%b z{$Bn2E18jCJ#59c#9^<;X>VvMGN`2irMyF#x8X@PBL(mL0YBDTGgWD)N6h^+hLfQF zvC?S$ESGW9HBe8E7=pS9YKP3Ay&&ZKbd|QTx0xDwFLe8o{~V9=2p!f72oMAJtx4jy z;;C;|zvXL5!w8Lh9`y?CD_Vn^ufEr6b}7ii!u;MfEC8-|+Iw5LdLc?dl987hSxp?l z{*^UMXlj~HOSnuV`$}H@fcjk=Xc6<+Cai4U=e8M%p;3rycN;aJ10+e{PdbkV!Ey4MBky22h5oiwc0eGH!K&>kFp)-nu+UOGi4t(bwS?5^>lvwu{i-#DXLx~IRVvK zM!WIycXkt%a-a~N4eI}cdFpi4%PHOqmuK7Yfal_OTD7A>fN1FwAlqABOENJr#ID-@ zEGT&u6-&7p*#}s@0lhwl`h$_Mo!?X%X=@}o>;7^dq3;dpqwQ3my-5D4AQu6cmV|odO7(%6_+qzhHy*WquanWa zxjRk~=7R1JYH2Gf!dbJP+NKT*L5zi8h$SOp24>M?GV^7h*7cx@6a=iBJ_pRjz3W zpQQfOc|eMg-_qe?S$%adJ2}v*`0nFg{TWI*fQ_2)oNgt3>oyS!bUH!gQeDf6xl|c4Psx{< z#6Lic0l2E<=Z`;Xba*xt0dPyL-pfr8Lm>u?EHjd6O8Uvt`U%I6j(D?^!Pqed*$;{r zrw1;*Blq|mDbkMST0%Je)}f?!NQ)pUQf@)AkF?sWY1s<}BBRjJu$8A>_e6DT+EI_q zo8ZQ)OYbJGi_uNM8VlYEurj@Niv8&ys?7K>j34Z`oA2EQ%wO%R>J*kCM2VsCIRJL8 z2r({CYZj%R-X@-EJEObWeBti_S}OwDZA@q4-l;@XMl^%;6Z8Sf&WJzS6yVObzZhp1 zUz6~oOLBZDmZDQ;!Osi@-$k$W$47lDPotv~c^%?_3MC&+tCTcX>$bjRyHj34eU(+1 zW*$50|Ez%upzBnrUOFE&i`y2dtL>WThFy=bnDY-*Pi@jOOBC{8-{&V@-MIoUD!syum^V+NKL_pjAwX{k?Fz}=5NNr-T7eV+*g-g7W~I2 zWH5igY$xSBn3~zjt=!7$I|HPZWJQWEhVpowU6uFCgX||>ddiN2g!o`qr+!XOo;m$7 zkRkQPVhsiw&p$b+k60f!rgirez^=Vi@FGx}q5qKzXPOu()Vlys)98}mw_Z_e{&Jx? zwLG00OK(Y^7ryqaf6ql>M+1DO*B%Z)6kb5VwFXbl+&dh6m< z33mpB1*gY{YbyQlSXOH`D|J_HN0U1EhUQLZ++kR)8}P(SDlMBDuwDT~&(WH*W8;Rc z$FSn&!|Z4F9_7`Ojt+5_wNz2vo5~+NQxm63%OZPvPjRft^Cx_gQ9siQuFGWk7HqXV zG{EWXJxxConL>RjU!yYO>*Dy@t+9Z2(<;SLE|MLt-6Cpk*;p`fgBvX#oF##GEK$rF<=B<%}P=fdhogm+$b?+jjmExUK3K_WO0mF0A=U^Z)WbiKYbJ*w#Z zXxiH$#}Gswu^=oFQ1{qOLoGcGFmW8TbZ@x2JVW4I^&v(xCS*wn=rK1tXLjqf7Kk_* zSlp<&9no}M;66aWN(a?x3M|#s>Wv;VzFL4POUSEr?_aVz;Ln(aFC{v>6z1FRPY(jb zEe+qBFj7GtTa-eZ(IREe+RsqKXLG@`%EXP{ha2=jRqZ=2G;*ECqwmAl(x0mfY7&hm zgG+U0_8A~IMv8gdW37irU0FdIY}9AQ@<{N z-OK+1@MnZ9Z}trnXOllP6q^8<`To9@)O;W+S^KaG3Tv!>HBX7M}7QKSz zK?>K!cizxk$an#SG(;%vken9Q=nsC*IO|rG!%A?|BTfRUJLAdWghLPlJku+fSeS(` z#v644iYZcRr)`a>71O9l*9)T#X2lrU54QiEg=E z&TGo@2A$n|fk|}TDZ2Fre!f*drpZv0siSrF0RO=!RLn51usM;Sjin)6zIS^oC1GSk z$8?3r(w&)Yie4KFsc=5&NX}*&VV)sVuB)txLUdf(V+6a_7VsHgJo!pMrG>`Mhw^wK zi^a0W4wRzz3i^U>AP~>4ZYU?>bj!-fR^%@99O}6jj@8mLALEkzABEmtGY#e5=ybfKjt- zhF{|b|JRBb=2J_V<EyV5h8KO68fw~#HCU<8Z?`)E)YJctm7G?tL0`8 zW=80PqSTco7z$V~&-K&0EGi%7+yvfL3Z}=A_GGU6RIR?yE#g)Iuz8DN$t_s#syp2? z%YykqJbUjnRR7O{7pDBp4jxZOixyqbgsL3(UraPu2Qy4Q{wO<D9j=zyx+-3Mqe#gLi_$+q;^B}Wgp7mSgb&CLei(A|RNVb*g? z;jP%60{I72!%opGUGH*m*Z5@O!x1D~m+Mqm44vRvsA`*CP5mw2WYr;6g=EXQy%!9P zOC7ylX(wv;i2-4p;yP3KWYs&%jDBXZ`(a=_r}`6VRcT~fg74){Z(QE$7x}DQ*G}w` z-TBlOiJ2>6FAZo)~hZVszk#GR2RhTB5czk*CCx&%T{by#M3w zV39fR$zB;(bp0y;c|AmPwe<~z*Rbu|#ooV>4YFNrTDBjd@ktkD4Tga~v*)U{jJw$q z`CB9Sh?3uBW4&M?EKi@D?=oTSp89=o$73-Ark9G6q3{t z)sLD-e)m2vB;5rS{|?zNpaTA{Y+!*w1dc+5B>yPkKYvAgvpC|=SrLG1Kk0wI9@6Vzsf~%Z==!OueZuESAgqyV-e$@W_ilr3c ze1ujfR-CvUdwC>E@C@Nyn%UHjlbL+#*`d{So*=vjhUpOmOL(UWUX!OXOeA45rdD$;OF+#_f9L!I==dN9ioNK`W(j&PvsddgQ#j!!PObfTfW?D^T z><2HzM%P*5Hz(ZjOnSR}AHsT|2bPQ~J};S65W z^r_uq#-YLWSWAvKVyehwciFz@rCKVd-i=|3RzZd^_OB-8FAn35YsdSWl5o)|#71ol z>Zp~z0N5Lh9o$fg=w_Cda+)LQP+j45P?o{Nh&eU7(68e=aKV7;a6zQetPG04poiKg z+D(}JmwLlNQie%BAo{1ghmf%*kgfW^SAKnk=n?(r7nBe#D@wfoJ{-vo9JBqO?+8wE z$l0QU>LMkon2_}iavK=MUH{mwa;l3}Nn>`QZD+36=udY5?M5(G#EQKh9VE@I*FJAabMgeCl}%6rs) zegF*{ltpi@NVJUSp@K%cbGlFF`;p2Q=R+x}=kdTI^BSZYxtwKG0AcH#z~?zIF}bheveBi#2$pZ-2>eu5IHT9Vl_$S#fe>@hnEddL>{z!3*1?#i@$P0u^sv2z@G(l9CcQvjqumQ5y=eUthe5%!DoL;kh;e zD=xR68ds;CDzU;Y`2nPY*Fwj$K`BnOE@q%;F1fGtkb)Gl%3`{^*6Z4q(NOY3PK&AJ zkazX_VL^)}1XVWp4%!9g?W}#H5B+UnoXJivM)6rw5(4*7AOvVHNC>T&Cu$nB>A}07 zMBu9kn1(KXeg>Mxu>}%TY;;EV@Qb%FM~hd_?=q&ie#N%A=$PzfVkCk9ze@rJp^@w| zy~-Goqv`m;s$7*++Uuy8?H;77ucU%xxl(+tTR?zY5Ru?zbd0q2+tQddZLZAF*62>{ z-fc&F`5qlUh#_S4gD5Vu5%`N+cS7l6OJ0no_6i!*RSM6aQ;7xTyqZwE>GN?&tfNsp zlRb-yz(6Hqccq*f*%n_`##6qWDVvI5N;&jPbdk30#S}Iy8IpHtu#V)DLI179>fXkw*}vJQE&R`Of@WXit2mDaR_^ zWK7<1;)C)#)hEYGlRU%BkLD3Z|mKd zEXo>SV%x8IChL1h5Yija3MyCmfE}9_quW48{52sj@B#~!{PF@kU#)(-{;}6#gLr2Q zy`3Ey;&E(uaF2~A@0R*sU4EhEv0?zXosLe(>q~^!rY|If)NuNjVb7gA?Y+HsE)eEx zS2trAjYc{Zo#nrE_DBQt1qkE6|C0vRsAJN<4j`WC#${D7<@(=ex`ESe(1nNB@ot#v z)LIDZfpQ)b?lb@62Z$Zc{1Gk-xMa$-DQ=sK^NZG=P{m$gPi!B$f})3g!Yik`YGO$%;Byv<`bPjFuu5?*}xA z2hyR6&>cu!O0WgOxGSDR=<|3P|L~M(ez#i1Xx+Rvva5^j%&-- z{AFtO3~^w35))YYk7TRCbjvVh9_@U)JJh8U{5nLgaNb6KtTg*(%FQKU=@N!mI4-jJsT;u*Iw|j@iIlOEx2;3MpH68>%r^*>} ztBUXFpBYq@t;IWlaKzndY_e%V)q~1`$zP@xnf7u5o;7*vg4k9AAA;u~ zbd7!4RbQN*#_GxlHX>x8>ISPj6t`--2x;$2lO-qNHJeJc*>^}-%TK$lXipC}ZOjeN z@4*uyMV!`_&1VXBTzEP_#QOp{#Rtw`z1h~+JrKk#4-#j`QY+m%84L}mZw#xc8lR@fN@hPkWOHL6TE znFZL)m1D5se_qp3IO>g*7r`H7;?#k(h&+3An;-txjJX%qwaYM3Y4EOJh?-Gqd)ca+ zT7mndtGZIga@FyPAgIf0?h$Uy)6_WR4i~7!pFImYqqb2I78N_(nzA&o2lE>+JW_CS zxjilmctA)pMI&Z3%6l%suM0b2p*_A|zqOW+d+wSQp!O7{Jkd`~c z^2=`C?ehM?Zz11^mNfjSAy$VyN8B{q5aansva#zRsd1yZuu2QCTbvSm`0yb`5%`v?gmxQ;ug928`=#5w5ee$L&DJ0~L-|%d z&@6s)Mca;ff=eU=M%m(rlg4t?6-P}jPLL+9{-rSW^#m&y_0a8S&8QxEoens>*UPBh zx&~@ojL=WW&1f)Tg9#$Z+aIJmop}o*xLpyA=;uu}|59lWtUqdHnz!;)=h!5knS`Xn zjIubO<<_Bz#3yQu#D2?XQQ_pxFB6+x|Egd<(_ob4 z6dld{>d^QVSSqJ|&!~vc7Zf;Ff&&Lj)X#!&NCeErBN!elZdrP{3JWzQGT3>?dz7YX`PZ&kz>g% zeo|lCRkNlRpS#R!F`?@r>EJ~h#3xyFDB#0$~AQ3%ozQUs61O2C)rx8eh_@A8NY``~ zbu_CQPbcv$wTo~0ymD_IORumi3Cv-aUK^^@NDV$DVLo)xWwk1rXMOlna!7(=4pgnL zg-#bljb+otK6dehL-Fi|$eB(v6a-&gQe_)bJa+SRT^L>*Ai=Cf*{a#Tn#?RXwgtHt z26Ic0;x~!#_jc|9FCBcE_p`-7sFPF_rLI$H-F>{5E0M!QAZzZK%K|$N5npivIoOxQ zl6i$G@oWy1MY4Gfl2Th_45~C(Kz8YQw)0qU2kSlswi(je9)*MNo}mWFKcw)vjQwuQ zV}3__5g_&&`N0hRbMq23B3dyWUfUq-h{uqxg|Fj`|6aeY1tR=hxEvZzi;_l9F`#jr zoD4!Yz}|8rFv3ny;iw2UhRwhZkbWc-Ln5K!ywv3$1MMb$C|KxiaeqaSJ+YO+%wQ3$ zJZQ5k1-al@g-hg5R-qrI!Jtwe+^U4GH})a}SW*1Y>NV8Yms1bVH@eQ>(E~%{2cImy z=`IU%p|*Wz*Vqty0L5Oryg*%O3cOIOd*(#1S&)E!9|tJ()DL#fwi`oom}qI_iChMF zKet0Mk&yRrzk{On2R{+9izM=x$cEARPl(X1WL{KGuBhC5ZC@~n@pMx4qo9PW+zp?) z{u%!n^3JJ1RbRvAkNPCQJmr~_-IwpdmfzxNwUN~x7*9m72IDc2AkCn!gi3=Q1zDnL z1UGsk`}yt}b^Kq^#Iahyjc`_8Mkt6m{^Uj!L|}J#UXt7a_OpM&j0QYJer!BV>FL%! zQ3}j&j2=eQBcC&|!L#S~E(J5slce@uiW#FUbti2P_pOiGV-_G0%TPRkP|1YFjT+vw zFmMmI?pN%C(nP|&*PMVDgH|pX`FBUk$vddo=oC_cw%?+=|AxV z;X}9TkyT*)=f!X1-8<&25!>ku9yLC@l3|WOglW7eMw0&{Z~lfYC$(Hd zW>JesA+UAEG|gQ4m2Hfh&j{WJ=aA$hAEauUBaTNO1OlEnO|e==CJGXgj-h^cLV1V-k~QYeyJeVr4LXM>bUXG=!%Izz$2#-?jSxn zKWsp>h}iR3&-~s2Ux>%0rC701G~wus>{2-oDP|$H5?I&-jE0IJx+HZ(FO3YiLOu*$ zTQ-t7aB+8!i+P~a>`zQB!;UhkrD39+&_p=UmBAi;P5UM7iFKa}2tkqP^?HjU{9coy zWn)4YbvM)6UbfEyzW{Mo%$SJe@#WJ9EHEsUKjWGMlE>zQ-B>-hV^*-m(wZPxqB}1z zeLbt{1Z+Q*+YZ$=?Q;hCLO{t_L8qU~LQOg`1_dU%t_Ez6lbQ5HdFi_pMjwV%<7F7| z@Y=d&+%rt~soCrG0-3EGgpT(qg$(IqemMnGcJO&oV&B#tgMdf>q?9&TiisYtU%D-Z zUOlSerQgLVjW3jmtr~K$Hu`QaWWs~Vz8nAjg8l8d|Hx?K*lFqk6t?Djz{mX0v<*-J zE6=fOOBC~;xs3>3C_sRRrUw=JtD9z49V|H_Q=iQK*j^&y84B=Cr4b80eLm?vFI;Lo zok39*EraYs)4#stoqcr?dBUJ#2AX^?v5ELi%GE_#Po5G}gRs>lFof)xK4Oip(I**7 z1Kd+~7^(0h9{cZRzeDIu{!}f;)_P_!67y#~9?Lk=Ufg)jaIi-1@0f)m0d_m4Pn42>4GA`zBBCo2)8+oX zzmI+r1M%^F-a)`Dm<`dU9MMygm)$omXi5?GHXGw0^W$;9jza{vo-`H}O4>IMDS{v) zyjH;BkEzyUTUl^UaJ!BQ5I7OF)737^@(9O}t=jE4l$!)!lH-7cNdarBa++TN7EbNL z9Jf;ASPU3|-&15QZKu0FS%YMn7zC8BYtPq>t{D1-I^Od&)|`(T`Ny)Gw#r752`}un zk;hlqjMCbCV*+ihQ4bISde6rS4&P2DdMj3j8qQNtj*OO#N|Hj9M)m*<)A?A-fF|C+ z9f~j!a&~r(Dszl`9`FRhnbRBR=^~=?R09-^*!JYn%?0xkp}{&7pMFey1Nk8HFuNGT z;N`{HB0%0%zGl3sQxvA>x9q)F4a#-d_ER+o>!y=k882@}4yK!In#R*f!RdP<6W@WH9_FX0r zl<{(PE`%KP@XoXYoM~}*u?DVN@syu&6=)bMQ}?NXz|_ASaajs1?sY2tiXMY5P7I|? zb(JU^Z`Zi_i2QdNO^5B)=|}23 zEu6#q&$jP!Sz%juuZ+r&y2WS|>Fa<(I}AsklWU2U%wEEC$VQoGuMr2c&vK zJM^he0s^Ha31UR_R~&9)8QvN^`++66!A7s16ZRz$1{!RfTF$`)&H|(AnZtj&k}1{Lav-UT6F;ji-JFd^v=s_cY zXi6s=_X9$wd?;X`H7p18@F)P!lLbIk;7_5i#l~z-U`^u+xvr?Ljua+x+SPd`qyIS1 zwG4s34ij9(fA6+FQZ2I&39_a-PvfSM(%b$Mhq+g!< z>UX}|mSL$`5hyWsJAa1|kS{-C$3Rz(W;qgWvUC^`PIYMpV3Bqs{rSR)lXHOSP_GNj zE136f@L1BdYj?`!B*irv@$y+%wTkZb4J?GT9ie3bwwsYFul;B_)t^O2N2fOyy0ZtW z#1rj0_`b$QeH)WL`#l+!jtA~B{#R!+y;&zccMVfUX9lwsyxmvp0E#%U4W0v|%7w(7 z3e1CI0S(Annr2Yz-T`W<$$h{GE0it;`(Goh2NSoiU)?tj7pvpQ_b|BUcRZ&oTujG& zpnUTpU&|DJ=;Rw9>Ld7+uVoxPaEtNjNC7d~;)&L}O(AS5v}TgI*%Fz9b$0zmxS=ls zcS|aR4Ge=0DOZuTiz6@5LLRGTOUE~aSMwQ<{r06<`-BU$EM0Tgl)d!2-^q}zlJ!^u zDL*Y>nzL9-r{z}M&IWACJ(2T!xwX1ZiAk^vil(>tus`Uib85)64XHSO6F>?%iT&+i zWjU|{Jghdp&;9hU%8d3CSxPX?kny~2W6%IV4568Uv<``8G-M7I#QBjm73h?;R$qKk zr~w920KK3YI8v?yjxl@PoPCnCT@KUjFUYQk-{o~@ajI?91^qrr&whk%4fueDHwcE+ zQ69ptXA)qQ?90PJaR|x)z<{el4!9|AOG)EqSpjXepk+}pp9-3;Wy_D)g`vkAWqks~ zL(7^6k{Q#Knwip{1@8^&=~kFt+f3(5f(vui=z};lfd=u!uHw3#xTLA9OvOuAufZ3< z%r+;qhp=B7cB2QTe=*tz7@T9(>dLF!dDL#+1|Q3QzL~2%u(eMC5IHFHs!fX~*$NpOa@(@WA1n~EV?2F{t~LYt8FQcDo1)O#OwL~IJu43H|7my0ZDebmTX zdBowirjbj>3F_bm+-t;BGk#}1o|RH(k)kIH(aEd8uWe;xryQ2Gl2xC!$m;-yZ_R=E zZwOCY2P+h=@6A#kGX#ZrWacB?Lu%7lLvWR6lX6V)##6d zh?iQ1o;2w-`YOsP?&H>_0F>14>eu3$^gWvDbrRbguSlLyJ@BjrW;W6(xjozx!>F5* zl1H31D(EwQIR;VzG>I#SpR2$Z!!)-(#&*l_3tpw~}KluLy=H-+Bf%$LVaMi50IrwFFPX zjc0X~hL3R9pHWWY`vej;A)fuC7TCmPrS`GC`R~M#f(n8nm<)Y~ja8&UzV)q)+EA{z zMpE=TZi=+G%)pjGeN>9|0O4+Rv~uyjCry1tPre7$7YLEaBmOsl-GcJLg^| z@19I8*YY8V$9R=f=8$Q$cby-LUS)wL>r*dH`P%G5d9NJ#nYlS-uRh0uiJRqbx*b)V z2KKERFTDu58LgLZdRcTQYZmFRlHs4+YcPMnLCKZY6?t@dt)5xTLG-n<*WJu%5hBLb z>nU57iNjqNKGN`Wue;IHFw*35I_4>nk|>dwwT9b+sM({=nKOuZssd{KB) zJPIx%4x5+!ygi>~mmCU?*B2Pt?a80&sbBYa4VxKYS+6(zp_gzGVSo3i?A4sM$)*Q9 zS?|;~Lu=4Ca0_>me%eo_FJfKPgJH!_#zDS+BGH7V_#$GOr+~Eki+oM7yHRkTLu2NI zH0+TpL$@DYqw{RVOsJtH?aTRCtSK2U;U=Smy>xSft~&6B4Wlg=H7bwPi?iI`jIeLv z75l8%6`v+L=QhV+UEEr87+|zcE^sRoTy+p03Fp&2sx8)B)DnTqSH+#IWVK-o^bt$> z=e8IwN*vwVo$ktIr5iQPU5POgPH(f=YQx%NBsV;GLDuDYU{4q!70Eqy@M5d>fOjpn z{Pt+gWb)wS+mq`D>z4T>T&I+ZzD=Wyt60)TDT~VEqjl4a>#?rOuA>GXiziPn&rkM1 z*s(}lOjUko1Ql<}`fy_`1I!s=>{alDz?5P*Klx@}2P3Kf`EJV5s&WFGxCu7|UY*v( zQp?o$W0+|qu+rTjzYnK+2-t-Y?FSTqg?SM0*GEjfg1#oC@RgR>@tz{Vuiizn-f;tD z^BK{a$Gc0_fDTUfz?SUG#MiC!gagsCSK32ycWI!g0k}VLNAUIm)HfyfQ7)?K_!G&3 zEC&#N4ZMDRZDPALp4F{jl%KqMen>aC{vO}zORcs=)bWHe9K5VzaZGqzJ5uS zwAeG?BbzC+NHTIXUd1Tst8fB9V;3dD%QRFY0NVQ>n=WjF7?nPnP9?%8f~^1gz3`r7 zxZM(nSDho>MQoPS7rK6nv&H@wHYw4XUTm9w zC1$P^9+OToDM2GE+nnrmz+rQlU#r;7##HMlb}Oqsz6Z+LCOm#paAq<^SBvOJm-Hq) zkgw?*^=%ZiuKs3vt2k>LHXzWN3_MZ^_p_iQPX_9}_9EW1stB+6Z6PDH2DtRqUFR9vbrpDqjTsp@h1 z;|;4B$o{tmdY7AHuG`rnqQsQ0o6WLd0()7c0roa!;_H;|ZIvubW0RhFkruKP87yS* z)8$;q@<4Lt87kRj?|MO~UoL3G%Apd9Jey?}GGtULW@I?6A04A5EymLHGI%_=UDx|! z^_j5jKyr-MASa#S>ae;>c=fz1`FY549r0TR_H3b^G!I6+TpMw_de&`g;@)XbNVmYp zw?8=5dDb27dL7VA3k8pDKa0ICyb}r|1FM&fOV&lEeU~p-8K2&w8v|YXU|{$80hV-YcGfTzcBP$ zb7Rk|!mk=WmKU_WSasR+s`uBKJ(HYubN<=MJ2#7;s1)Ba+3IW7@2j(G6d!mu9`xTQ zvg=of-@2FgZchKH9L&KInGki(Cvr!^#*$rcY%I5>{nKB4J(u?k64K zK-}i6xxR_Q+Q7*&rw3;PV%Mo_0`IH3Q*=$SBz0NY+*y})oID9WC9+Yx;?vD`Cw|v- zcF1amf|H`MO9LlQUIM(LvF?mqC2&8NMq{=79_Mdw{%L0Q$TkB_Nn;lWE?__!uki#; zL8S?CZG?|I0*7*dtz46gH#huTfJ1b!ksaVhDxjcEfCS2-sWQ-r&CR{p+iUhGPJZ*Z za=mO3NRwdWd-J?$y0hzhYSMrMK!21s0B^2cUUzT%d-4A%U2}joEN{5>6mpJ?<%+U@{aO^*uXvCTJmGeyV^rHK(Cx){ z&E$)7Of6r(Nn5zxBpiH`AaDuLXEEQUpC(R)u3oTMSH`=%Ze4)ld+GasXT*PMahL#I zA!dKeifLEww!F!Lz~jRZ{%-+oh1HP}0M3#khG>D8H~~ktRJ4wkAgNSHm;)Sud~C+a zh_Xv-5ip>HrY2b-kL(!ufmZIczyIsL?)|0UXXm04WQ2ga9BLZh#a5rR-TuAbUhqC} zX~|Tk!}la!UO69MxADlSADqB@V}!)Pi^JZox-B&S-~>(Jcxuh|s<}S%jl8$zJbYTy z_Q#Baxv_;6x%BynNELvy6`2fC^I!h(%umjzL-w*)3C z;3!^ek2H7`??@QpM7U zS{J5TA-i(|Xft#>1 zF0hmwo}7!6a{d4ZPml3OPqaY_T!8@K;3?zX4=M0N7l0|%9+VX;>n^Omv?4G2XG%_= ze9TE^rbeNRp#5uOuI$TR@X04lkfZCsem0)6SI=i&e$J+P`iXGk0j?6O=UJ9p5E;hu}0dW7Y z88Fzk)&hlz~ef5T#EC$Atx> g!3Ye{8)_f^_i2d*y?pb12?G##y85}Sb4q9e0INm)VgLXD literal 0 HcmV?d00001 diff --git a/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0.svg b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0.svg new file mode 100644 index 0000000000..e3d3665349 --- /dev/null +++ b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0.svg @@ -0,0 +1,2180 @@ + + + + + + + + + 2023-06-14T09:59:51.356040 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_33.svg b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_33.svg new file mode 100644 index 0000000000..25b101e6ab --- /dev/null +++ b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_33.svg @@ -0,0 +1,2203 @@ + + + + + + + + + 2023-06-14T10:02:58.661558 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_5.svg b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_5.svg new file mode 100644 index 0000000000..f0e583f0ad --- /dev/null +++ b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_0_5.svg @@ -0,0 +1,2238 @@ + + + + + + + + + 2023-06-14T10:01:47.825040 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_1.svg b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_1.svg new file mode 100644 index 0000000000..55ba22dde8 --- /dev/null +++ b/doc/modules/cassandra/assets/images/unified/shards_graph_lambda_1.svg @@ -0,0 +1,1943 @@ + + + + + + + + + + + + 2023-10-26T11:50:35.670645 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/modules/cassandra/assets/images/unified/ucs.png b/doc/modules/cassandra/assets/images/unified/ucs.png new file mode 100644 index 0000000000000000000000000000000000000000..2b193e48d7681ee8d0b5e241bff79b85501cba1e GIT binary patch literal 46473 zcmd42WmsIx5-yCpOMpRxTaZ9-4K6_f2@qTcm%-hF1PK=0U6K$qxI4k!3C`f|`mOA< zv(L`??w|Yb@~nBLS*v?>cXf63TW{5bswl}|qJhw0U|=xiWF=q2z`&!zz`!1%AOUyw zlWCk_V9@d`BqUViBqXR*9PCUjtidoavZ3(_$m+4Pcz%bEg&zW+hzVr#Za?#j8-ZtR z3X_+G3&fM4Vw{QTBGgn>&-;vC@6`5&sUj?dv06$M_Vzt1+)lkF4q^Ny{D-oJ!^X>@ zR7d*L^dAd#T$c;CnXrC}Mkm*DWwbCq8o9-@3ejH(QGW^}g_rb0M4&<)NG?_vSzKBo zqN*@|WTQla;Wj%24Q||AK0bbxE$9Et3Pa@^Xgu?|*d%b6>L#s=C=b?;S>`TzX53Wvl+C@q2h(STP#i6*&~_VoCajQtEv%jE4Bt99ziWLuG(D!H#- zMEnW6Z8#fp#t7`s8776zV-R=Yr98XPbhg6>2w3-76ftnG*Z4719|o?L%cPW`!6on3 zCpxJz^NBTL^rsj}ApY90rpR?aPkmOWZ9JlG^{7hCjcXT_;Py4)*GdDxV zdzqhvqbejiDD|C4^oKsdu-pm3wc2=qq}6yddfoRaeWzelMY~5SB)wKj6>-`!m;yg^ zF_4+@oDP1;?llRVt&SHcKC+PwE56N`guO5&HANeSh31LHIgD5aQq~%|0;8<{Q8v_vqgO)=t``O|3ys^Kf&*@|6*L2H zwv5&2PlV+Q{ETkM2uXRvpb=r1SyHzNkMZJMqg|_apJvU^B@)ueF%PP9LE_aD$!?9n1KWU_8YTSErrpKpwv1+mi43dI zkE@e^`3|C}17btgcyHCJ+q&2mu`06Kb!6%0*oZ8GH{4;i5`N@-g>^@MN9+?!B{xDp zgL{W!{SqnL50W;bR4YYHH-M8LQZI>Y#KGxb8=$FZO{+&QLf?#!k3Wu6Nr;ZK7G^`l z@E7&{oV-nQO09!t!VVPp)W?}~5F*~=p<)G7GbX8?8r z6F*0N?*5#?!~;wZ&IdmOOMrR%5`JVPl7Eu;vE9qL{%IX;UFipAueV-7#mDB4mDZOP z-C);*SN$wM1p6J=9XC?fZ#P6)R0tOc@dz)NzB3D{O0s|$kr-`NgmT=pj^jUl{M3Y} zz@)JqHzonroE76&agROW+(_+HWB$UN!R!@}7q7rvs=lF~{svziyU3+TsYrE#Y_ir8 zd4jE|DXZ`G_5`gZXH}}e=aS-sd^Yt`mBTlgqa%|jqrGG1V{SQe6L}WhmJ}mFZE!Nh6yCuUd`XWjTe`upGNo>FdJ` zk++!znMJ;ukJYf&24EaJ6;nt*4$pUFATtaEyrJu(Wsk%{R+Cql{mR(u(|F z)LZeC)K$N=w;7u|-KCx4oKBDImi!xMz)Q^`hrhr(hRRSIcU>gF>AhUL#&4y&}U6-CKqs`XIU&bZfD>vBM{M z?xBqb=U;YtOAgmaPaLl#m|U2U)#wpL-hWYNbbe*2or3Y4V3A-4wFMumP_Q5|-**A0 ze&tO1b3|hVFBish7qgFG50i)Qs7;|&pRFoyHpx>z)Qr@qTL)ElTkp>m)IIZ@-*Oq= ze<2tmi068j0#138bnas|Gjw4czqdb2v*hlhkx7$5(@U}dZrR1zTY)bnbMT9*|GCl zc1<-t*Ih>RvG@MiC&)}nAzukR%DGp5W;`7Af#UEw^g_)h-sp|q}8=c*b@#pJLwu86~#_F;h%I)aj8H4c#(W! zD(`5EKJjVv`7CHjVB*Q>lF|iiSmMDFr z&z1)}&OUHORf~2D4S7>v=WTsH8Sh&<^R)t1PzZ?b@pzqTja16e6vrgC6h1tFxo`Qe z-A9w=H1}(-uj){vLg(4vwh(*&@Z~>chGKsf?bF|F5;-kA&E#e1V31it`8m)$=v#PH za?`YNTf+4CxsdXc=zt>BNBv6kH1y&#)Nuw{o{Xsv(GP3c@wIu}Jl#LGKG|cU?0<*cjgd6uGnZb;n|H9Lw(jSf7sb{W>0u4u4Jb{KA0;pZCWT02Qa zpu@$1{o}g&AQmVa;)n9bB}N4s8Yl+3VTGfn`qu>$iXe>e#~ms-L=;(Cg4S$6wf(B8}tN8gzzJw*aq5fl-)KoE9W`7P-G7AQnJRIMd)E}dl6r&a& z{$DY{!l@!QZ!6THtZ!_r)Il>kf-y-U`&H99;QoY-J_Z82zTVze@5{vx>+2>uu8YpH z^73e1rf=O-MIP@Q6;pXIktw}6lmz#=we^&guny{X*>0~Z`q?BtV3D8fmU~{>dfo2V zB=Oo&2vI}{yRHNgt$v{NY0Z<1g)I9cuNGGh^|go)i+lvVK+~?aZp#u6fDCZr%cs`c=4tpi)M zvjW_|Cx^yOp@#R5Nd3p~idaEu&=81HR)>jXXQ+hYdHTU{;#bXujsJw?#`$6 z&3`st>%B2ws6E!}ar#G5vP@AXl4P^|@kvRwO}_H4E)={X{cbglMd^FT`S#m8ur7mu z>w(Vtxq#*^O0iy3{l%O5gbBWpsu1dDNqD-LBptc5zts?D0l`v{HV&yD7MaY?ey#hd zpdjh)bg7JCOzRr@6V}#D-v^6;V3OV+Zo$q_5KI&xR;ZM2x!b-L0eVpZo7Gr&bUx%Aw3Ix&79|?FKVP#J zh_BR&D?_Gwg?D>@*z(w&YS(OWdVlEq7>#Dn zJ(RwFY!bqr7oCavJ{D4oh9AgH=Qa2s$=(;8_{!^MKQ4}>j1$i7`;3M8$6Fhj@MhoE zvsGD6_EpESajB|;J-v8s7ebP`%nl|Gfq~1k_}uR~#_DsP8V;tsa0B)P>usuSJyXjg zN!HMV5~n?vwd?NGw>GgAKytG~KKc3iS|z|PqKp(e>4md>YogG-Snu4y4^1=9WDp7W z!kVQ(Y8n>~fnLU=A+uGI7az`5S)JNB!=?U+xPS9{`S~t~)fB7XSc`EFRw}a0<+aW+ z3AFM_(u~Z|xk>?{pC`o?`^&FZYwm!+oJ=MH_LXho{im#9;b*v996V$$#qBf<-`%3W_RJBnk!qXH5tMvq9b zYrgItiX>pbGauow=Ly=lHn4WSJ6*GCFkPa%TqLAleA^pCgZ@2n8i~H% zT6G~BAM;8~RlJ)(SCJdWf?GgZFsgiCiG$U9K6yq|ksD6meS>fpWQt92iX95h1&F(dfLr`| zow_eOwi9W*UG39V@;!C;0_@JW(fu>rF6K;xw_4X4#gC9FL`G^Xa>kQ_z1 z-$zRpm4DU~RPHRkm;D69NpiV?x#iqL76T(kqLo;HL+dZzkrN?{yX4UwV6ibe-Osp{A)>hI_&JrJ1lr z(utOFa}T+zj!#Bp_7F7Kx%Ot~>uC^b(34E@Ax+@8bxrWj3YpgsOy*6*22++FUZpt_ zG##D3;iC-mcU|(HMnkCB6egQB6TBj8y!`1|UFvhaK}6<#HhSUm(B12o6V)O*^poFa z4TC(wqgk!^a%ZxC%Ku(bb z4(}rc2mfl!dEwCtE1H>?TblP!3uW4y9O6XTvfi_(zi}1;l}f$g3|x)+JrRhPEGDl_ zQX(iAPlh~Qk|v}Qkz3weQUJ~$YVmyxC$C(B=Dk(BU}!LsO!RhN+r}Gm|JGA(%Q_@s z{0QR69Rc70CLd^1hTK`bS!~%IM4^)(MDsQO6`4&zwuEs|u>4A=s(E$h`&QH=rv+L! zc76X)=|=#opo2d9TWuig;a|+QRo6Tyx*eXb{LoiY>p`vjgR(b!D}Q8ny>gnSz|;L1j7@2q zsUTuEgeEm~{l%luAiLLC9}oh#yd;ft&6+G(HH$x#IvFkvshSIaJ&lGzM$!SPkUF&T z#d<3VOd*Am4*?@Xy3?n05}MMSRh_My7m#JKyrfxVs1`VLJSc2`ZBJdxr$K4fgXllN z#$Sa&#h43~Y|-<$i|BGyK~*e9EFhkymRm{sBXNvf+zAqt%#6r9k4WimIFk$U%o!TO zb(*UxL9-I4ro`ms{@~ivy9NvLQOe0~C*boqOj{Ggq-J9xaZ_HtZqHLhf8`}jj5NEHyn(mY6EoU0FL`E9eW)e{|0moa9o5iorZwuV1F78RGIQn~3KinY42j(=!;H292=yBzt7u;9r>JyG_F&?1hApA_#3z`V4p^Fii(jCc| zR>`Mb9)wDF&&nx=_DF`+z9XYuwqNoMo7oGY>+3JL_UuOvC(nmhSJ#9-ypL)R& z!(a?XV~}AyImBE%NFxbppWbfxTc-nA{6YAr+&tl+Ye#|iNW_mFg}aJ|G||CNBeWin z^(SP7P9Y0%e5;D8;-X&kKRZSx5(ZsCD2>6PM(uKqLvlm1kgOHxFEQeR$fZaE{HGCp z3U(2^rz{A@XPkFr3_G>(=x<`xh&k-5{km~5osk|KcmjikA{fFFTeWaKB~Tw#Mj(xU~t<@u2;8`b`RcpA!QIni4qt&fuHFnR|I|-h~nw-#mGf_xfQaM}~T|$?c z-yWJMW!GYdk4~LKKKZO!3vwit;INoM3-aa~ag%=!S?(++AN(+47~@nWgu1lV3T{}? zs6P}NMM#4%lOMfvT__-^-$0P*H9EsX@U_>-vuO9DhK{I~XY^3d6;FlR#cR^GdP1Mg zG$AthrR#Kop6ek;sk&yw3fCKvBN6W>f2$+}P)Tb4?q~2%;qfD?fxY(wbjY2vb)ynz zCv@GYmaxoNaCmxx3a0DcD2a)5dNGe&5Ar=^t!p!6_mg|!GH4u z*Skk7cOE&yByvlDjdUPfnhV}a%N30VbmSF-EmKaiF28j7X@^*op}tRr7V}*&s7FxZ&-}-@d2V@h>RM*KK1kc}00sX4qjZ#;8kWjv(l>hu)}jKf;}j#KU!JCw;*U)^Vm{wG6Uqis6J-QeUriG z1`Du>i|bp@WVD!=uaa3{kGTR;(k}!y2+oJJP^(knf_{XGX0c%7lf+`KaWVeXM4DGN zRQuFsYFE-K%H4f!4dE7syvbUvV725bZ=ZIc4?U0@NaW(vx%ou1rqptJBY04o+p73_ zMd-_E!)aQ%&tyy*w1Jgo2jebZvv9^t2ju*e8iR!;o{<04ypc@HZZ97b&Ba#J{pC(9&Co)PX0uCuRfyW*pgJv=< zpYOO6|I*=-(USI)`>6+geLF$C`2?YBNzrb)F#)QgPJSqOyf=qk0MiMY=4i7l7~$HS zR;_ji0aMg#t034DQuJt|H?kjb9YXg4*PgrttlxmWFts*lYysbIU@M_hJ|7u=oz}h~Kyxcq$4S^+{k&PQ#FF4gzMRPq zP0E6&62UV|<}0`IsTwP{a4n}K1r0d5$?|=}*NXzaL~y4gX1{p|{}bGzT738GzGug5 zkr!hI8;loM$7`1kxY^}0#-E_+s-1XTcTn2;2##k)|YlZuQkF*F4Ij@&5GF=z>5|= z1<#x3+0*;CXBUYbd&j>?IOYCm(dk$Py08P&npFpKu?8W+ow^!{G3OSKZMJE~tx6xH zL%(xZ(&j%a&D%{gi{Mg|y3d%w)}fA97=B*vGi3e2(azw3KsVCr-SY8?my0Azm$iLk z8k822Uvv_e@A(gqf1tyOqxR2Ta1zYP?sJ~vPa#h--y6-^2w_QVW3?6<@yq@S&+F;% zR%YdXD1<2jt*6)iB!9H~#eO;gUxgy27XX*GgSudJ?Q7MoRJ-G2HS7<`m^Om#yr5|E z#az2x$pm3k49CRNt1SSw`R4h_QRrw97m0g498b)D`h=^0{dxKTpz*~Jun~dDDqK};?$!V&N49XWQx|*5?&`e*wyya}zoAp@bxnz31n$q=;9F>P?qAUbr zX;sv`1?_;-l`!hh2&ap__zVF)|{ z&BsipHja_<4rUu-G-#B6pHgGQC3nb|{~2+jm$C zVN=2wdeni<4iUs;VN;f?L8~y>gLUUShhm_}cqDJ-tPwXNK4)4Due8KnEMxqnDV^17 za0h#u!>K?M0oSJGUKq01=hTa9O(4TPF-rl8BQRcwj<|32xxQM9`W9`V?T)FTxf;T= z+fmOaZ6+6Vxj#RhvB`=~BDi3#BXWN>J{8ob^)yskU|7T1==OR~B}w2syR9GWf6eL` z*j%J~^QDMLss?}l6d)$%1Zbc=e1^i3?jh^X1Yn zjvLnMdzuYA-P!fK=t`KSa6207ieQm3((bMVeE`>a@3f7iWGBU)siZ`J?QcdRTas$a z2#>6m`@y`n5dhXxBg^K8t^V-o<5Sov;*Cx+IgHC#O$69vnE5KpW+fLA9n*3I6c`u< z*>c9I()5@FpvMKB(CbsZ1GOW7Y|}lgL^e}%Wt%|*Cm$uC?>I3m!Qx#R+aG5(oR#(J;P*_W?)&8rLGmaZ_+Y~lP_kR%7n$}FvD)r7oWx#+qvkir4cD|+GWKL z-$9Z%MjJ-Xw%z9g9hw-(45pVky;g(-2#MIOiv=XH80ebAwSIj*I{C3}hX5@NGnd%d z+dOT2Fg$fF#@Cq+P3-~eYELc6{z(t#^0l_np16S!niPy#%?$5l3JIHD$L+PELx4n> zDMjv<2jG|Q7g((iERp#3bKfx4&L$@6x@h^o+JKu^M7m4oQb`$ZcN`Wr3kn|h^(vg% zcwm_P!BQ6a58Ik8n>7aa< zS6vQSb?D?C2k~-j?QI2%CM;jK~&)z~7%b;9s6kbW-pE{{pw&1b*?Ja?7 zo{C@jx5g-M?G$JaQ~05izHvsH_^M;O2#qUJHy0*1T+IGD2BH}u-Yf}}#nQZB1@Vg_YD{>P)7{+Dgjn^nGJ(c>HUQ7BN?L5kVyq?%s6CHgS8! zHk}!WX@k46q0b&E(rGNJ?ipF45R?R)HE{#-_MjluwgO2s`S^PMw8M?sj6Pg2hKqfw z4R@FI@g+^`%ppPTK;keC{%t~Wnm6{I_$fBOn%HZC*Nh|kA$K#%UI*!o>ATlMOCs|> zPQv89Hq);_h0RjBg~or=-8dx(I^8qhJeI;B`Cnba#h${Yp@O(fKfWm9f3G(a!tPYp z9QNCqg@{Gq6p8o-gDf)82>&5h|0*!D0F-kD-uU?+EIQ6AK#sp`444%9eNz5UWffRB zjCQaedw^5?Kc|Z%l_n}9gkRHB`ux8PF3`>&M4go!2!bm$lKO|n1PVZHSy}|E;Uc|1 zx>k_@y0%1V4EtlC{sweZF*wD-xgMeaObRH7Vd6&17Ck*;e*2r&_^XHa@c`pqcJXnC z?l;%@7bX7!VA8XLTfb`lF)5U{K**qNj_XU}Z#4K<4-+_mVE$hov})Ts&PfG8j@+mN zDKF6Veea#iBNXK2+w&9k!mh5aj7phyLra@)&>CD1yJyWZo2I`3;WIaYI4yUC4#7c< zjEqJ;t1*!I-W`WTMzTR7^%L0jBe6yAaF8k7s3L_gW}1ekr{I&5ldm=s^^Y!>d;>B` zd2K(!!Xsr_O_x+v$w^J6+N23MGuzf}Bv21)tS`3*p&wsEGuQf9N+ko3Fi$Gw0bY`8 zSSSFnIYdlMOxHbJ&z~QijVrHBs3}WDQ3&tZw=CI39bY$IZ>HiB=0BF4`fmpS%UXpq zqWH4P;O$dDHLzeozv$?)(w2uCbi2l5Bznc9XkY8&)vn&bL1#AhlXU6v(_vBl^|~A^ zrR3*`1ZYxpQ9S{`c$YHt>m2B=a$v5Gdnmuby06E`7y(qG=i<$jw%tnAv|h(6-bDnE zLw5AkH|-CxN&=|73lMmK77gDXFIBO9!xLXJ;>Fv#78X^}UV}^?a_uA4$0Ak2oIgPL)~IX=rFpqfTekl!b!;&I}je zT(uwUy?@1#{CB9cvO{;Mzc~QAht(e-6+vE{qgNE%=F#NhP3KeE`{myE5)#g{6=3tl z`txSCXXEZuYyQX-jF4fV2gsa7p%yHs*VWOTBLN~==w|t_$Sno|fO|21i0YF9-sNph zI9ml5U)r|Conkm+;tBk~s|O~_p`S7R47 zZ`&|f|Cr{!nLKs1YA%X-M(J|}J6~hhLlY$g7rEeIO2x@JTF;tp-%T<#;(Wrk>zOIy zz3fm|=5f9QS2L8#i=E1A*A2jtD1OJyWgv#|`FTt<^pv_Zjf=1(1Vyqte#pjZ?z_b@ zE0&oKoUUQ-C);;vi`xKUzOj7nv5$eX_$`TAk)OavL*M(w&g9n*QKApTxzZ682RwTU z7lfElz_EfqY`$gN1en9**KQc3JOi6;x5EOvtSm5ztXl7%X;*6=_iE8>W+o&An`L@W zp*@eB;5xpsDyi$f;LMEuMZnab7Hd^@*FzylChNU1SAhNio|E#7=N71&BuiQ3x>Lls znDp=-*faaC7c4C`HuFP@;J*8&oQO%!`ULb>5Kl*kEW_(Wu3r>V(56@9`^!RSp#j!b z{izQ48C+FFE6Z9%86>|(XY!$?5`U!1x$7}+B)WbzjHURvE5@e4^PmoyA=9gZ5SA=r z!|cXxoH2hz5>`CdVYM^#La?hN-SuZE5N=qCTuAsTzDW;CC3pr%gEhKK*L4xB3)w1F z&W#GD#%yTsHu6vHy9Q5x&n&yy`0(lhJe;+%tTy7&>6~95Lo>bD;5vL6irpB>NkXLO zX87>5eqdze{^9oUn_&YG^w=%v)cPPp={*73BIU96TXT_6ZNoi39TLQ5E!3(seXBY8 zrXLsetgg@#Q)bYDqp+Z0bwmQY3eJ9OBzx+9ds0Kc>SdN(EdA88kM0yMGlqQ2e(+^B z^E|gg5EnmNDS}7>3Icxa1OEArvoQts{f#AM+}sh9?$a9tVg}KN@Bk7?CIz zvn-wvH?$fPv^xC|+=x?y3;&PbS@aq%Os?p4W1kL!B@t{UD&i5S=Jceo&$#o6)Q1ux z?%H9|M=u~^d7D!%DG^K4#a(OuQH_ibw#y0M=p#%N=S;$?-=S0&9>E*>D0?T*v#rq! zDdV{A10c#`glX0T?yQTP9&^nMwUsA>Eg01y{9z|4pOH*{?Ad?Z_skmuHo~Kxc%NnT zsPO5q%ebZ)DWjMAhKtj7VYoO`7MX<_-=->920sx`ESJ~Qim~?v@+X`B8Lex z*+Ccn!(@o7fc@T4$A(A7lDxk;x4G96(GA?Xd8=-;_`|O_iEphF`srMBGs)O`06og_ zZJ78uK(mvz%y~e%gDMTyTLs29_E$boVkOTI&8ytgz)|CzAux;V5uLqKM+9JnB|dC@ zqZ6?1tvk|AOIgR=qtiKfGU@!Oty#u;GHa190}k_FIoaDwB4SEMvN@_JUyrFc;>iqK z1deI0E5eThVKEb#A|}%q1g8)MI{sZOoZAF&Mc$pjg$AAN@MATIB1 zZJc)|%4=S8LlWM9CF38~sIgrv4ksA4fAQ?EY>FZ*0J=_a8FKRd|NDL;G3EuX;{ zL#C4<;gFDAZD$AaJvH=Y_7Mh#8Dz^-Igz=--s=RO3o0uj^T6_H>+$f6<+FU*|h}PD=4$fM4Q$*86-c>fT5&UZWQFK;c&2Fjf$vdp;MOT ziPl2tb<}qKRbur1$LwuDC67s0zRLC97-ndwQS31{fI?dCG5LmxGue^epXwB`)@Uci zHv=ao0wwGqErNb#ku5-~uULNf3P zG{tGp3~~+ybl);!5R3H9=j}vU#3(%XIJJ>)QjqEE z(;<1OumeuYIfAdj`A8qgB5vR#Sv6d4L3P*zanh_BoU3Rr*SjxNFh(FP7jsrNRGdM? zW+cRXN^N2iDbr=_weCr48k9+PS|VILen_0W<+;RbeXz#kE>Opp%q7p*tO`o6agGjuxF!YIQBqTv6uy;&f#s4c#R_2D6~(FyLlFw+h99G zit;v$cJoIv1{&<I9&Lz3ny{^J!fP9uf?feA!&=nd4n0h6vke+J z&B4h&^n}cz_o*i$sblr~VX{khNCtlx_NFdwn~bVh4R_&!*l2fpBgywc7~3Hq`(Qu+ zDW_Ys#-X=A?>ZW9F7~a~M8d)YASIK3#Y5DQU=vAfSfI6rV1iPizr8aUmrT^|AcTW3 zR<{3mTQ6k9xKLm8=+XT{dHqUx5>vs8mPP5aRn7meSN97%Y@+lo7H!UaHTlzXl7~x? z`#;u}rJW%2jQ=x-OI%Kj`YB_BRQ#XLVG0e9apk!8rs5Cp3xN*szOwe7ypjLIv9pW> z9J|4CjURuwj5u+C%lN)RTb=2T;T7=%*75&hivMp98ihG#`SSJ{$ZRs|jecrAT|zG4 zwB-kYa>MPh{OhA?yQN5N+d4#>x$oit9<<*p?=d=9sJn)qi~k`($t)v;_wuv=EJXrx zy}F)$AbSr$DMKKqDkL$-Hwe+%*@t+RGdx#*uMu7 ziE@B3yIrk(z$GOOulebrpbMD)e`n08p#FLx@Hdkj0Hk|Cd2IlnEhHpF4A?eh_G{8C z#C(p+v_sq~9CrX=I`s4}|Kl&0{Xa>2>V9JChzH-!v;~38Bn9>+0vhR$ty*U3K>*rZ z11O`;ucgglK>pbZAWCV#OD&FupcVQ7xoMri;G$Dg!+|_9Xxz^~xC$|ZK5?odNx0pm zZ`!sHmR3*-HD1Su<_%GRsNkyu7>_1-|)K z11R9?`&FWK(S?PGc;67X*6Z{TvFU{5<i(ZpQK66LvZcol+T~bwNxI2s2Otn=M$MPVz#5{LIXWJsW zM^ha08dW1BuqbO-Z%g$e$OTuENwgq)13@CM$zGM{_%;< z4Va=rx7B(EuQLPn+A1zCzV=C#0Lvfh7wpGQG?SkEK{V0hc=C{9PxfAeSd!Q<8VCCF zHbIqdzl=DK8#R4qCJPT00L=m!9y?4JAZ`?fG{}41lQ>W6v%{q&AENud@obPI^2JH27(n9j&;qBfQ)+wfd%A(+wPHW>N^3ipwPfbakL(k(N;HBkNh176xRkuL%bI zwU-f^d>EKWWI=&SiysZld;8=5ZU5Wj#1eX<;z=Dj=LbNF+RIY!x?9?*<}?Stfn;(# z8_da?^|?XQm|xNnn45Y4^(u2-kc?>Sr^^q(50g1l?Kz*#>*viWoU}^yz;*HR)aLDR+9S(hpT#v*$YZRGen#tZGJ4O@K7edg*y>=v<9QB9lX@!l#soJ{}DU-;|jh2Zrb-;_Hi zQi=}!Xk&M5JaQwwS5p;T7-i!w!uN+2&gHy<;axRqS0+ZdcsCOo*xKDNa4-;oMfoup zK2maM>XmG&D0*tk4ROZ#SD4D{cim}A<<>tHC3&sicV>8xp5+;aSe0xj!ISL#6ZQsR zs30s7`6I`CKr(vMOe5EDNPw>y=BJP7qpH#bXZvJ9-RIOe#+~o+aHgo+++nuD2{ZL{hOFgx|w<0ws!WuVIL@i&Le7|=Q&?k|+M{_8;Yr=p2A zG(t}(8ScvT_2hIcC(_z%h^@7@D157GPSAkWb7uEm8q+vU&ExR{aLQ-Pw1KC-e`(OC zygf;s4NR{rG~6du?x=C5xvLY&dwen2tP`VpRDBp3Sjo)PA>8H8&(&;3dlX!svN#~pSRAvJ%;dU+Aq^xs5wMu9pUVZTUY@bZn zA~U$;@kDm%xxJf+4quL+%`>`WBNPby zi)D^~MK#I#Mqqq^_^xUoJb%|Ah7c!^jB@v7i!H{>Na@0an2;CZKjd%esbb3Pqz(37 zewtPAFMZO%)_t)pYP=g56gvYLyL4z`lR0R z{+DnU6Nl4w*;vu8%xqf#i|onv2e4l*ou<7Y_EN#PagC7fFiF%c-&(4?QGWseLq_y( zepb4;F%Xic(G$#w--2~RCJ-IV`mF~Nci91=+QaUv2R`IcT72HP{;c*d9damey~#do&B54{(Z$&H$wZ+*5~ zP5cQg$odhy_U3G|JHJRN>V7|*DDPQD)QeKlh_p!sm=?-ez}^M8b4u~rL?l4 zYncBcGAdS76_nz+wD8ZTO`7JW7(CYf8&hwtKidbOkWF<&*xKYqkI{}5E5`ZCQ!*z5 zn}QC0aCk^|fv<85S3d$d^ZmdJmU4?UR2MyN(v1DomHFz~gRa(l;vB$+J1r!ex){P3 zfp{eX=hHJIME--0{xUU;_*p13<$}W{^4z1HpyIhByJ*;sh**3bb1*_1r?>G)LzCk4 zj>i+p`V`ro8ryJ}=>;VH8(R}Q&47rto+P&*CZ3tu8gF!-=1XcG-U|2`VAT;CIY3RW)CvxNL3`9)eCC}~C)+*@M; zXwHfY;(<>XnK6gt8>FeT$5ay7a*H}?XP}i0G4VKf2QL^#(Lecrphu zj&PYJbW=!Nb13$6Mr#z}X2+@IS$-t?>w@)}V^n4(`=j-(a|_H}`uHz47SqF=jeoY6 zWHw9dDLrvet1+JM9NuGjUPqCV=8g-ZbSI-;G@Xa+H>n`?)vN!;+F8sfkclM#!MOUm zaU4vOt-C!$ux=K$%*IwveAdd9O%r6C8Un@L(CcdE1J27A0=sRlD5Wm})rmUThgyK4 zL68q=bG0!B&_P%O4XnoU_>N8tfBGG7E#CYx28ez(ckLI z3~!psXY9AO+{{~-B*NnK?`jg1%>A59$Qbbfr$;1qRR;v`OojBLLGop^Q=nSJ*1ABE z6-CPObM|j{QOq3o%Bl`q|KU@_E5Va!5)OQogtt?il}HR^WZ?nz zk~B;|^EYJs;Yy$Au*9gK3ox~JKJDbO3Y~)LhHe&ShNpA+O6+8|XsxKE?A*H8d~n^B zzH&Kyd9$#s`P?-a7d`y~LHbgJ7}Z8Fj-FH(ARhnYtEhydvcC9=ImoqYa6^sFW5Z6Y zxzJgQMAx0r&flvumLpD!#nC|OsvJ6Y&|P)qKBF;7hj$^x*gGUcN1Hv`YP|pT`y) zSVgvQXB4YnB+xaWs{wMeGZkw~baG6{TTkM~gwP&2VLNP6cOK7n4l)hcOV6Gdmeoj} zvn2TS6Z#_|lUuIhVM!;1+Q>e7vQ@u&0BQ9uq*BR$*ib`+uA=ki24 zI`_|1LSMWB0<*<_U~%@cb(Pa)yzGQ`Zg^(v^<{ANie~V&?AN&H7>8`D)BHwbdNkWjNVX-%17^ zGfI^7+i=Smf)op*eAMsMZ*@ZdP%ej~$wPjg+k^-j`}+~MPDuqSb;3iHVGZi16P~px zbJHeAa^_wW_Pshya2Ae_ty7_yZ4KU)&%EcSjiVD|{o@pD16i2sQ!Mk0{&o?lyigP~ z*4k$DFHiz+oT)iZoKORgw(q}Ke5Tg;5^SZp*|(vFKj-|iQtdI|Uhf*i#?GCL^Vw?* zbWxY2$v7V0Q~p}Aub=+Wtqt6ymOit$&-M$i9bM7if&vMqJi)O*d#NxxZ*(T6DFj z;{`L4UQUIK=jpt(Pxe!O#5U&WI6qPVi|>K=B~8qz&~^*b!zkI=O~cMb-cMN%yWRif zje8yQU%3c?lwQWS9w>?)cruWuRzv-7sW60}KY{mbkf^G&)WG}|H-+6=(KIo-9WwW> z(lR>FB#eA3;)q>QZH*yAVEG%YtB3e4DUvk`vEd7jUvyrRkBvvH-FB%nDlf9a1p%== z0{Wc8HxjMD4!=jvxvtZgzY|X7HMs8J?WlD0oQx~f`CF~25KuYzuCLjjmm36O>nxo4 z4UH1p>yS}{M}rLPrmbVoWG7i4%UGJWk0n$elenxK9H&a#KOsM2yEhA`HV2YpywLnx zSNmiQ8fTFWrl=U3aq_70iEprA6YbNiJ-0E7JdZ|sI?`VB zmz`WAvoJVr{QfQEQ+Yo&=_SSCwkv(df%a9gUHX_>S4qH?=9Ww?Hl5`7m4k8M2YJ@K z-z$hSUlMNfQ1V&SV8cqe?eH{+1*=N?@3Dl@iTRhAJF|(TSD00PuO%j86XeJGaXj~` zPSadK)eu-JLg^xmJ_#!7xroRb5hPcx$`9}51e+h>Qkx(0gtWFP{x+;A7I3F;Z;mE< zNRBoRW?A<)9q~rK#9X{Y*;c}S-!1q55%rZ}QFdM1GJ+1EG}0Z?AV^3K4FUqv-67o} z3JfXIB1j0*(jna?A=2F~-Q9e9^nTv&IQR({JJveaxmVI6i#`po#-BMMPHnTzSg@O! zvFYCK3_3IDsAR;gV0Nw+#ZAk_ety@QJWIL_>?=F3%(`#;h5VG&(RB{X_iLP{bHQ}h_|KGJY zQ#JPv>7P@h&!{Cp7SH`n94s+Wk%J@W@>2kAj&iA^W>G3tDf&Do;fJu5O={ao5vw$!1IMdw5pRU7D^ zehpb|D~9xRSf{X(?4KoTG?!UR^HC}3{98=xSKovBiaN%Ay+#Z605(QtK6eM3copR% zYd^3S1tpCHNt^|8wD3rw$0rZTt^2)rirP0dzQOF&&7;S+QvZXB0AhHlit~OaHT(Dx zEGavh&{JEmdL{Q0RFxM<%_e=(;Y-@)Eaho^Pm)hpWJbkuSIYzi9;d?R&b>4Lhc^Kn z@I4Dpj?*=p>DEkS0z0x4-=BP_0a&|-sdkAHVoL6c@#=jNmK8PUQa{Ug4UsZ?6M|eR z!p4Sjj4t_dCJ``!g@L@vcD5KpQMO~(y~{vg$2Yv4T|gAzuzs0pn;{`K;HKcA5$46< z@_ye89ISP7548=^nJ`Z{1@4^E$lJei2alpHc_+@6a`7DZ1-XdvEZg;X52Yo0K-Bi8 z_9#P0fcB`~qS|CvHJ{7*9*y~%5#;H81&Sv44wnx)JJhQ1?}@-mfP7mdp+V& z#W*%Z=w)h5&{<9JhLZK=h@VC)$?-Gk<`a$ocswDb$>}Av~X9) zfUoAFQi*|Rf&@P&Y2x5^rcsX1F(XWZ)@iJ`W9h0W*uKim3?sNvg9IUOl&a8`#1Q|3 zrZvOtNpyIwCt%+`@}%kbL0C+Bq?|h-;E_=D$UlhCJp&jw*sMQBRd}yf9VxScx8Nh< zBKm)t7=Q*Z&i-R~ZU3V3MPZ`E((Vy%6jpu%mROdLcrFR zuBOMbd#_QMr6a{sEcz*CIAE#&eQ4x0WVm-5_=uOWH|c8}exW)7kcvAs<5Sh4rXqd& z;^!3M8+CpwubtTsb^{ZYHl3NJ6Nfh*6HT>LW9r<}fANm;Ig~~7Pup*L4EO=`nK^qv zQr|0ymns*8fl_-x=FA{N^Y^dd#$KH^|f zPq8%{UJw+A_x~`_8Z_tq!~7~l`gf2As@MGgdI4~@SQ#h=X@s8y1ZnsLt6H0oa7%XM z-*I~}&^GTK8MOr$Q`;_njk3xh{zr_J>QJ}k-4|;>TDtRDO4h48E`}!$UBZT5N>$Y9 zw56Zopc)PKrcHNc7BvhU6n$U#7$UCk_9RZksSF#$MUAWQAQJfw;AV#J4eSxMndj&b`Qs+5E#=2e!k7(DlCqSO9=&+n8)Wdt z-K=l2Ve~l;X%WwNt<5U`94<7gP!;MUP4#33jgDisjpGr9q90^vYt%>=f~Od6xR~I zzfx{Mtd=X*hr9?e)u6u-%_Jo#52-mCf_ec8))`V6pOIr2`3<+Eb2sZU@LydS1a=#N6%r zSfwlM1;jU|2LxMT|3RoK4m@k?-TC^JO;XbD^Cyo51aTdAXI^&eO@4r#0J52*KgU!^ z43$WvQ?8piBQ@5}jDL@NSYq9cY(sjc%>B85`X2{ujcraJ6&8HaVH3A{1Rh+0b!;*m zUQ`TFD&23vN|sd;VhqRWQzZoO_1crc#fYbik>OQqJoRVPmHp>xNrgL-FCYH<5YtOa zq3Oskon};>%T@UV$)2h#Q@mf2tR-tUZxcZVs8ZRr+FN{_54I!7x5Mnu4xseM9ebUL z1iR8x)H`WPfnBi`e~*U@Zky{_y;taxnb?w=o@+DWhGWzo%L#Qcx_5krW~A1v;KXifa!!o=<SLzEeQwxYy3w*bQzm|m?5BS>uY-QKo2-k!t-nS7!y$0=m! zlPDmX*M^Ay^q;K{-tQ#$VgZSpV#V$w=`U#pjz5PVA!2~pf{ zf@0N!IRC|2!P~q>it^pL2@kMRQxDE|3B$`C5W`dX{_K0)Yt4)m{qDZjmmGR~bF~bD z-0dsBVnAM@gW%ChikD(+wkh1}QV|{ZWRp+!5Z^9m^b@kbqh|ZjI8SrlA*@ ztMYwm+P?WWPP4O)HUmU~G z0NJUY{=wojOBk;J6BoBFA>8^^q_m6DrP+2qH$OXs4i^MiOYCO(SqvIUHS|5RCL7;H zQj08opxIwdvIogKNE;g)n$-?eQW2yf3?}=}Tz97MK?t^4n&J)F^`|;*3t37|Hu6ID zMxBDK!Bjz{|03RSV^o`olwb>y-#@dH9!&HAz6hw}=Z ze)z(2X(ss$f-bVu+B|4XTufnJG780L5YZDC`-zg42^-JqtKB-sjjS(AARDjq>kDTN z%K^&mxdsxQDm%&7uV3G?+=olVn;{dHZy$I>{g$Do!)8E%L%A{r#;VCB9m=@5LQ4)} zc!7Jv{{QpG0Cc1x!{caFCGVAnN2aD1ixF;%nc~Kd%UTB?)Eg- zAbl<2phioF4W*5$azdpRCV6~~+x(T+gMs=jMZLxS%4YPFf7Nc31<*`0Nu!x<^NCWg z{>>1_T|v^(_gquFbIY?>>Kr2zoVe@5-xU+~5OGe+D><-qhn%XnpPMs$oYQ5zGN&RL%1?@UV|1Ghew5BnE4vTQXm@>q+RI}O)^F@ zQI?qg!XYVZCHhXS(upCX_@O0;_uG%}G~VZ=(fMCed%4{ynV!{j>9~{}LI)}wKnhLl z?Q}ssG1GTYJ%onH2D|?}9a|Uq{B9n!OF6#E^|QNppGII}aUPx0&?#>`BtW^7V;p0G*SQ*0tB!Li`+h8)c_&=S7 zX1iYoa+GPxEe9cwiJzBnJ$RbpkuKEloUdNk3R1kS0l+#I}%3CaxZZt@Kk=?S=O47Ia4cOS05d~P+uKw{slh3vKj@&qG5 zP5)>ufg@hvw;I7skd>fHrlv|n^)glT4!(Y|)cL`BezbC+&XYB6)<=lv-HGgnO3mK? z0F!p%*|T4Y?>@u(eYz_lRDC~5XT0siYEZ-s=Rl!MMKnw)1UaJH2Q+tan+h(H)}8$@ z3{%OLFCCsGbw#vounhz8e~*+h43I)yL33GAdS7*IfeLFeLiBrGiqx`7zd!TTaU6h*+ z83C{6#ifgNhu;}8G8^}9B9*aQYc&w`#xL|9uMCM06)s8TD$zKlAxx2%jTGnm;~~HN zB-JEhN-R~7Cp6%V&j^vkvE(qMKH=kz~ zE85&6R7L&_hiun_xF}vzYS?7FAGdU~LP#!KT?)Hf{4aw`y+oBy*=aeS|8Ze4yZx0k z#QIgJctIVh9HE-es9HKy6&;WQtCj*WD*^Zl`Blb|ct-RJ zT|+s!Tur_3Db{cfYOSCk@!;bmh!kmo*A}Bq^7F>AD!Iph>j)eWjxM&8o9Fi@UwHpa ze2;~A!B#5Jw54rWWk5e8Vnzwk8$k}>J0;Jw4II&_aX!IkbPIg$%rb9Jq^nJoul~M~ za!@lZE)EVSSho9)r?X!5laU7VD9~d0)efuH5F?0J#<`k@EuB)8hQZ$_=xB!JidnYF zg$XZ>$>`zK6zS++PmEY1EP|l==ES97_N#SOZOwi)A8Dn5i%gYPI4=Anf>APv&Qeq& z|7M7ZWglYj`-R!xPAg$*ucWBx9Sc&CAM1X1$g=ADo4^PzlYnXsdx* z7r!8|p8=OZrq|QZ>Yl@#@Cacfg#eYd4Ez)k_{wl9l zd`*`;UHnMZbV>tO`idFer<+14H8ybL-G6dtTBBgYF+pjAjBAr|7KPe=S&_35f_o{w zo~>BHiGLH=M8yre4al#p^yPIokExf^dWtOIL5k3oIE9r2dPv&3w-uNRM8XoQ>53eglx z;Sd#bTNHwd9_kf5P$xn9f=*`t9{E}C^aOQPxDTezQy)Fs_I%2J?k&Ui4&9lX4m*$c z-n1Px4oQ>fqDfgBzp&sxaDi)CfNPct(vIUWZXeKxSr2uOd9t4*gf|xEOTq(<}!@L=pDF~6)igff430G-XSUNsa}s-wf#S86TY zm+zO6Gal4*#JPtvk`t4LKhfnJdRv-9KYt7l{DHEO2_s&~Op6%BH8Bs~K_nA}b=2^mPEedy; zSTz$6Ia)A6?ENeF*;6x<(@#P^23PLR%LVXKb(R3^2i$7NOvym_vS`$TsQ207^qna| zz{OQAMrF~<$~L)sxyu*yB`1CYa0^Zt}Ni zAaDZGFsvMyH2~;g(Kl=PGdc@;oF%cGJ0Vp8*Jz%~a9cSj}vo=@Lj7pwT) z?`C%33JFQd(tG=wbShpG(Q6Ri4U|IZLOvG2pbTZBf>qg~aA~S?S3Bb|X~Y)Kr=L-hE7m<%lDk zfgjQG@ebZRBd2*!Qw=l`E?4RjUZgTotA@Fof$N$4?HBV!JDN`8_!Eb-njk{uWl&7> zcbn$OJUcyY%M|7dPnY#STbtv4>gkpwNThq!FF=p+jQ)uz5V}+=R0%cPQy>l!skD&Z zR~ML04Z=1T*S3hb(~h+wrOBVy<&6tg5*;0}F8O_k*c$LHFMP^I*SOk0?j+*yAs2ly zP@+$3C&*0ROJZrmMJu2ej`kAVtLP+ao+=-Cd%_Ute&j%Y6;HC`((_5J8eX+_Wgd zkp#UCH6)Nf)t!p1J=7O_UF;3~cRw^(+vlqcA_`ULTkYty7yflugb&b^OjZ*J-Do{% zOisBUIUaegNzg}v6C4^58Uuqi?5c-aApw!m9+1s5_Q3k^W`dB+-iuVBzrKrS;#*80 zH@k>m6Gb(uXUf-o-&A$r&kLXfeh(+dyXlSiDjP+4^^96DJ=81ICby4R8ZBstp7i2AMF1<55kj7UDtyC8 z6504I!lmr3asVofueF+B)+~;<@XPv-11Qy~A-5E=6*gj>BU*V@=U6#jFdvuqORPWL zH*86gYiy|EgdE5!@Ay$Y&8y7+rMvGSzvj9odo{;3#aTK(s&+$8 zN zWpS4ddpaAMHtNret%fLZs*)NoA0!B2A)d;9{=VwGXc68@tP-g*5ZwnA8>r&g!zHL= zH+v`^JjK$RBNQC&8`R}>yX3U4yZS5d`56#S;}#jp>E+HdpSSye*v(Iy*Mbj9egdOy zzUQx5U2!JL;g+z8*Ma;z2CE~vID4ENa&k!)R1{esG*OvsoAq9Y*Kngj!(^D(U?fE< zCmVdvNP+San<#oX)>vc^F-0aob-!{GRaR^Uc}gZm zBE4SDT!o;BlJD2T6g>4uEb8$$_=0tAc#XRAfBB6iv`1^VZPq^1r2SH;=o6Wa)M61j zuA||aW&GoHL{&#Uq&xcPVGtJ6VXHWuz$ED{pSF7&{Dswg!l zgNX|0Vk5G;T~9l+Nm*3v@ZBtODx zfDnErNw&j-FHk8J@1?CDuGD?l8);K#;(bpLH}f8UD30Y(%FDkm93xTRItGy&*#|$I z4{f~qiSR-rSZT3Z(~m~Gq7cO4^=xhj@NNc-?F?MZttp=kP&r6SwNzOP#Lc=t<{U3V z1O83mQ-g8iEO360tp64fOX4J2m2QIZu_dbU*{J4ye2|T&q=jG)p_zx+a7s!nY_f1S zjrky{?w}~I2q>N^wx$^~gj*rCmyS1{EU|9F%9Jf7G)r+#Z&>a11y}~v z%Y}YAefjuWi`?M}!m^QEZ0PPfmTf^I9mEI`>X7Hx?tLTzDe97OX}I&Gb=ff^`S#46 zf2{t<_#Cbj3WPlGb91?*ei->Tvoj}~F-i!g-lq?l9<~{}Sp%^(P}TcDc&vPZ$2K(^ z3PROsv*DP{v*lTZtz*T)t=S>pl9Cg{?S21(glbfC$ByUue3jPz#q|d+E+_#hNpO&> zskSOsz8wKPl`_@9Ny)_@$e5aRvtE@j2qzIz4EBCd}VB@~VGh%M@e)BgSW!T~mJJOPn#faW)I$=ZY5NJIsl9vos zC5ErqH4I!Xk+ke69{u1d1>xp}$SY#h*c~i1G^VJ$f7BK)aYfZ_Z>}?S9_u6)n!bV!gOG}yFqw_ zyHNN9eQ(?64~ZahrbG(Y+wi__9Pn?7yDYI%Vf*0}rhWdjW|r-Ei#WVdFMY70!cE1k zdR-4$ECfIe!%6lOHROug&>XZJ^A!BSThJcoxYt(ZDWu+X$u~9YK#@*VX4r>Ff(c+p z+I25+X}}aoxWTwRpMF@3D&zJ^U<##3e`NfFzv#ni+teS&kv;JUC%z~f5OMg_Vi+efB)Jh+~x~V8HnIMZW-#X4JLN{zdoiUo7NQ1@M zn#Z7(0F~<{-cR>U_~lIm3?x{8u%3HahQ7ED%LzA9G@QKDz22*1DVVCZ@LUPtBX}FH z7jp(us&51&sK%-Z7!eyK05ZD2XW=nCLDymU0A+LPil=!+`kHWWxffZ98e=cDx!@!EgL-8P<+U*WUtg|e|9SC<+r9R3Bj!Mk>YWlYbj_6N7*L7$b2ZX zwXA7fMec=1yud$@;$Q&l8}WvaQ*3Xy6#IC>N}IFf6^RYinfSX{;YwF^8mw4wH1LSZ z;Se(e+6-&%A^yp2|Hk0)L-OF`Sh-U0B#dtKC_=TH^Zn||ykmW}f@HOCE?;qusWv4f2VgG7RlSc4hZArr^YWBHXer6xp;ni>& zhocgS5yjUtqJ^6>YU;>&@xDAN`Nquvs;s!+GPqSy6S*X{GcVARcfF8GZ3XUo*6=iY zw)n8O?n*jaJ0GdKV?&uC3FhDY73nROUAg?5g)pgU^!SWmkH|@*Y2jGvihjh*kSs)o zEdQ(8m&mOtu-D`I>sH1DpRP#JBqXRUdYxEm>}e}+kIAm#TXaD}j7V7U1?q)oUStLH zalubT`hEt?JNL{zyr;ol>!>N%M@(pKg6)^zkrxgyg##<9PMPD_Z<7*I5daGictB?N z7Hc|N{&^iwB{wGCQ$#muC>lgy_=saek^NSCwY|f(WdD0aD{hN_GN1@R6=r=n3&-e(VFXTEo#JOlq%Q&SmfC zM4`NelHF=mG%Y{5EZ0_z4>o|qjS)VZ;9W5FIrQ6_Q>Q8W;o&VSoA6lRtx&pSpR%L5 zc)3sHRt(~ufhP&`q5qhkj_BcXMH>}0grpaCHP}6VvWl?BWCd^%Q@mFl89(Z^$0O^DN znNynv76(SGX=GM-IRq;j65e*VIdaA~?ce~FmE3ma+xsVnP| zvfSD<>^^+NqATl8T{hVw!pKMk0byG(tZCad?>XhZz&p~0rxcKl)N`*znU%)-7VG)GpIX0nt&L8^G7WVrAlTtsaH*KA~I zlbOk;ZgEX%v}b~hY0Qi=8|_o6g>(N|z8_U2*omAax`;-lunUA&Br<6i+#Q1irEr!| zt2U+cjTbO0PC2f!uCAWH3CDDoyX{bHP*jk&VH+fn)$p)FteY>ES~T$Dqz0 z!t)YGFE*$9#*Y~uftnfOEB7#|(4goKL#K3Ct+_|w60#_pF1SeP_QXypWMM#N#W%6_arM z5=U&EER%uS?>UMgzLYZ2`Y1RMWJu=O zn|V)r9uvQWD9)IknJZEs8ZPI(8Lz%O({3+CFG5C>qSoj@gLK0qZSE3J%x!7TXyz(4 zIjpHl+)w=^{tVn%c>-*uo?Ja;DwMt`5vw&NPlrS3NV-cs-6gF>&D=~fjUaE_pUKeff6Stl=GsntPlT7RY(U%(S@ zytGKjiMBr9+)$mQGByH?Ev7%P^?!#$&sb3sD1;#+wTv(6iDu=*8uG8%<%%(whfV zrz0&Ngv{=(xXR;kq{E8Puu{IN`wE`en_0zLc5DdQSiSeha(QEW5XQYg926^d{RM)z z9}I&%K(!t#XfcTt3Ie^rmq2ry)MlqgG(TR*8_WoJt%s8=1%*lgde+u~hca*?9cIOG^EPNct>>P^=s_#|x*{*#Kv zC|tq!QJ8tUuQGC@j!HHPl%9LN+Iku*jEwwa%Fa1C$IF|tuXs>exbO7}qLa-&Xjzd4 znt_ZKRNBqv(Mvx=6w#%u6{<6TiixRUleHVAXP!=AfWe!el(f^O#w7A{vma4<>(o3d z-I$K9GRDp{2;>rZn8202w7;5oFouc_X4X(LE*_2{WX8{Ivc;#-hz3z zL9lh$Ds|1N%myzNW{8HY8b!H`MX#9hsBAJ!V6PsR6INr@-P0psV^dJsaOgS7%fj-g zGlI-trJ~_2Ni$1lUvF>Wy9CAC*rV9EIBEfb>8G8`qSrK0F)=DT_8lS?i^?R_sbfA6 zP@C17utt43gv$40o#b;~OCMi3W>Go>Z)c`pmQ%#*;eq(5Fbz9g7KKn=;psq@9D#(i z^pSI4{|en7b9ntaU|RfyG`^i$Gd_6xI^ zsB^au@3ifS;k2U`5ovlBykLF95|xs&yT-M@ACA9Y&#OoTQJLqu_x*lf7P_HUgQh?^ z@}tfteI3zZF7fOM0~5msCns2ly4cF9?-82^+Pm8Ba5Dj&1({d%r+!94WeZGw8uMDN zUP4>hcCUJC2)1%woPbVXVE6&_4Rc!AqE$I$LJA6ltx;p5m6`cYt{j=Or!}o(pvV;$KYTh#7Iqu= z>ct;CO_r~G^$B^YW5eBE?%>9_n|L24BUv1@bxoH)>5S`IyY zAJfwFL%;PSKC{DWf`!#p*X<#F{XqxG3vyB1#XE*n3Zzi&tQSTL4i#8B?l$u^l7*`U z4+NX8aMJ6})XV5Ah7?7$aN5xLzBX^uWlcT*u6`0O=xrKr#H#7=S2cU!pUFa2dFmx?b$4F$p^ zlapAQxcfUkU|iqpaax+y%jy)Nid|{MI`w0WLKec2{B)8+Y*tNku$O&gNM7CGfgqyvTVG$l;&x@o zUjB(vOk?~fT8AhSToi^>A2`qW%tumuN>~Lo`L$}V$8$plmPPDKc0Rcr0i*;Ovr9jm z!lYtz_% z^2~6xlA*&Gt+N8sdvp$&0r#69E_egUzOo7y)3Z*U-zW!(@wRT*UCi$o-=zc?Aya*bhn(wz)QiYyzNf1%C zD2EPcK;=$BEcM>nUAkdsHYGcDS+P4j!_$sMU#AG4>qSeLV_OZq-`v*MKN&C%uzKj~ zl*t^Up&?RC;%LD%E_+L2e0nX(i>TXT#u9=RBFz9vOjEcY4sXg;i4y&)SbSppdgUPY z)D#r{)nYXq99iW(e`J$!5jjQB!oQ@WgT`#TLYpXaTDjwxH?^0MXe7&c^@M&w72fjo zj~QQ_#$d0dhX@IOt}Avt0}KhYpuguI04pLxe2Om(HuNDBYx2yejI2;IEd5yM>AT17 zsW27PuD8%{cbXq1^M9ACobHhN9axkmDXUx@2?G3YLm{MnJ`+)#c@G5wYt9^#+i>a3 zfQBdrOzQ{f_0L5feljA(X7vww_dQy?Vyt96CtItky!Zl0EGby8S)6n zFZENrB$`sH8UmOw`2EriL?o$W1GdL5YKXn`j%>|FX zSknIAKK$>A6#Avb#Y>|pEd)Rc*@%PVhxw2Wab8nAKw`i$MeL=ZHC_)8tU8!SKF|8W zj82I}XYK*&9~rOrI8M9;2e%NrG~Y;+m)$cSX6D!{2D0ONSV41SV(;EPA&-CnV7UW_ zdOja-k$Mj$QHk<-Dc#iktp;FK)!i?n5BAoSKh-p$Iyb8jsipSpcKqfFb>yov&Kn+V z6MI%38NNkDc!KHoCmR-pkgr2D(^H^kB4|Iq@!*k>dpw|*gLao;7*F3HPZPXar*L24 z!rPDbEePE2&}g41?MI5?H@{1vA`}QyNxuB>B{&T%LDW}Nj4So%^WT*%T|JiQ{$@Y+ zidH3K*YiTQT9ts79_?Iw;fUg8snM6j4w)=ry@Ki;#pqkU$C0u~;7)%b0~sOWRzz^e zEZ;|_ugIM7aisN-HujXu-`{srxM2R8h0oRZ+)Uqt?FLN-y*0h!k7h}pacOx(?h{*j z8s=cC^{OE)PBcOZUr|3%ro{G4UuBdjeP?Fsx5vXD4XWR84}DkUWO@TrLjW_IAcp~q%z{>ndM(T)Oi{{wzC0WC7m4> zLN^qWP*W7#A)SZ19tUOAJy$i4*hUivsK>&ji$f$r3B3z9i+*TPq7kb4elvei= z_+ch^GJtbQK&f<^r3-7XZPT7>$zBX`wK?pWVSk8R)p*0|G6QZ2 zIrpB(q+j_`ye(Lz5v25Ka@c8=V7k7IUl}E4-BXdUu*i9%@r)ox;w81Ywg`U*h1_rMzj8JfKRM^|5>q7K3|r zsZ;)aiv0G$m))kvNCQ=JP`d_NNVHKH#-ju2X0g}45OSnSS`P+~U;y|(U`zYXgKH;> z*i%KJ9ZKaJ1lt}<>MQosrlwEYKI?T*)StMbX7{p=L6FNX-Re)T9cBdBJ|X!T-J)qK zf$Pizh@VGXuLyRl-HbyNxU1L@L_2yT^Tg2k3BKXDio4*oC-`Q}hmPLAS+a8dyb;9zxTebk# z?I8c5n;ZrT=~eF_Kh~|mKZ|Qvi6cWE4E(I6zfo8wH6u2TV8`&HhJok)yC}DI_@5V0Hc{Uz%UnxH=T*!JOP;zsi<7m^4+2J?v z=8f5D`j;k#j$m}T@_Q|&oQof$jXbUsWgbBJ3!R^Yf(3p9d%pB^vL9loD)6Yai zV4==Z;+M&MP7#zhC*K_xen&2_F!DqNczm!@hVnZOva@a zT8G84f$!@w*Cs9iV900iS*e4Ac7Tg>DDrujFN2dPBsw3a1pcy82`!FY9_YMljj$m1 z&99f=YaiFU@Sf9-V5GpOVpWlR$do+P`_bYS2UK=XT&cUq_CkOlhT|AU4KN5d#)Eqf z@ZJ3QiNlbg_Vzs!b?xccN8KOJxRrtqWQRP(T*z>8w2Ssnuj_@rBfD{rzVXOG)%Zpd zF)bZzjh?lU_KxK~3NY^mXdE*8S@e;T03x(0YqwljjV^x-rhbCn|2w|#zynj3n7cy z6<>_#9q-G;%ftc?#ZY~$6x4mAjrY;9aAIdO<%wxeHini5oVxVm9~lvDGei7jekcmx=_ zAJ#GcZwT-tC{Q<3rpd;8;sCTMhHzdfuX%lw*a~rPBPGvOH)m^!cxorPSXB6y)k*2| zmS!V5ySH}OHFXsa$3=%o;PT3=vD7XBE`;Ty!~&KNS&Wqf6Ck`2dc1gBl+z%-Ov9!+ zcj_eCcjBayvkKXgwLi$NHGbSjoL+-o&T*H5Z;b7@1{K>Ke86ykBbQ#{Fv~^c#VRwRXFW3IWEW z_I0W>c)Z$b2;w6!1SeSEG+G7TfK(i%FNHDvHOR6Ax)0)e7^a&D-q?%uk0vDL;Fxj$ zVULS5u0z9Gx&5Ul#_^UxL)nhnJ0%GG^8~<@lGPYO#eq35_AU4ZiZJVgN`u>_lQ>*lem;g8I;+B~j+kBZH>Mgm}0+4bR$mW3U=nq4%C}*_t zWDkP$SLB&yRQA1!6TI*9(D1(YCeZ19^9t8SH(Km=DhzuTZ9emiUTpl2*&>3up6dXc z*A(da4(N0ggi3jE{jhl+dn}D4)z3fUNVqf2UYIL0unE0E(v>}tzYf{Vjdv-%H{L}Y z?={Z($*R3O`8TkYs(T`mCwfgRWGD!N`GlL!YFzr8w2at~xX_;dImt0h^u%YSY884qNLEKc&k;qi?Me>Cyo|(gyrUCG_YWz=1Sm8`O~X!!A5cYLkV^ zFU!s4U!tnsB&R*PR@T&_+*xkV@{mJ;O48X)i1|);1Y}qUxCSFG<{k<$D-gQD0Kt1e zbKz4c)B>*JRTgu8zpVW8%aQB8)qTt79Dk+0H3q@QlfP8Nep1BbFV|~~hi%@PF)n3f zsK^$l&P>V+Yz1CUFuofB@RD*DwU?fEO$Sm;|usJuoGUS2dHzV=Yj|zp+Kj_5FyCL-5XUmWBc9H zyy{gXbsKjgEF3C~siC{t(L#0|BK~Li<;$pK3~->HI#+j6uq&CD4N+0ooXjuhE08GBYSqpjyNZAN9v;T4S!m9FX{yZVq?$^Zt7ySoBp~ug*0H`LYy7Q64Z<^qwea1> zaOA$TD6ot2loww`()yFQjI2jr6=p1m?S7P;IPj^;c+QMF^~cHIlhi<0EA5N9?IG;P0uF;WJ>fL}qVq6gQXy_5u=*)jMgoCfW{;GYpK+b(?Sih-KN+~S&lp$^= zN4c5!i?18*F2dD`(SLa_;+mB7a)lLdXUY$QSLmLWMT?AVEn%;YT7*Gmob_WSG8NWp zYa3+kpZke%(YF|y>R>$=DCfD;*RIW^;EN9kq8!l}Gdj|?REpbhrp1OCG>>pAg$@Z; z?B`7xl53cM?~nc%`gl)N!TNH2H2M+NqZ5MWRv>6_0FzWoR(=;idLOZ44OZU2l)u44 zrD0QlV#ntc`DTle^zst7dfRRGs$lj_=M26T{5dx{70*;ylP+nIf@|KDUzsmL^a19I zirrI3D+Ajl)M&r-EF}v>Nr! z-IjKW-Vy#>PlF#_8nD)KAospSG5bgQ0zkUv*8 zwf#|XEj3@7wvX2eL3VuQ&>^Z-8R2^GLtkTNg2dieL4t-FuB^2Vr zGgna}2u%Pw#*j(^l(#nb;h0vTMX_evEpZQv{TnFx?xUb$3G1qU zbhYB42|<190&S^S3*4%?)Z{^koaj?v7%f7})P{>y%2A`#We9_==S%VXZLv`t!d!*>GXDIC!8_w$ z-LhtB#-!PV6RACbyRs+0&DN(k=7 zFD)-AFzX(0)eIa|N@IPR(Pne|96q*60cO~KG*Z36mJ@)->J zvD<_mn3*;}F*tN%%bJ0iDM9a|wr)uA!CXv!n-iEY%Xj^&$*`* zR{082K_^$**E;*m45`fF34ncqXG_$#0qIEc_7own68AlQQUN#l_Yd=UHkoouO0eF% zc{3#ps&h8;RQFGQ6}~2&11n`vvWG`?F}T#*u9E&U57Y)a4cP0eUwi;BI@q*rh71N-FwBJ@y z9R`>a;J}|;eRq2`m?r*kd!qC{C^>2aP4*)}RS@RY=}x{rr!deT2E%)BhKdR8q$Kh| zbu+ivIKz3byMT;p2E&*7uU9)hE21fDE=k4r1_-c+ufYEJ=8~BM;Y{^)NBs95tF%-S z`+DrN8Ry72lce4t(2tQEBUpS(TUA{>Xf$8L1auCDVqymWGHtDPTpnnA?*---_j)nb z1R}l>fr1kp2HRAtvZP~Cm=6rY%zhyM=8Ij)GE80a%K*|-twFXIsf})sju0Z?&S1c@ zgdm-~nFZ||>p`>c6753rr*ED?A3v4^F7Jx5`>2GVkLuPW8yqp97)%2=3ob=ZNk+WA zQhobSaHPY;BqC!fUauDEhPBR4CpG-Mimc_Z@N9SazN5lYz512I)$#jeS^FadSf|3s zkb$?CPq?YhZKoAfXoMm<$D*R5YJW+uuIAHZdD34rbV*r=c5!h*+Ptm22x3LemXNps z4wv_)Xks9U#>CjMyGG$(MpQ=@J2~sIm5+m$YbUj7LadQOmhH;RsohXgkHAnrHA=(p zUnc53x!4-djdh)Sx#Rg9(^tK&S)^mvhjwJ_D(LEFi(IwxY`sA}iu12u71+-bj^@AUW^KYqmh3(zf@UZAFm(xPtP)9Jl%o z;fY$;{3l<_ZKo?sgO|2$XeqaBz{utzZ31qQ5!jM)!M zZ%UKYsxqiQ->sRse7k(Xf{p!{X&KaJMt}YsP-?O@M}6>1QF1d_xjhl8@s%uhlebcI zZ8g-2Ac}bejHGyXH_|s!paX_&l3Wy3XFD)-y&-Q;HX)t(e{~#QU zy|xB#-ggI*wyXimKF7gMZnvaJY-@jRj`t|tz$4OuwTW*#wcKO3w zwc>-t=e@^FS>}sa+9JX}-ZKI&7QQU!X1!1=PV|%y2rb}12T?)}`#C|GrV=n@0UylB zd|feJHSe9T_ZV~q{X(U&4-^D#WACJkCerfi$jY1Jj)-#)Xq+O7Jbh+FBbxI22bRnn z6p1HDA5XWBHyEw>CsO2}n%`Eb>NO*2=SdE;5wA?MrEn+5^#6r5Ums-1CQjs|sF=|> z>zAA;d6}SZsiifY3~tKtk@a9Y3JC1$4aju;9ZnR*}c99G#Ldp-ei{r%ZZ&Yf!WQ4fx{&^n?E z(u9Y=?anap6L`cD3BP{-`^TRl^xS1S(_4Fiztj)fZWcQ2ua1vv0`25)ei#ov&$klv zuo1&cr2?rUE1TG!V|rFsjEjJ^$}P_w3UPi&sOLMWEhSF1L}WCP8_SPbDnPWB ziub{09#uAjZTysZt`ytLz>C<ETsZ9KtdwzNJx>Y{m)Qql}DASnWZ^q`a=Dc}%8=bSaT_jC3+?_cm<*Tauo-!-f5`(Af{)|{dj zkeeIjs-ss3>KsLdekQ55$6e_tk2M~ukHeiRHTfouR2XlJ5l%-O)WmaoB7u~+O_=mc zq#nH}ve{1#;=Hw=J8W$6A2vS?6FQvFKJ8U3c$Fke4jeB41D`X$>}5S?Aj*hMqA5Yw7@df=nl?}#ol zJ^6MfD|Twjt*%rrHQdF~lH3P6b$}sn9|)q#1aQ*{rGB*+-qL{HI3XQ=ZZX{eksmP+ z88;$&F=k+@-tc-Y=np$iLD;@#;AH}516VsmzSX)idwO?2+v1zYn~lj39k=GLfMM-7 zi22>T))7*y=|;cu$g@hLb4!3$M8jXU{V!G@oFUuV z4IzhX;U`YZXg6K8Gasd@&C~bq4XF{%v~miJ!wMOCfNxv@kx&>c@(JM%Dvhw#PNb4! zI>(uDqAPUp`xWah#3*y}d$dejKR>J`b^y1BZ2xr}laxIGp%t*_U3$$Z!jzV((d~BG z>qdyPz!9|3aWMs4@b6SzjSahmbJuu92slc1*Ah@=#^qOXKa}>Ul@hKcBZ~c zT!wdR(uBD;x8I`W_ieZ0CyO8m2zED^3mj&WV?4d*yvq!ce#-x9TI<&O><9Qg!{}pM zA(K-3XY;3;Ur8&L85J%O3w@JMZp0l6iAKPDGg)9Ybsy|}4R=jvlbt=#ouvtVBBkD) z{)_W5bl=u!gJ1k6N@an~=hlJj!kJewxEJOBXI1j-|6D{uN%42|$u{_q`ukO0sWM3QZj>k^fpslGvpxw*Sz-Va{p( zyI+fqlN=Yt`LmcU52#A)gD%~#B&<*o3EV7dt92p(6f$NU}?`zYb{DWu~5=T<%K z;GGp6loJn;#J82ssXu}-P!aI+pf2>z0~~q2fE;!T@R>cjgH&5g^?pdED4*;$XEthh8!dTXvG8H;K)dnBS z&hG@tm@&GAcCc(xpftP)0(e%)eMRf? zvk9kYAK?P?^2PphuA^qey(77qDXO@srD(pVt+)PUOunZz#83YU(EdD=Cu!z!aN)tlJeC%6mOpPXHO^%`_r+=qM<1iyu6GAQY#7j_)v%+PeZp_@6TG5h&5a zU!R~C}`Fa$v`&f`-^WP2_*=Lvvw$FRAG@$H^SY|={L$nG(_UC)*+ z#k#Y6RzKUX!QsHzP)hm2TS2dvBgX;>#yy`8G2Z{-`6%jmOXZwXNdy25x9L6VypIpy0lXri|vi! zF-!?LDP9Am3@Ywa{rFpq_K6|s3!|G~?!e`bfcxb#Q_$cT z(;z0kBv!~m88z+o=Y0v3!O^Dfj#7_hvGz9FI0TW)gWfCS0h7 z@kYw?l85e9<|s8%=hX^1m2exlP`&Xhz4g~G&u#pQm=?Sc@q6>%F`eCywrQq#EL@>W zkFnwd9NP(Xm&QwlEr~N|_d6`-Jev(}YjXg1aWZ$vyZz_od9Hj$5_$f>b zA9qm-NXsa^twMj=U&JQT5OFPd>T5VHf~V?7e9h(iHRwVl$8^lSVsY*Mw8d_dI!K7g zSeu@%KjwoPW9qI{H2n0Ir3O=EJ)Zm}?=LcjR`WZGZ5SK_kl2Ge#50rZ17=q?F`m)4 zzI9itaYb?4GsDo~su_HI;D|w_3S}2S=R;pPjwG$-^ts;*I@x?EbaO&(saJ^*$|Sv2CB=qIFfaOK~WwPfF*+Id}t{Wz^h! z1W50BPi=^-#K}F@X%C2`4YRvLP;Bq-7Q3EA?6F987=gvqgjbh<=*h_Z`*_JZ>^eIsSt)EX| z_N@8FIidu$bSn0D_Ka(gu8eNquz^je-q;p+BH0zU$I?k5AB;wHFKm&%L$NM@KMO*% z_216Dj8n8hh!Ll-cT64dfakpnQl&4WlFA@mr0@xskQYBZQ~s%!-Evq@X^etohx1UP zq=j-hi!R*NBzyVZ0mhCrMkeA;|FDYy?7Qf*GXs%Yi}%+WV<`=cUSIollIsz~MNlc< zFaNfeyYK>s5&zidFC@(qw$b1~mw<2^Oh1wkCcJSWxi^@rSbXRKlrfLR<6Ysx{;Z7c znP`JkSZ%v2F|6JI?oa%(%7_<|CNogLjIq-X$}ovJyrV5}B`V(6JGOey&Kg@tByMMi zr$LNK>X$zY-?%jPp5xuieI?AH6?M|EF_nVd)j#YGbj+F8QC@+T?g}Cty{}P@%m+>H zX8d+w2t^GI)sv=jU4A*}fwaKe32y3s1D6+%O^lV^jH^nFR~E@+Bd$$ND!C7dHnyZ) z9-XZ|^R)2nD)ImDks4K+?BV1)?%u6$7U!1a8#5<4`LkOzJiaDl0v)y*O*?daH~165 zSsM8gHSZiTg)9hLkm&KV$$pH_{FXLP1mf7SK-VU4LlZpy5w1rccUaO}`P{c5ZGDJv z#pFlRO=ERfu70^CnMFXj)cs*c*RyQR(znq&Gj*7eoBnHjuGO}+#&IH^oW>MlrsgNOc) zI?k$LeR12UsSO4X@y#_2@!I#IHSop6kYuR^CKDrSONUI}!;lcdut@C)!6Ix5=-oGraiGS=i?Cy(=`Yr-mP-r?G7zkVR_42YSze9knTptXPSd(0O{4+baZr< zUC|tl)BYZa^{yyd`$&HR3at#JL*3|KG}oRLa(kDWL@h{HS+i9w`H&nBbvSC)UhBh7 z>Z6km54}k?{<^JQ-gH$n2=P{t z0CjK(^y=b9r;dh$WVghUoL6GQ?APL!?$~C1iZy5Xx7K&qQM9pY=Q2gp50CUoyb=KR zdA$5?K?a$&=MIhLOQGxv;n|=rh)Aiv+h09s=vAu&})hgh@$6SBdO+Z(Q|ryl~x;K#DK-iUd>+ zS3lE!ha7%Qku-Y9xwaNV?T(S*%y7qJ*R6x`TUe~=+!=e|nm!>UE9a`Kw7H;dOD^UY zOX;VLug)bDE1>vitFzL039I+UEeBm|u82@*fD9-|XpMtlhiS@kO9H0*h+1>rmCVm9 zpnX@m1SUVa?~!b7@4M?9wmu=xvS@Ylgxw;Al}%AS_h8BQmXyL2(w+BRZthU6vq=_n zGBM6^gnMA2n)D9V6W@Z{I4T6$nSqaWAIU3xw_HY1kqEQBJp@{U+4d^`EOT~rcSD4B z52$J;Rhlr&XxG^Adq}})N%TP6Q<{UaLsE-l?)U&qIv zgV>U2HcY!D<&10Y?<-}N&i@E#WB|rODZ^4EP^LSi@5l}&6;J%MM~T#mQfi_(%Y^5C|UqP2PINjaT?$b3ZC$-PyEl1{CdE8w*k74 zlugIXr#T&0<(CpM+;>_W+58sowqYw785uP*b2K$f#lKuz*(bjIBI-leR)#VSoz>U} zql3wrv(Tm{iAW}Kx&g_3#&>Cdy{zQs6MG19n$9)q3xq!0)TK!F9`&&$Axq;r4}NWN zcR5p^rb`izoKk~$lwC^*tuo<94kv4muxN_`d+D_)#pfA0KX|@*UuQq zAY7L8&h5`r_4k5qXoA9MGKT8v>bT6zXx{RsVV!#`O5C#Tz~8;*75(DxiT<2%KqxtZ zUTlR;FWsllpVX;YRp?E?ly5E-nJ>?Cix* z6a6!UaVcOuip#}4n}0kNf3M;iH?^+S<-E@9`ucif6&s=IX_s#h!e{(JE*F`^f&pS} z{e;sYj(&j2&0S&97KlW5e>|V)0w{4JeHoHXcKK*$C0E^>$BO3i<;!<{9AvwDdSn3t z8*29X1|UL^kkj9t*GT6 zFW^6ScTJF5m!I_BTS}Vi8lS2Mq#_0fHJvkpg=c(d@QEl|L9@0|PfF&1|Bw=HZ~iK~ zD^y5Wy&XWQ2Jqw;s^07clhV{=%f6Sh`5(XjMpkY<;w#5LH}mHle-H#Zss+}`ZF>E) zj`Ixw?*YYSAG;oobA9e_f{>vH|9q|$bsnAnzcdHJ<2lSpxS%kl@Kko`&>#TFodU%; z6W1scU&Y><#dXSS4Zuev<^ZZS1>F)G-R@v_tNjn20(88NP|6?*>kU*1J;0s|DYn|S zF^dAIc#JA4DgY*~6Ck9&!eHE_mm^=5^RPy+JYecZa)XwWdcqF-!Y)%`vOz>_pd%#c z3)%+4lh_KXtj(5IaAZNuk}vIdUB_*b9cUB|upcDkogttQOab)tyyZArDE)isx!6`a zxnMSkTmY=P&p2kNs;ZLy-F5whgLURkmfu-Jt*wr=HK$_al>kZaJ>BE8@mgnweu-^0 zXWP#s8sd}ydwKJrt>@@9LijOS*qOWU*Dw1Tm+5PM*yEWJUu^G-5kOdBpsq*`K92)V zM%8}v)Xb3XVa{gtOY<4c5x4xK)W?ac$b?sj+MfH)vKziB9=$r z&iMB7*Ll|VGI>R%l_4gc`OLBe00{FA-yW`HUGW0|hK-im9n}G!zVluW3nQsRzMLPo z1Ca18h^0`IQ~On&K zsquKtMhd8`tC;8!21QrV;o)wAh83;bO+K|nbb572&qW|50ZH!WLLmDf6<}R7K;rrt z#_M_u!)OQu5{1|-sLv??9dkA@n*=gU`uOUYkp(w|Yn!6`JTGgK& zt(7n-r>$@~+F)AzpEDf+Y{`-g_FxjUCKRk!# zL|$hYm&0n}y4P^xyQ+RUIk_4ErC|&!zbrVI*fB_DlE9-x730ebJn7F1tQ}wCL2Dj& zhC@+Ojm1lHNCYhu>c9{3(O>);#KRoc0m1O4;GjHGEzVMmh=}x*VonIp7G*7-u zUP;_@g|a8IiOc!#55PF4I>^J2j{Kj zYw2i~)=#A;a!MQLWi^JgBc$&_-5t%saj1o8c^T$js%tA<`Y`w3-?i4>amofG!_C}j zHjWKv4{SXTHxq^H?tWB_FB8yaNn{zum>L?U=nKMjqmt2T&CpmD{vYwrBVwu08X01G zllf+i9^C^2uLv&THXgw;{E-VZ8lC8|jvT-&_LfL?5vy~sO5a!vd{f6M8yKljc*Ml9 z0c-G2Oif)u+xYxYf@n)$RyI&C)*~)P$*gH}%GME(xoH(Ut`6suTYum7O;{&*9YI7wn< z-eJh>7awy~#B#D+){}1Kt=oQtC1DhMPEvf;fI-O(H0^Y{9tTCnl;a<|CW|E;SvkV- zR!n?-n0l;8sTatTd$15o{;X%k4CaUg73%(d6Z z&AyQVm2brcCzoxnFsE)oa_U=Te)Mr3< zoi`?b#r%gixYEKJsk5&98qB8eoF6tcT^1tZ;9Qc%`8{EvZr(3NfBMzFuOO~bvK;&l zu1K?fE}XZ9M!AC9pfVqp;}|cNF!tpAdz#p*lU3E#2s*~A3+11i9)r}I2|E*Y`rL(M zB_!Kb*p^9MXqrLLPEGUBu(YuY<`swZ%h6T?=cljl=Ci!@xzNoLqgH*>jD6v7-tKn% zH{}F&=|U%emrpvE3-!igc5;$fa@`1(Aoo(!_P|ocqzX%j?A#QTj%p3gToXrB$zyy- zl$mZFYRHsDgYQqgPpGhI{hW{ThRw<`zWNOVWoC`k^z{3$cqbDl56hnt=1m5Vc5Y4U z<~{XBogtegd1J<|1b~m`y5l%dWu6b}$h=s-Jdaibq)lI*>9us#*&~;fhTF63LGOOc zbDOx0;$js_k#ES?dcM*7r{J|cO{Aw=+*U^}_uOq8uFB@l8T!nu93hKJVmef3D=+zd zYc4CD@IVU+(P1JOooaAF86YQqD;9e_c)KM}89Ef5Z48~_aouE8KfU5!;($NB8eY6C z`eQ50Kh3hg6t0Fu(IQ}(_kCsT@AwzWv{#!^+Ip1mO1QufrMS;p{QF4WgG2f1!nr}8 zinjoyp<4-+BR`E}=8HOG-nPCt33InX*#fM#b1&P-M%pnz9Y|dS4+iOF%rQ zDaqIlio-oP+_`j*1=1wFYp?5IL<@{Vrg-YN|GXcVUcu>7V}x5t#5KQz5Ag%{4k>hk zu#(j^MG-L*c5l;=V}7Y~agvbR@|niD?c0Xy-5)Gg=gLfWg&}p-85oUNSm~QTdA=4?!?u$#+p7(;Ff=_XmioJA#f0vDwiQ{ zV8W+hn=b;Fc_u^qz%EsDB*SuRuG9D%M}xK|%nYPQO~|ME%B`*dJn*qJ_#@C@y3}bn zH%7|X{l$;{AphrU@w@U1j!{!B=-SDehMC2WZ2Y3Py`!6wLAG6PX#y+1v^4hQ4*cLJ zv9o__-`%#+yAaTadc=4kPfx>GB_E+{yfLafm|2#roNsr4Z3mK9MR;`oR6 z1`0V_>d-?v`|aVc~Vl{Bb9P@ZHB5CXldrc-<|yt>cc5Y}^_LYrzh zoOO+}dxIoZ^vUVb?!jDyAnWv78($V~ELpc0l-|@8FZ3c)@u@~anfWkG4aw58oRQa) za9v(KA{?kaEVs??k7>tWEwaYYTrIcg=9Go#OcQVpH*NQePZv8FIO%Dg{+M3v$eD;h zMT)uXzWSxF-sFNvNj6z}VC#4dZ&!9rpEqjEQ%8LxQ|$yaUl?80q%?GE0eOL$yyBTm zo?U_vw~(XAJN<=~@KGv;0nz+UrLs50W&9IIByP~RMJt5XaNi<92ie4I| z@X>>2Egi8&R838$+DD+Z8L#cCT+diY5}nl(S|9%mbP&xu%MLS#Fkft;%;Yo62Fp@` zK>IA`D(*-LTSqZGSZ>_$fZtiQ$ydXLr2TFnLRa+7q9w0tT6xxFk~p%NzSv)%p2nE8 z{d&z$8cIQFWdCn{-;;i})n?<~l-z%`dSC^}gR%r6YeeJutoyeW z0g4WQXW0nNclY1cSrabl+ogm5-=l@{K=@Z)Uz}Y1-xF`. -`unchecked_tombstone_compaction` (default: false):: - The single sstable compaction has quite strict checks for whether it - should be started, this option disables those checks and for some - usecases this might be needed. Note that this does not change anything - for the actual compaction, tombstones are only dropped if it is safe - to do so - it might just rewrite an sstable without being able to drop - any tombstones. -`only_purge_repaired_tombstone` (default: false):: - Option to enable the extra safety of making sure that tombstones are - only dropped if the data has been repaired. -`min_threshold` (default: 4):: - Lower limit of number of SSTables before a compaction is triggered. - Not used for `LeveledCompactionStrategy`. -`max_threshold` (default: 32):: - Upper limit of number of SSTables before a compaction is triggered. - Not used for `LeveledCompactionStrategy`. - -Further, see the section on each strategy for specific additional -options. - -== Compaction nodetool commands - -The `nodetool ` utility provides a number of commands related -to compaction: - -`enableautocompaction`:: - Enable compaction. -`disableautocompaction`:: - Disable compaction. -`setcompactionthroughput`:: - How fast compaction should run at most - defaults to 64MiB/s. -`compactionstats`:: - Statistics about current and pending compactions. -`compactionhistory`:: - List details about the last compactions. -`setcompactionthreshold`:: - Set the min/max sstable count for when to trigger compaction, defaults - to 4/32. - -== Switching the compaction strategy and options using JMX - -It is possible to switch compaction strategies and its options on just a -single node using JMX, this is a great way to experiment with settings -without affecting the whole cluster. The mbean is: - -[source,none] ----- -org.apache.cassandra.db:type=ColumnFamilies,keyspace=,columnfamily= ----- - -and the attribute to change is `CompactionParameters` or -`CompactionParametersJson` if you use jconsole or jmc. The syntax for -the json version is the same as you would use in an -`ALTER TABLE ` statement -for example: - -[source,none] ----- -{ 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 123, 'fanout_size': 10} ----- - -The setting is kept until someone executes an -`ALTER TABLE ` that touches the compaction -settings or restarts the node. - -[[detailed-compaction-logging]] -== More detailed compaction logging - -Enable with the compaction option `log_all` and a more detailed -compaction log file will be produced in your log directory. +* xref:managing/operating/compaction/overview.adoc[Compaction overview] +** xref:managing/operating/compaction/tombstones.adoc[Tombstones] +** Compaction strategies: +*** xref:managing/operating/compaction/ucs.adoc[Unified Compaction Strategy (UCS)] +*** xref:managing/operating/compaction/stcs.adoc[Size-Tiered Compaction Strategy (STCS)] +*** xref:managing/operating/compaction/lcs.adoc[Leveled Compaction Strategy (LCS)] +*** xref:managing/operating/compaction/twcs.adoc[Time Window Compaction Strategy (TWCS)] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/lcs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/lcs.adoc index 5b0adb8b49..43f2e944bb 100644 --- a/doc/modules/cassandra/pages/managing/operating/compaction/lcs.adoc +++ b/doc/modules/cassandra/pages/managing/operating/compaction/lcs.adoc @@ -1,81 +1,95 @@ -= Leveled Compaction Strategy += Leveled Compaction Strategy (LCS) [[lcs]] -The idea of `LeveledCompactionStrategy` (LCS) is that all sstables are -put into different levels where we guarantee that no overlapping -sstables are in the same level. By overlapping we mean that the -first/last token of a single sstable are never overlapping with other -sstables. This means that for a SELECT we will only have to look for the -partition key in a single sstable per level. Each level is 10x the size -of the previous one and each sstable is 160MB by default. L0 is where -sstables are streamed/flushed - no overlap guarantees are given here. -When picking compaction candidates we have to make sure that the -compaction does not create overlap in the target level. This is done by -always including all overlapping sstables in the next level. For example -if we select an sstable in L3, we need to guarantee that we pick all -overlapping sstables in L4 and make sure that no currently ongoing -compactions will create overlap if we start that compaction. We can -start many parallel compactions in a level if we guarantee that we wont -create overlap. For L0 -> L1 compactions we almost always need to -include all L1 sstables since most L0 sstables cover the full range. We -also can't compact all L0 sstables with all L1 sstables in a single -compaction since that can use too much memory. +include::cassandra:partial$ucs-recommend.adoc[] -When deciding which level to compact LCS checks the higher levels first -(with LCS, a "higher" level is one with a higher number, L0 being the -lowest one) and if the level is behind a compaction will be started in -that level. +The `LeveledCompactionStrategy (LCS)` is recommended for read-heavy workloads, although UCS is the best recommendation for all workloads today. +It alleviates some of the read operation issues with STCS, while providing reasonable write operations. +This strategy works with a series of levels, where each level contains a set of SSTables. +When data in memtables is flushed, SSTables are written in the first level (L0), where SSTables are not guaranteed to be non-overlapping. +LCS compaction merges these first SSTables with larger SSTables in level L1. +Each level is by default 10x the size of the previous one. +Once an SSTable is written to L1 or higher, the SSTable is guaranteed to be non-overlapping with other SSTables in the same level. +If a read operation needs to access a row, it will only need to look at one SSTable per level. -== Major compaction +To accomplish compaction, all overlapping SSTables are merged into a new SSTable in the next level. +For L0 -> L1 compactions, we almost always need to include all L1 SSTables since most L0 SSTables cover the full range of partitions. +LCS compacts SSTables from one level to the next, writing partitions to fit a defined SSTable size. +In addition, each level has a prescribed size, so that compaction will be triggered when a level reaches its size limit. +Creating new SSTables in one level can trigger compaction in the next level, and so on, until all levels have been compacted based on the settings. -It is possible to do a major compaction with LCS - it will currently -start by filling out L1 and then once L1 is full, it continues with L2 -etc. This is sub optimal and will change to create all the sstables in a -high level instead, CASSANDRA-11817. +There is a failsafe if too many SSTables reads are being done in the L0 level. +An STCS compaction will be triggered in L0 if there are more than 32 SSTables in L0. +This compaction quickly merges SSTables out of L0, and into L1, where they will be compacted to non-overlapping SSTables. + +LCS is not as disk hungry as STCS, needing only approximately 10% of disk to execute, but it is more IO and CPU intensive. +For ongoing minor compactions in a read-heavy workload, the amount of compaction is reasonable. +It is not a good choice for write-heavy workloads, though, because it will cause a lot of disk IO and CPU usage. +Major compactions are not recommended for LCS. == Bootstrapping -During bootstrap sstables are streamed from other nodes. The level of -the remote sstable is kept to avoid many compactions after the bootstrap -is done. During bootstrap the new node also takes writes while it is -streaming the data from a remote node - these writes are flushed to L0 -like all other writes and to avoid those sstables blocking the remote -sstables from going to the correct level, we only do STCS in L0 until -the bootstrap is done. - -== STCS in L0 - -If LCS gets very many L0 sstables reads are going to hit all (or most) -of the L0 sstables since they are likely to be overlapping. To more -quickly remedy this LCS does STCS compactions in L0 if there are more -than 32 sstables there. This should improve read performance more -quickly compared to letting LCS do its L0 -> L1 compactions. If you keep -getting too many sstables in L0 it is likely that LCS is not the best -fit for your workload and STCS could work out better. +During bootstrapping, SSTables are streamed from other nodes. +Because many SSTables will be both flushed from the new writes to memtables, as well as streaming from a remote note, the new node will have many SSTables in L0. +To avoid a collision of the flushing and streaming SSTables, only STCS in L0 is executed until the bootstrapping is complete. == Starved sstables -If a node ends up with a leveling where there are a few very high level -sstables that are not getting compacted they might make it impossible -for lower levels to drop tombstones etc. For example, if there are -sstables in L6 but there is only enough data to actually get a L4 on the -node the left over sstables in L6 will get starved and not compacted. -This can happen if a user changes sstable_size_in_mb from 5MB to 160MB -for example. To avoid this LCS tries to include those starved high level -sstables in other compactions if there has been 25 compaction rounds -where the highest level has not been involved. +If the leveling is not optimal, LCS can end up with starved sstables. +High level SSTables can be stranded and not compacted, because SSTables in lower levels are not getting merged and compacted. +For example, this situation can make it impossible for lower levels to drop tombstones. +If these starved SSTables are not resolved within a defined number of compaction rounds, they will be included in other compactions. +This situation generally occurs if a user lowers the `sstable_size` setting. + +include::cassandra:partial$default-compaction-strategy.adoc[] [[lcs_options]] == LCS options -`sstable_size_in_mb` (default: 160MB):: - The target compressed (if using compression) sstable size - the - sstables can end up being larger if there are very large partitions on - the node. -`fanout_size` (default: 10):: - The target size of levels increases by this fanout_size multiplier. - You can reduce the space amplification by tuning this option. +[cols="1,2"] +|=== +| Subproperty | Description -LCS also support the `cassandra.disable_stcs_in_l0` startup option -(`-Dcassandra.disable_stcs_in_l0=true`) to avoid doing STCS in L0. +| enabled +| Enables background compaction. +Default value: true +// See Enabling and disabling background compaction. + +| tombstone_compaction_interval +| The minimum number of seconds after which an SSTable is created before {cassandra} considers the SSTable for tombstone compaction. +An SSTable is eligible for tombstone compaction if the table exceeds the `tombstone_threshold` ratio. +Default value: 86400 + +| tombstone_threshold +| The ratio of garbage-collectable tombstones to all contained columns. +If the ratio exceeds this limit, {cassandra} starts compaction on that table alone, to purge the tombstones. +Default value: 0.2 + +| unchecked_tombstone_compaction +| If set to `true`, allows {cassandra} to run tombstone compaction without pre-checking which tables are eligible for this operation. +Even without this pre-check, {cassandra} checks an SSTable to make sure it is safe to drop tombstones. +Default value: false + +| log_all +| Activates advanced logging for the entire cluster. +Default value: false + +| sstable_size_in_mb +|The target size for SSTables. +Although SSTable sizes should be less or equal to sstable_size_in_mb, it is possible that compaction could produce a larger SSTable during compaction. +This occurs when data for a given partition key is exceptionally large. +The {cassandra} database does not split the data into two SSTables. +Default: 160 + +| fanout_size +| The target size of levels increases by this `fanout_size` multiplier. +You can reduce the space amplification by tuning this option. +Default: 10 + +| single_sstable_uplevel +| ??? +Default: true +|=== + +LCS also supports a startup option, `-Dcassandra.disable_stcs_in_l0=true` which disables STCS in L0. \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc new file mode 100644 index 0000000000..3fdbb6d5a7 --- /dev/null +++ b/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc @@ -0,0 +1,235 @@ += Compaction overview + +== What is compaction? + +Data in {cassandra} is created in xref:cassandra:architecture/storage-engine.adoc#memtables[memtables]. +Once a memory threshold is reached, to free up memory again, the data is written to an xref:cassandra:architecture/storage-engine.adoc#SSTables[SSTable], an https://cassandra.apache.org/_/glossary.html#immutable[immutable] file residing on disk. + +Because SSTables are immutable, when data is updated or deleted, the old data is not overwritten with inserts or updates, or removed from the SSTable. +Instead, a new SSTable is created with the updated data with a new timestamp, and the old SSTable is marked for deletion. +The piece of deleted data is known as a https://cassandra.apache.org/_/glossary.html#tombstone[tombstone]. + +Over time, Cassandra may write many versions of a row in different SSTables. +Each version may have a unique set of columns stored with a different timestamp. +As SSTables accumulate, the distribution of data can require accessing more and more SSTables to retrieve a complete row. + +To keep the database healthy, Cassandra periodically merges SSTables and discards old data. +This process is called https://cassandra.apache.org/_/glossary.html#compaction[compaction]. + +== Why must compaction be run? + +Since SSTables are consulted during read operations, it is important to keep the number of SSTables small. +Write operations will cause the number of SSTables to grow, so compaction is necessary. +Besides the issue of tombstones, data is deleted for other reasons, too, such as Time-To-Live (TTL) expiration of some data. +Deleting, updating, or expiring data are all valid triggers for compaction. + +== What does compaction accomplish? + +Two important factors accomplished by compaction are performance improvement and disk space reclamation. +If SSTables have duplicate data that must be read, read operations are slower. +Once tombstones and duplicates are removed, read operations are faster. +SSTables use disk space, and reducing the size of SSTables through compaction frees up disk space. + +== How does compaction work? + +Compaction works on a collection of SSTables. +From these SSTables, compaction collects all versions of each unique row and assembles one complete row, using the most up-to-date version (by timestamp) of each of the row's columns. +The merge process is performant, because rows are sorted by partition key within each SSTable, and the merge process does not use random I/O. +The new versions of each row is written to a new SSTable. +The old versions, along with any rows that are ready for deletion, are left in the old SSTables, and are deleted as soon as pending reads are completed. + +== Types of compaction + +The concept of compaction is used for different kinds of operations in +{cassandra}, the common thing about these operations is that it takes one +or more SSTables, merges, and outputs new SSTables. The types of compactions are: + +Minor compaction:: +A minor compaction triggered automatically in {cassandra} for several actions: ++ +* When an SSTable is added to the node through flushing +* When autocompaction is enabled after being disabled (`nodetool enableautocompaction`) +* When compaction adds new SSTables +* A check for new minor compactions every 5 minutes +Major compaction:: +A major compaction is triggered when a user executes a compaction over all SSTables on the node. +User defined compaction:: +Similar to a major compaction, a user-defined compaction executes when a user triggers a compaction on a given set of SSTables. +Scrub:: +A scrub triggers a compaction to try to fix any broken SSTables. +This can actually remove valid data if that data is corrupted. +If that happens you will need to run a full repair on the node. +UpgradeSSTables:: +A compaction occurs when you upgrade SSTables to the latest version. +Run this after upgrading to a new major version. +Cleanup:: +Compaction executes to remove any ranges that a node no longer owns. +This type of compaction is typically triggered on neighbouring nodes after a node has been bootstrapped, since the bootstrapping node will take ownership of some ranges from those nodes. +Secondary index rebuild:: +A compaction is triggered if the secondary indexes are rebuilt on a node. +Anticompaction:: +After repair, the ranges that were actually repaired are split out of the SSTables that existed when repair started. This type of compaction rewrites SSTables to accomplish this task. +Sub range compaction:: +It is possible to only compact a given sub range - this action is useful if you know a token that has been misbehaving - either gathering many updates or many deletes. +The command `nodetool compact -st x -et y` will pick all SSTables containing the range between x and y and issue a compaction for those SSTables. +For Size Tiered Compaction Strategy, this will most likely include all SSTables, but with Leveled Compaction Strategy, it can issue the compaction for a subset of the SSTables. +With LCS the resulting SSTable will end up in L0. + +== Strategies + +Different compaction strategies are available to optimize for different workloads. +Picking the right compaction strategy for your workload will ensure the best performance for both querying and for compaction itself. + +xref:cassandra:managing/operating/compaction/ucs.adoc[`Unified Compaction Strategy (UCS)`]:: +UCS is a good choice for most workloads and is recommended for new workloads. +This compaction strategy is designed to handle a wide variety of workloads. +It is designed to be able to handle both immutable time-series data and workloads with lots of updates and deletes. +It is also designed to be able to handle both spinning disks and SSDs. +xref:cassandra:managing/operating/compaction/stcs.adoc[`Size Tiered Compaction Strategy (STCS)`]:: +STCS is the default compaction strategy, because it is useful as a fallback when other strategies don't fit the workload. +Most useful for not strictly time-series workloads with spinning disks, or when the I/O from `LCS` is too high. +xref:cassandra:managing/operating/compaction/lcs.adoc[`Leveled Compaction Strategy (LCS)`]:: +Leveled Compaction Strategy (LCS) is optimized for read heavy workloads, or workloads with lots of updates and deletes. +It is not a good choice for immutable time-series data. +xref:cassandra:managing/operating/compaction/twcs.adoc[`Time Window Compaction Strategy (TWCS)`]:: +Time Window Compaction Strategy is designed for TTL'ed, mostly immutable time-series data. + +include::tombstones.adoc[leveloffset=+1] + +== TTL + +Data in Cassandra can have an additional property called time to live - +this is used to automatically drop data that has expired once the time +is reached. Once the TTL has expired the data is converted to a +tombstone which stays around for at least `gc_grace_seconds`. Note that +if you mix data with TTL and data without TTL (or just different length +of the TTL) Cassandra will have a hard time dropping the tombstones +created since the partition might span many SSTables and not all are +compacted at once. + +== Fully expired SSTables + +If an SSTable contains only tombstones and it is guaranteed that +SSTable is not shadowing data in any other SSTable, then the compaction can drop +that SSTable. If you see SSTables with only tombstones (note that TTL-ed +data is considered tombstones once the time-to-live has expired), but it +is not being dropped by compaction, it is likely that other SSTables +contain older data. There is a tool called `sstableexpiredblockers` that +will list which SSTables are droppable and which are blocking them from +being dropped. With `TimeWindowCompactionStrategy` it +is possible to remove the guarantee (not check for shadowing data) by +enabling `unsafe_aggressive_sstable_expiration`. + +== Repaired/unrepaired data + +With incremental repairs Cassandra must keep track of what data is +repaired and what data is unrepaired. With anticompaction repaired data +is split out into repaired and unrepaired SSTables. To avoid mixing up +the data again separate compaction strategy instances are run on the two +sets of data, each instance only knowing about either the repaired or +the unrepaired SSTables. This means that if you only run incremental +repair once and then never again, you might have very old data in the +repaired SSTables that block compaction from dropping tombstones in the +unrepaired (probably newer) SSTables. + +== Data directories + +Since tombstones and data can live in different SSTables it is important +to realize that losing an SSTable might lead to data becoming live again +- the most common way of losing SSTables is to have a hard drive break +down. To avoid making data live tombstones and actual data are always in +the same data directory. This way, if a disk is lost, all versions of a +partition are lost and no data can get undeleted. To achieve this a +compaction strategy instance per data directory is run in addition to +the compaction strategy instances containing repaired/unrepaired data, +this means that if you have 4 data directories there will be 8 +compaction strategy instances running. This has a few more benefits than +just avoiding data getting undeleted: + +* It is possible to run more compactions in parallel - leveled +compaction will have several totally separate levelings and each one can +run compactions independently from the others. +* Users can backup and restore a single data directory. +* Note though that currently all data directories are considered equal, +so if you have a tiny disk and a big disk backing two data directories, +the big one will be limited the by the small one. One work around to +this is to create more data directories backed by the big disk. + +== Single SSTable tombstone compaction + +When an SSTable is written a histogram with the tombstone expiry times +is created and this is used to try to find SSTables with very many +tombstones and run single SSTable compaction on that SSTable in hope of +being able to drop tombstones in that SSTable. Before starting this it +is also checked how likely it is that any tombstones will actually will +be able to be dropped how much this SSTable overlaps with other +SSTables. To avoid most of these checks the compaction option +`unchecked_tombstone_compaction` can be enabled. + +[[compaction-options]] +== Common options + +There is a number of common options for all the compaction strategies; + +`enabled` (default: true):: +Whether minor compactions should run. Note that you can have 'enabled': true as a compaction option and then do 'nodetool enableautocompaction' to start running compactions. +`tombstone_threshold` (default: 0.2):: +How much of the SSTable should be tombstones for us to consider doing a single SSTable compaction of that SSTable. +`tombstone_compaction_interval` (default: 86400s (1 day)):: +Since it might not be possible to drop any tombstones when doing a single SSTable compaction we need to make sure that one SSTable is not constantly getting recompacted - this option states how often we should try for a given SSTable. +`log_all` (default: false):: +New detailed compaction logging, see `below `. +`unchecked_tombstone_compaction` (default: false):: +The single SSTable compaction has quite strict checks for whether it should be started, this option disables those checks and for some use cases this might be needed. +Note that this does not change anything for the actual compaction, tombstones are only dropped if it is safe to do so - it might just rewrite an SSTable without being able to drop any tombstones. +`only_purge_repaired_tombstone` (default: false):: +Option to enable the extra safety of making sure that tombstones are only dropped if the data has been repaired. +`min_threshold` (default: 4):: +Lower limit of number of SSTables before a compaction is triggered. +Not used for `LeveledCompactionStrategy`. +`max_threshold` (default: 32):: +Upper limit of number of SSTables before a compaction is triggered. +Not used for `LeveledCompactionStrategy`. + +Further, see the section on each strategy for specific additional options. + +== Compaction nodetool commands + +The `nodetool ` utility provides a number of commands related to compaction: + +`enableautocompaction`:: +Enable compaction. +`disableautocompaction`:: +Disable compaction. +`setcompactionthroughput`:: +How fast compaction should run at most - defaults to 64MiB/s. +`compactionstats`:: +Statistics about current and pending compactions. +`compactionhistory`:: +List details about the last compactions. +`setcompactionthreshold`:: +Set the min/max SSTable count for when to trigger compaction, defaults to 4/32. + +== Switching the compaction strategy and options using JMX + +It is possible to switch compaction strategies and its options on just a single node using JMX, this is a great way to experiment with settings without affecting the whole cluster. +The mbean is: + +[source,console] +---- +org.apache.cassandra.db:type=ColumnFamilies,keyspace=,columnfamily= +---- + +and the attribute to change is `CompactionParameters` or `CompactionParametersJson` if you use jconsole or jmc. For example, the syntax for the json version is the same as you would use in an `ALTER TABLE ` statement: + +[source,console] +---- +{ 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 123, 'fanout_size': 10} +---- + +The setting is kept until someone executes an `ALTER TABLE ` that touches the compaction settings or restarts the node. + +[[detailed-compaction-logging]] +== More detailed compaction logging + +Enable with the compaction option `log_all` and a more detailed compaction log file will be produced in your log directory. diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/stcs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/stcs.adoc index 162d2fec5e..c31635ae0d 100644 --- a/doc/modules/cassandra/pages/managing/operating/compaction/stcs.adoc +++ b/doc/modules/cassandra/pages/managing/operating/compaction/stcs.adoc @@ -1,42 +1,96 @@ -= Size Tiered Compaction Strategy += Size Tiered Compaction Strategy (STCS) [[stcs]] -The basic idea of `SizeTieredCompactionStrategy` (STCS) is to merge -sstables of approximately the same size. All sstables are put in -different buckets depending on their size. An sstable is added to the -bucket if size of the sstable is within `bucket_low` and `bucket_high` -of the current average size of the sstables already in the bucket. This -will create several buckets and the most interesting of those buckets -will be compacted. The most interesting one is decided by figuring out -which bucket's sstables takes the most reads. -== Major compaction +include::cassandra:partial$ucs-recommend.adoc[] -When running a major compaction with STCS you will end up with two -sstables per data directory (one for repaired data and one for -unrepaired data). There is also an option (-s) to do a major compaction -that splits the output into several sstables. The sizes of the sstables -are approximately 50%, 25%, 12.5%... of the total size. +The `SizeTieredCompactionStrategy (STCS)` is recommended for write-intensive workloads, and is the legacy recommended compaction strategy. +It is the default compaction strategy if no other strategy is specified. + +STCS initiates compaction when {cassandra} has accumulated a set number (default: 4) of similar-sized SSTables. +STCS merges these SSTables into one larger SSTable. +As these larger SSTables accumulate, STCS merges them into even larger SSTables. +At any given time, several SSTables of varying sizes are present. + +While STCS works well to compact a write-intensive workload, it makes reads slower because the merge-by-size process does not group data by rows. +This fact makes it more likely that versions of a particular row may be spread over many SSTables. +Also, STCS does not evict deleted data predictably, because its trigger for compaction is SSTable size. +However, SSTables may not grow quickly enough to merge and evict old data regularly. + +Most STCS compactions are minor compactions, which merge a few SSTables into one. +In contrast, when executing a major compaction with STCS, two SSTables per data directory, one for repaired data and one for unrepaired data, will exist during the compaction. +As the largest SSTables grow in size, the amount of disk space needed for both the new and old SSTables simultaneously during STCS compaction can outstrip a typical amount of disk space on a node. +This phenomenon is known as space amplification, the problem of growing SSTable size, and the issue of outgrowing a cluster's ability to do compaction. +Major compactions are not recommended for STCS. + +STCS depends on the calculation of the average size of SSTables to determine which SSTables to merge. +This process is called bucketing. +The following options are used to calculate the bucket into which an SSTable will be grouped, based on that average size. +The bucketing process groups SSTables based on their size differing from the average size by either 50% or 150% more than the average size. +Another way to state this calculation is that the bucketing process groups SSTables with a size within [average-size × bucket_low] and [average-size × bucket_high]. + +include::cassandra:partial$default-compaction-strategy.adoc[] [[stcs_options]] == STCS options -`min_sstable_size` (default: 50MB):: - Sstables smaller than this are put in the same bucket. -`bucket_low` (default: 0.5):: - How much smaller than the average size of a bucket a sstable should be - before not being included in the bucket. That is, if - `bucket_low * avg_bucket_size < sstable_size` (and the `bucket_high` - condition holds, see below), then the sstable is added to the bucket. -`bucket_high` (default: 1.5):: - How much bigger than the average size of a bucket a sstable should be - before not being included in the bucket. That is, if - `sstable_size < bucket_high * avg_bucket_size` (and the `bucket_low` - condition holds, see above), then the sstable is added to the bucket. +SizeTieredCompactionStrategy (STCS) options are set per table using table options. +The `min_threshold` option of a table is the main value that triggers a minor compaction. +Minor compactions do not involve all the tables in a keyspace. -== Defragmentation +[cols="1,2"] +|=== +| Subproperty | Description -Defragmentation is done when many sstables are touched during a read. -The result of the read is put in to the memtable so that the next read -will not have to touch as many sstables. This can cause writes on a -read-only-cluster. +| enabled +| Enables background compaction. +Default value: true +// See Enabling and disabling background compaction. + +| tombstone_compaction_interval +| The minimum number of seconds after which an SSTable is created before {cassandra} considers the SSTable for tombstone compaction. +An SSTable is eligible for tombstone compaction if the table exceeds the `tombstone_threshold` ratio. +Default value: 86400 + +| tombstone_threshold +| The ratio of garbage-collectable tombstones to all contained columns. +If the ratio exceeds this limit, {cassandra} starts compaction on that table alone, to purge the tombstones. +Default value: 0.2 + +| unchecked_tombstone_compaction +| If set to `true`, allows {cassandra} to run tombstone compaction without pre-checking which tables are eligible for this operation. +Even without this pre-check, {cassandra} checks an SSTable to make sure it is safe to drop tombstones. +Default value: false + +| log_all +| Activates advanced logging for the entire cluster. +Default value: false + +| max_threshold +| The maximum number of SSTables to allow in a minor compaction. +Default value: 32 + +| min_threshold +| The minimum number of SSTables to trigger a minor compaction. +Default value: 4 + +| bucket_high +| An SSTable is added to a bucket if its size is less than 150% of the average size of that bucket. +For example, if the SSTable size is 13 MB, and the bucket average size is 10 MB, then the SSTable will be added to that bucket and the new average size will be computed for that bucket. +Default value: 1.5 + +| bucket_low +| An SSTable is added to a bucket if the SSTable size is greater than 50% of the average size of that bucket. +For example, if the SSTable size is 6 MB, and the bucket average size is 10 MB, then the SSTable will be added to that bucket and the new average size will be computed for that bucket. +Default value: 0.5 + +| min_sstable_size +| SSTables smaller than this value will be grouped into one bucket where the average size is less than this setting. +Default value: 50MB + +| only_purge_repaired_tombstones +| If set to `true`, allows purging tombstones only from repaired SSTables. +The purpose is to prevent data from resurrecting if repair is not run within `gc_grace_seconds`. +If you do not run repair for a long time, {cassandra} keeps all tombstones — this may cause problems. +Default value: false +|=== \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc new file mode 100644 index 0000000000..3959286524 --- /dev/null +++ b/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc @@ -0,0 +1,129 @@ += Tombstones + +// and Garbage Collection (GC) Grace + +== What are tombstones? + +{cassandra}'s processes for deleting data are designed to improve performance, and to work with {cassandra}'s built-in properties for data distribution and fault-tolerance. + +{cassandra} treats a deletion as an insertion, and inserts a time-stamped deletion marker called a tombstone. +The tombstones go through {cassandra}'s write path, and are written to SSTables on one or more nodes. +The key feature difference of a tombstone is that it has a built-in expiration date/time. +At the end of its expiration period, the grace period, the tombstone is deleted as part of {cassandra}'s normal compaction process. + +[NOTE] +==== +You can also mark a {cassandra} row or column with a time-to-live (TTL) value. +After this amount of time has ended, {cassandra} marks the object with a tombstone, and handles it like other tombstoned objects. +==== + +== Why tombstones? + +The tombstone represents the deletion of an object, either a row or column value. +This approach is used instead of removing values because of the distributed nature of {cassandra}. +Once an object is marked as a tombstone, queries will ignore all values that are time-stamped previous to the tombstone insertion. + +== Zombies + +In a multi-node cluster, {cassandra} may store replicas of the same data on two or more nodes. +This helps prevent data loss, but it complicates the deletion process. +If a node receives a delete command for data it stores locally, the node tombstones the specified object and tries to pass the tombstone to other nodes containing replicas of that object. +But if one replica node is unresponsive at that time, it does not receive the tombstone immediately, so it still contains the pre-delete version of the object. +If the tombstoned object has already been deleted from the rest of the cluster before that node recovers, {cassandra} treats the object on the recovered node as new data, and propagates it to the rest of the cluster. +This kind of deleted but persistent object is called a https://cassandra.apache.org/_/glossary.html#zombie[zombie]. + +== Grace period + +To prevent the reappearance of zombies, {cassandra} gives each tombstone a grace period. +The grace period for a tombstone is set with the table property ` WITH gc_grace_seconds`. +Its default value is 864000 seconds (ten days), after which a tombstone expires and can be deleted during compaction. +Prior to the grace period expiring, {cassandra} will retain a tombstone through compaction events. +Each table can have its own value for this property. + +The purpose of the grace period is to give unresponsive nodes time to recover and process tombstones normally. +If a client writes a new update to the tombstoned object during the grace period, {cassandra} overwrites the tombstone. +If a client sends a read for that object during the grace period, {cassandra} disregards the tombstone and retrieves the object from other replicas if possible. + +When an unresponsive node recovers, {cassandra} uses hinted handoff to replay the database mutations the node missed while it was down. +{cassandra} does not replay a mutation for a tombstoned object during its grace period. +But if the node does not recover until after the grace period ends, {cassandra} may miss the deletion. + +After the tombstone's grace period ends, {cassandra} deletes the tombstone during compaction. + +== Deletion + +After `gc_grace_seconds` has expired the tombstone may be removed (meaning there will no longer be any object that a certain piece of data was +deleted). +But one complication for deletion is that a tombstone can live in one SSTable and the data it marks for deletion in another, so a compaction must also remove both SSTables. +More precisely, drop an actual tombstone the: + +* The tombstone must be older than `gc_grace_seconds`. +Note that tombstones will not be removed until a compaction event even if `gc_grace_seconds` has elapsed. +* If partition X contains the tombstone, the SSTable containing the partition plus all SSTables containing data older than the tombstone containing X must be included in the same compaction. +If all data in any SSTable containing partition X is newer than the tombstone, it can be ignored. +* If the option `only_purge_repaired_tombstones` is enabled, tombstones are only removed if the data has also been repaired. +This process is described in the "Deletes with tombstones" sections. + +If a node remains down or disconnected for longer than `gc_grace_seconds`, its deleted data will be repaired back to the other nodes and reappear in the cluster. +This is basically the same as in the "Deletes without Tombstones" section. + +=== Deletes without tombstones + +Imagine a three node cluster which has the value [A] replicated to every +node.: + +[source,none] +---- +[A], [A], [A] +---- + +If one of the nodes fails and and our delete operation only removes existing values, we can end up with a cluster that looks like: + +[source,none] +---- +[], [], [A] +---- + +Then a repair operation would replace the value of [A] back onto the two nodes which are missing the value.: + +[source,none] +---- +[A], [A], [A] +---- + +This would cause our data to be resurrected as a zombie even though it had been deleted. + +=== Deletes with tombstones + +Starting again with a three node cluster which has the value [A] replicated to every node.: + +[source,none] +---- +[A], [A], [A] +---- + +If instead of removing data we add a tombstone object, so the single node failure situation will look like: + +[source,none] +---- +[A, Tombstone[A]], [A, Tombstone[A]], [A] +---- + +Now when we issue a repair the tombstone will be copied to the replica, rather than the deleted data being resurrected: + +[source,none] +---- +[A, Tombstone[A]], [A, Tombstone[A]], [A, Tombstone[A]] +---- + +Our repair operation will correctly put the state of the system to what we expect with the object [A] marked as deleted on all nodes. +This does mean we will end up accruing tombstones which will permanently accumulate disk space. +To avoid keeping tombstones forever, we set `gc_grace_seconds` for every table in {cassandra}. + +== Fully expired SSTables + +If an SSTable contains only tombstones and it is guaranteed that SSTable is not shadowing data in any other SSTable, then the compaction can drop +that SSTable. +If you see SSTables with only tombstones (note that TTL'd data is considered tombstones once the time-to-live has expired), but it is not being dropped by compaction, it is likely that other SSTables contain older data. +There is a tool called `sstableexpiredblockers` that will list which SSTables are droppable and which are blocking them from being dropped. +With `TimeWindowCompactionStrategy` it is possible to remove the guarantee (not check for shadowing data) by enabling `unsafe_aggressive_sstable_expiration`. diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/twcs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/twcs.adoc index e0553b2896..9127edbf4e 100644 --- a/doc/modules/cassandra/pages/managing/operating/compaction/twcs.adoc +++ b/doc/modules/cassandra/pages/managing/operating/compaction/twcs.adoc @@ -1,75 +1,108 @@ -= Time Window Compaction Strategy += Time Window Compaction Strategy (TWCS) [[twcs]] -`TimeWindowCompactionStrategy` (TWCS) is designed specifically for -workloads where it's beneficial to have data on disk grouped by the -timestamp of the data, a common goal when the workload is time-series in -nature or when all data is written with a TTL. In an expiring/TTL -workload, the contents of an entire SSTable likely expire at -approximately the same time, allowing them to be dropped completely, and -space reclaimed much more reliably than when using -`SizeTieredCompactionStrategy` or `LeveledCompactionStrategy`. The basic -concept is that `TimeWindowCompactionStrategy` will create one sstable per -file for a given window, where a window is simply calculated as the -combination of two primary options: -[[twcs_options]] +include::cassandra:partial$ucs-recommend.adoc[] -`compaction_window_unit` (default: DAYS):: - A Java TimeUnit (MINUTES, HOURS, or DAYS). -`compaction_window_size` (default: 1):: - The number of units that make up a window. -`unsafe_aggressive_sstable_expiration` (default: false):: - Expired sstables will be dropped without checking its data is - shadowing other sstables. This is a potentially risky option that can - lead to data loss or deleted data re-appearing, going beyond what - unchecked_tombstone_compaction does for single sstable - compaction. Due to the risk the jvm must also be started with - `-Dcassandra.unsafe_aggressive_sstable_expiration=true`. +The `TimeWindowCompactionStrategy` (TWCS) is the recommended compaction strategy for time-series and expiring Time-To-Live (TTL) workloads. +If the workload has data grouped by timestamp, TWCS can be used to group SSTables by time window, and then drop entire SSTables when they expire. +Thus, disk space can be reclaimed much more reliably than when using STCS or LCS. + +TWCS compacts SSTables using a series of time windows buckets. +During the active time window, TWCS compacts all SSTables flushed from memory into larger SSTables using STCS. +At the end of the time period, all of these SSTables are compacted into a single SSTable based on the SSTable maximum timestamp. +Once the major compaction for a time window is completed, no further compaction of the data will ever occur. +The process starts over with the SSTables written in the next time window. +Notice that each TWCS time window contains varying amounts of data. +Then the next time window starts and the process repeats. -Taken together, the operator can specify windows of virtually any size, -and `TimeWindowCompactionStrategy` will work to create a -single sstable for writes within that window. For efficiency during -writing, the newest window will be compacted using -`SizeTieredCompactionStrategy`. -Ideally, operators should select a `compaction_window_unit` and -`compaction_window_size` pair that produces approximately 20-30 windows -- if writing with a 90 day TTL, for example, a 3 Day window would be a -reasonable choice -(`'compaction_window_unit':'DAYS','compaction_window_size':3`). == TimeWindowCompactionStrategy Operational Concerns -The primary motivation for TWCS is to separate data on disk by timestamp -and to allow fully expired SSTables to drop more efficiently. One -potential way this optimal behavior can be subverted is if data is -written to SSTables out of order, with new data and old data in the same -SSTable. Out of order data can appear in two ways: +The primary motivation for TWCS is to separate data on disk by timestamp and to allow fully expired SSTables to drop more efficiently. +One potential way this optimal behavior can be subverted is if data is written to SSTables out of order, with new data and old data in the same SSTable. -* If the user mixes old data and new data in the traditional write path, -the data will be comingled in the memtables and flushed into the same -SSTable, where it will remain comingled. -* If the user's read requests for old data cause read repairs that pull -old data into the current memtable, that data will be comingled and -flushed into the same SSTable. +Out of order data can appear in two ways: + +* If the user mixes old data and new data in the traditional write path, the data will be comingled in the memtables and flushed into the same SSTable, where it will remain comingled. +* If the user's read requests for old data cause read repairs that pull old data into the current memtable, that data will be comingled and flushed into the same SSTable. + +While TWCS tries to minimize the impact of comingled data, users should attempt to avoid this behavior. Specifically, users should avoid queries that explicitly set the timestamp via CQL `USING TIMESTAMP`. +Additionally, users should run frequent repairs (which streams data in such a way that it does not become comingled). + +include::cassandra:partial$default-compaction-strategy.adoc[] + +[[twcs_options]] +== TWCS Options + +[cols="1,2"] +|=== +| Subproperty | Description + +| enabled +| Enables background compaction. +Default value: true +// See Enabling and disabling background compaction. + +| tombstone_compaction_interval +| The minimum number of seconds after which an SSTable is created before {cassandra} considers the SSTable for tombstone compaction. +An SSTable is eligible for tombstone compaction if the table exceeds the `tombstone_threshold` ratio. +Default value: 86400 + +| tombstone_threshold +| The ratio of garbage-collectable tombstones to all contained columns. +If the ratio exceeds this limit, {cassandra} starts compaction on that table alone, to purge the tombstones. +Default value: 0.2 + +| unchecked_tombstone_compaction +| If set to `true`, allows {cassandra} to run tombstone compaction without pre-checking which tables are eligible for this operation. +Even without this pre-check, {cassandra} checks an SSTable to make sure it is safe to drop tombstones. +Default value: false + +| log_all +| Activates advanced logging for the entire cluster. +Default value: false + +| compaction_window_unit +| Time unit used to define the bucket size. +The value is based on the Java TimeUnit. +For the list of valid values, see the Java API TimeUnit page located at https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html. +Default: days + +| compaction_window_size +| Units per bucket. +Default value: 1 + +|timestamp_resolution +| The timestamp resolution used when inserting data, can be MILLISECONDS, MICROSECONDS etc (should be understandable by Java TimeUnit). +Do not change this value unless you do mutations with USING TIMESTAMP (or equivalent directly in the client). +Default: microseconds + +| expired_sstable_check_frequency_seconds +| Determines how often to check for expired SSTables. +Default: 10 minutes + +| unsafe_aggressive_sstable_expiration +| Expired SSTables will be dropped without checking its data is shadowing other SSTables. +This is a potentially risky option that can lead to data loss or deleted data re-appearing, going beyond what `unchecked_tombstone_compaction` does for single SSTable compaction. +Due to the risk, the JVM must also be started with the option: `-Dcassandra unsafe_aggressive_sstable_expiration=true`. +Default: false +|=== + +Taken together, the operator can specify windows of virtually any size, and TWCS will work to create a +single SSTable for writes within that window. +For efficiency during writing, the newest window will be compacted using STCS. + +Ideally, operators should select a `compaction_window_unit` and `compaction_window_size` pair that produces approximately 20-30 windows. +If writing with a 90 day TTL, for example, a 3 Day window would be a reasonable choice, setting the options to `'compaction_window_unit':'DAYS' and 'compaction_window_size':3`. -While TWCS tries to minimize the impact of comingled data, users should -attempt to avoid this behavior. Specifically, users should avoid queries -that explicitly set the timestamp via CQL `USING TIMESTAMP`. -Additionally, users should run frequent repairs (which streams data in -such a way that it does not become comingled). == Changing TimeWindowCompactionStrategy Options -Operators wishing to enable `TimeWindowCompactionStrategy` on existing -data should consider running a major compaction first, placing all -existing data into a single (old) window. Subsequent newer writes will -then create typical SSTables as expected. +Operators wishing to enable TWCS on existing data should consider running a major compaction first, placing all existing data into a single (old) window. +Subsequent newer writes will then create typical SSTables as expected. -Operators wishing to change `compaction_window_unit` or -`compaction_window_size` can do so, but may trigger additional -compactions as adjacent windows are joined together. If the window size -is decrease d (for example, from 24 hours to 12 hours), then the -existing SSTables will not be modified - TWCS can not split existing -SSTables into multiple windows. +Operators wishing to change `compaction_window_unit` or `compaction_window_size` can do so, but may trigger additional compactions as adjacent windows are joined together. +If the window size is decrease d (for example, from 24 hours to 12 hours), then the existing SSTables will not be modified. +TWCS can not split existing SSTables into multiple windows. \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/ucs.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/ucs.adoc new file mode 100644 index 0000000000..4798f0b6fd --- /dev/null +++ b/doc/modules/cassandra/pages/managing/operating/compaction/ucs.adoc @@ -0,0 +1,626 @@ += Unified Compaction Strategy (UCS) +:stem: latexmath + +[[ucs]] +The `UnifiedCompactionStrategy (UCS)` is recommended for most workloads, whether read-heavy, write-heavy, mixed read-write, or time-series. +There is no need to use legacy compaction strategies, because UCS can be configured to behave like any of them. + +UCS is a compaction strategy that combines the best of the other strategies plus new features. +UCS has been designed to maximize the speed of compactions, which is crucial for high-density nodes, using an unique sharding mechanism that compacts partitioned data in parallel. +And whereas STCS, LCS, or TWCS will require full compaction of the data if the compaction strategy is changed, UCS can change parameters in flight to switch from one strategy to another. +In fact, a combination of different compaction strategies can be used at the same time, with different parameters for each level of the hierarchy. +Finally, UCS is stateless, so it does not rely on any metadata to make compaction decisions. + +Two key concepts refine the definition of the grouping: + +* Tiered and levelled compaction can be generalized as equivalent, because both create exponentially-growing levels based on the *size* of SSTables (or non-overlapping SSTable runs). +Thus, a compaction is triggered when more than a given number of SSTables are present on one level. +* *size* can be replaced by *density*, allowing SSTables to be split at arbitrary points when the output of a compaction is written, while still producing a leveled hierarchy. +Density is defined as the size of an SSTable divided by the width of the token range it covers. + +Let's look at the first concept in more detail. + +== Read and write amplification + +UCS can adjust the balance between the number of SSTables consulted to serve a read (read amplification, or RA) versus the number of times a piece of data must be rewritten during its lifetime (write amplification, or WA). +A single configurable scaling parameter determines how the compaction behaves, from a read-heavy mode to a write-heavy mode. +The scaling parameter can be changed at any time, and the compaction strategy will adjust accordingly. +For example, an operator may decide to: + +* decrease the scaling parameter, lowering the read amplification at the expense of more complex writes when a certain table is read-heavy and could benefit from reduced latencies +* increase the scaling parameter, reducing the write amplification, when it has been identified that compaction cannot keep up with the number of writes to a table + +Any such change only initiates compactions that are necessary to bring the hierarchy in a state compatible with the new configuration. +Any additional work already done (for example, when switching from negative parameter to positive), is advantageous and incorporated. + +In addition, by setting the scaling parameters to simulated a high tiered fanout factor, UCS can accomplish the same compaction as TWCS. + +image:unified/ucs.png[] + +UCS uses a combination of tiered and leveled compaction, plus sharding, to achieve the desired read and write amplification. +SSTables are sorted by token range, and then grouped into levels: + +image:unified/sharding.png[] + +UCS groups SSTables in levels based on the logarithm of the SSTables density, with the fanout factor stem:[f] as the base of the logarithm, and with each level triggering a compaction as soon as it has stem:[t] overlapping SSTables. + +The choice of the scaling parameter stem:[w], determines the value of the fanout factor stem:[f]. +In turn, stem:[w] defines the mode as either leveled or tiered, and stem:[t] is set as either stem:[t=2] for leveled compaction, or stem:[t=f] for tiered compaction. +One last parameter, the minimum SSTable size, is required to determine the complete behaviour of UCS. + +Three compaction types are possible, based on the value of stem:[w], as shown in the diagram of RA and WA above: + +* Leveled compaction, with high WA, low RA: stem:[w < 0], stem:[f = 2 - w] and stem:[t=2] +stem:[L_{f}] is the shorthand for specifying this range of stem:[w] values (e.g., L10 for stem:[w=-8]). +* Tiered compaction, with low WA, high RA: stem:[w > 0], stem:[f = 2 + w] and stem:[t=f]. +stem:[T_{f}] is the shorthand (e.g., T4 for stem:[w = 2]). +* Leveled and tiered compaction behave identically in the center: stem:[w = 0] and stem:[f = t = 2]. +In shorthand, N for stem:[w = 0]. + +[NOTE] +==== +Leveled compaction improves reads at the expense of writes and approaches a sorted array as stem:[f] increases, whereas tiered compaction favors writes at the expense of reads and approaches an unsorted log as stem:[f] increases. +==== + +UCS permits the value of stem:[w] to be defined separately for each level; thus, levels can have different behaviours. +For example, level zero can use tiered compaction (STCS-like), while higher levels can be leveled (LCS-like), defined with increasing levels of read optimization. + +== Size-based leveling + +The strategy splits SSTables at specific shard boundaries whose number grows with the density of an SSTable. +The non-overlap between SSTables created by the splitting makes concurrent compactions possible. +However, let's ignore density and splitting for a moment and explore how SSTables are grouped into levels if they are never split. + +Memtables are flushed to level zero (L0), and the memtable flush size stem:[s_{f}] is calculated as the average size of all the SSTables +written when a memtable is flushed. +This parameter, stem:[s_{f}], is intended to form the basis of the hierarchy where all newly-flushed SSTables end up. +Using a fixed fanout factor stem:[f] and stem:[s_{f}], the level stem:[L] for an SSTable of size stem:[s] is calculated as follows: + +[stem] +++++ +L = +\begin{cases} +\left \lfloor \log_f {\frac{s}{s_{f}}} \right \rfloor & \text{if } s \ge s_{f} \\ +0 & \text{otherwise} +\end{cases} +++++ + +SSTables are assigned to levels based on their size: + +|=== +| Level | Min SSTable size | Max SSTable size + +| 0 +| 0 +| stem:[s_{f} \cdot f] + +| 1 +| stem:[s_{f} \cdot f] +| stem:[s_{f} \cdot f^2] + +| 2 +| stem:[s_{f} \cdot f^2] +| stem:[s_{f} \cdot f^3] + +| 3 +| stem:[s_{f} \cdot f^3] +| stem:[s_{f} \cdot f^4] + +| ... +| ... +| ... + +| n +| stem:[s_{f} \cdot f^n] +| stem:[s_{f} \cdot f^{n+1}] +|=== + +Once SSTables start accumulating in levels, compaction is triggered when the number of SSTables in a level exceeds a threshold stem:[t] discussed earlier: + +* stem:[t = 2], leveled compaction: +.. An SSTable is promoted to level stem:[n] with size stem:[\ge s_{f} \cdot f^n]. +.. When a second SSTable is promoted to that level (also with size stem:[\ge s_{f} \cdot f^n]) they compact and form a new SSTable of size stem:[\sim 2s_{f} \cdot f^n] in the same level for stem:[f > 2]. +.. After this repeats at least stem:[f-2] more times (i.e., stem:[f] total SSTables enter the level), the compaction result grows to stem:[\ge s_{f} \cdot f^{n+1}] and enters the next level. +* stem:[t = f], tiered compaction: +** After stem:[f] SSTables enter level stem:[n], each of size stem:[\ge s{f} \cdot f^n], they are compacted and form a new SSTable of size stem:[\ge s_{f} \cdot f^{n+1}] in the next +level. + +Overwrites and deletions are ignored in these schemes, but if an expected proportion of overwrites/deletions are known, the algorithm can be adjusted. +The current UCS implementation does this adjustment, but doesn't expose the adjustment at this time. + +.Estimate of RA and WA +**** +The read and write amplification for each compaction strategy are proportional to the number of levels in the hierarchy: + +|=== +| Strategy | Write amplification | Read amplification + +| Leveled +| stem:[\varpropto] stem:[f-1] times the number of levels +| stem:[\varpropto] the number of levels + +| Tiered +| stem:[\varpropto] the number of levels +| stem:[\varpropto] stem:[f-1] times the number of levels +|=== +**** + +=== Number of levels + +Using the maximal dataset size stem:[D], the number of levels can be calculated as follows: + +[stem] +++++ +L = +\begin{cases} +\left \lfloor \log_f {\frac {D}{s_{f}}} \right \rfloor & \text{if } D \ge s_{f} \\ +0 & \text{otherwise} +\end{cases} +++++ + +This calculation is based on the assumption that the maximal dataset size stem:[D] is reached when all levels are full, and the maximal number of levels is inversely proportional to the logarithm of stem:[f]. + +Thus, when we try to control the overheads of compaction on the database, we have a space of choices for the strategy that range from: + +* leveled compaction ( stem:[t=2] ) with high stem:[f]: +** low number of levels +** high read efficiency +** high write cost +** moving closer to the behaviour of a sorted array as stem:[f] increases +* compaction with stem:[t = f = 2] where leveled is the same as tiered and we have a middle ground with logarithmically +increasing read and write costs; +* tiered compaction ( stem:[t=f] ) with high stem:[f]: +** very high number of SSTables +** low read efficiency +** low write cost +** moving closer to an unsorted log as stem:[f] increases + +This can be easily generalized to varying fan factors, by replacing the exponentiation with the product of the fan +factors for all lower levels: + +|=== +| Level | Min SSTable size | Max SSTable size + +| 0 +| 0 +| stem:[s_{f} \cdot f_0] + +| 1 +| stem:[s_{f} \cdot f_0] +| stem:[s_{f} \cdot f_0 \cdot f_1] + +| 2 +| stem:[s_{f} \cdot f_0 \cdot f_1] +| stem:[s_{f} \cdot f_0 \cdot f_1 \cdot f_2] + +| ... +| ... +| ... + +| n +| stem:[s_{f} \cdot \prod_{i < n} f_i] +| stem:[s_{f} \cdot \prod_{i\le n} f_i] +|=== + +== Density-based leveling + +If we replace the size stem:[s] in the previous discussion with the density measure + +[stem] +++++ +d = \frac s v +++++ + +where stem:[v] is the fraction of the token space that the SSTable covers, all formulae and conclusions remain valid. +However, using density, the output can now be split at arbitrary points. +If several SSTables are compacted and split, the new resulting SSTables formed will be denser than the original SSTables. +For example, using a scaling parameter of T4, four input SSTables spanning 1/10 of the token space each, when compacted and split, will form four new SSTables spanning 1/40 of the token space each. + +These new SSTables will be the same size but denser, and consequently will be moved to the next higher level, due to the higher density value exceeding the maximum density for the original compacted level. +If we can ensure that the split points are fixed (see below), this process will repeat for each shard (token range), executing independent compactions concurrently. + +[IMPORTANT] +==== +It is important to account for locally-owned token share when calculating stem:[v]. +Because vnodes mean that the local token ownership of a node is not contiguous, the difference between the first and last token is not sufficient to calculate token share; thus, any non-locally-owned ranges must be excluded. +==== + +Using the density measure allows us to control the size of SSTables through sharding, as well as to execute compactions in parallel. +With size-leveled compaction, we could achieve parallelization by pre-splitting the data in a fixed number of compaction shards, based on the data directories. +However, that method requires the number of shards to be predetermined and equal for all levels of the hierarchy, and SSTables can become too small or too large. +Large SSTables complicate streaming and repair and increase the duration of compaction operations, pinning resources to long-running operations and making it more likely that too many SSTables will accumulate on lower levels of the hierarchy. + +Density-leveled compaction permits a much wider variety of splitting options. +For instance, the size of SSTables can be kept close to a selected target, allowing UCS to deal with the leveling of both STCS (SSTable size grows with each level) and LCS (token share shrinks with each level). + +== Sharding + +=== Basic sharding scheme + +This sharding mechanism is independent of the compaction specification. +There are a range of choices for splitting SSTables: + +* Split when a certain output size is reached (like LCS), forming non-overlapping SSTable runs instead of individual SSTables +* Split the token space into shards at predefined boundary points +* Split at predefined boundaries, but only if a certain minimum size has been reached + +Splitting only by size results in individual SSTables with start positions that vary. +To compact SSTables split in this way, you must choose to either compact the entire token range of a level sequentially or compact and copy some of the data more times than necessary, due to overlapping SSTables. +If predefined boundary points are used, some of the token range can be sparser with fewer inputs and skew the density of the resulting SSTables. +If that occurs, further splitting may be required. +In the hybrid option, the density skew can occur less frequently, but can still occur. + +To avoid these problems and permit concurrent compactions of all levels of the compaction hierarchy, UCS predefines boundary points for every compaction and always splits SSTables at these points. +The number of boundaries is determined from the density of the input SSTables and the estimated density of the resulting SSTables. +As the density grows larger, the number of boundaries is increased, keeping the size of individual SSTables close to a predefined target. +Using power-of-two multiples of a specified base count, i.e., splitting shards in the middle, ensures that any boundary that applies to a given output density also applies to all higher densities. + +Two sharding parameters can be configured: + +* base shard count stem:[b] +* target SSTable size stem:[s_{t}] + +At the start of every compaction, recall that the density of the output stem:[d] is estimated, based on the input size stem:[s] of the SSTables and the token range stem:[v]: + +[stem] +++++ +d = \frac s v +++++ + +where stem:[v] is the fraction of the token range covered by the input SSTables, in a value of 0 to 1. +stem:[v = 1] means that the entire token range is covered by the input SSTables, and stem:[v = 0] means that the input SSTables cover no token range. + +When the initial flush of the memtable to L0 occurs, stem:[v = 1] since the entire token range is included in the memtable. +In subsequent compactions, the token range stem:[v] is the fraction of the token range covered by the SSTables being compacted. + +With the calculated density of the output, plus the values of stem:[b] and stem:[s_{t}], the number of shards stem:[S] into which to split the token space can be calculated: + +[stem] +++++ +S = +\begin{cases} +b + & \text{if } {\frac d s_{t} \cdot \frac 1 b} < 1 \\ +2^{\left\lfloor \log_2 \left( {\frac d s_{t} \cdot \frac 1 b}\right)\right\rceil} \cdot b + & \text{if } {\frac d s_{t} \cdot \frac 1 b} \ge 1 \\ +\end{cases} +++++ + +where stem:[\lfloor x \rceil] stands for stem:[x] rounded to the nearest integer, i.e. stem:[\lfloor x + 0.5 \rfloor]. +Thus, in the second case,the density is divided by the target size and rounded to a power-of-two multiple of stem:[b]. +If the result is less than 1, the number of shards will be the base shard count, because the memtable is split into stem:[{2 \cdot b}], or stem:[b] L0 shards. + +However, the token range is not the only factor that influences if we switch between stem:[b] shards or more (where the condition is greater than or equal to 1). +If the memtable is very large and able to flush several gigabytes at once, stem:[d] may be a magnitude larger than stem:[s_{t}], and cause SSTables to be split into multiple shards even on L0. +Conversely, if the memtable is small, stem:[d] may still be smaller than stem:[s_{t}] on levels above L0, where the condition is less than 1, and thus, there will be stem[b] shards. + +stem:[S - 1] boundaries are generated, splitting the local token space equally into stem:[S] shards. +Splitting the local token space will split the result of the compaction on these boundaries to form a separate SSTable for each shard. +SSTables produced will have sizes that fall between stem:[s_{t}/\sqrt 2] and stem:[s_{t} \cdot \sqrt 2]. + +For example, let's use a target SSTable size of stem:[s_{t} = 100MiB] and stem:[b = 4] base shards. +If a stem:[s_{f} = 200 MiB] input memtable is flushed, the condition for calculating the number of shards is: + +[stem] +++++ +\frac{200}{100} \cdot \frac{1}{4} = 0.5 < 1 +++++ + +This calculation results in stem:[0.5 < 1], because the value of stem:[v = 1] on the initial flush. +Because the result is less than 1, the base shard count is used, and the memtable is split into four L0 shards of approximately 50MiB each. +Each shard spans 1/4 of the token space. + +To continue the example, on the next level of compaction, for just one of the four shards, let's compact six of these 50 MiB SSTables. +The estimated density of the output will be: + +[stem] +++++ +\left( \frac{6 \cdot 50\, \mathrm{MiB}}{\frac{1}{4}} \right) = 1200 \mathrm{MiB} +++++ + +using 1/4 as the stem:[v] value for the token range covered by the input SSTables. + +The condition for splitting will be: + +[stem] +++++ +(\frac{1200}{100} \cdot \frac{1}{4}) = 3 > 1 +++++ + +Thus, the number of shards will be calculated as: + +[stem] +++++ +2^{\left\lfloor \log_2 \left( {\frac{1200}{100} \cdot \frac{1}{4}}\right)\right\rceil} \cdot b +++++ + +or stem:[2^{\log_2 3}], rounded to stem:[2^2 \cdot 4] shards for the whole local token space, and that compaction covering 1/4 of the token space. +Assuming no overwrites or deletions, the resulting SSTables will be of size 75 MiB, token share 1/16 and density 1200 MiB. + +=== Full sharding scheme + +This sharding scheme can be easily extended. +There are two extensions currently implemented, SSTable growth and a minimum SSTable size. + +First, let's examine the case when the size of the data set is expected to grow very large. +To avoid pre-specifying a sufficiently large target size to avoid problems with per-SSTable overhead, an `SSTtable growth` parameter has been implemented. +// LLP: I don't know what this means: determines what part of the density growth should be assigned to increased SSTable size +This parameter determines what part of the density growth should be assigned to increased SSTable size, reducing the growth of the number of shards, and hence, non-overlapping SSTables. + +The second extension is a mode of operation with a fixed number of shards that splits conditionally on reaching a minimum size. +Defining a `minimum SSTable size`, the base shard count can be reduced whenever a split would result in SSTables smaller than the provided minimum. + +There are four user-defined sharding parameters: + +* base shard count stem:[b] +* target SSTable size stem:[s_{t}] +* minimum SSTable size stem:[s_{m}] +* SSTable growth component stem:[\lambda] + +The number of shards stem:[S] for a given density stem:[d] is then calculated as + +[stem] +++++ +S = +\begin{cases} +1 + & \text{if } d < s_{m} \\ +\min(2^{\left\lfloor \log_2 \frac {d}{s_{m}} \right\rfloor}, x) + & \text{if } d < s_{m} \cdot b \text{, where } x \text{ is the largest power of 2 divisor of } b \\ +b + & \text{if } d < s_{t} \cdot b \\ +2^{\left\lfloor (1-\lambda) \cdot \log_2 \left( {\frac {d}{s_{t}} \cdot \frac 1 b}\right)\right\rceil} \cdot b + & \text{otherwise} +\end{cases} +++++ + +Some useful combinations of these parameters: + +* The basic scheme above uses a SSTable growth stem:[\lambda=0], and a minimum SSTable size stem:[s_{m}=0]. +The graph below illustrates the behaviour for base shard count stem:[b=4] and target SSTable size stem:[s_{t} = 1\, \mathrm{GB}]: + +image:unified/shards_graph_lambda_0.svg[Graph with lambda 0] + +* Using stem:[\lambda = 0.5] grows the shard count and SSTable size evenly. +When the density quadruples, both the shard count and the expected SSTable size for that density band will double. +The example below uses stem:[b=8], stem:[s_{t} = 1\, \mathrm{GB}] and also applies a minimal size stem:[m = 100\, \mathrm{MB}]: + +image:unified/shards_graph_lambda_0_5.svg[Graph with lambda 0.5] + +* Similarly, stem:[\lambda = 1/3] makes the the SSTable growth the cubic root of the density growth, i.e. the SSTable size +grows with the square root of the growth of the shard count. +The graph below uses stem:[b=1] and stem:[s_{t} = 1\, \mathrm{GB}] (note: when stem:[b=1], the minimal size has no effect): + +image:unified/shards_graph_lambda_0_33.svg[Graph with lambda 0.33] + +* A growth component of 1 constructs a hierarchy with exactly stem:[b] shards at every level. +Combined with a minimum SSTable size, the mode of operation uses a pre-specified number of shards, but splits only after reaching a minimum size. +Illustrated below for stem:[b=10] and stem:[s_{m} = 100\, \mathrm{MB}] (note: the target SSTable size is irrelevant when stem:[\lambda=1]): + +image:unified/shards_graph_lambda_1.svg[Graph with lambda 1] + +== Choosing SSTables to compact + +The density leveling separates SSTables in levels defined by the compaction configuration's fan factors. +However, unlike in size leveling, where SSTables are expected to cover the full token space, the number of SSTables on a level cannot be used as a trigger due to possible non-overlapping SSTables. +Read queries are less efficient in this situation. +To deal with this, execute sharding that performs multiple compactions on a level concurrently, and reduces the size of individual compaction operations. +The non-overlapping section must be separated into different buckets, and the number of overlapping SSTables in a bucket determines what to do. +A bucket is the selected set of SSTables that will be compacted together. + +First form a minimal list of overlap sets that satisfy the following requirements: + +* two SSTables that do not overlap are never put in the same set +* if two SSTables overlap, there is a set in the list that contains both +* SSTables are placed in consecutive positions in the list + +The second condition can also be rephrased to say that for any point in the token range, there is a set in the list that contains all SSTables whose range covers that point. +In other words, the overlap sets give us the maximum number of SSTables that need to be consulted to read any key, i.e., the read amplification that our trigger stem:[t] aims to control. +We don't calculate or store the exact spans the overlapping sets cover, only the participating SSTables. +The sets can be obtained in stem:[O(n\log n)] time. + +For example, if SSTables A, B, C and D cover, respectively, tokens 0-3, 2-7, 6-9 and 1-8, we compute a list of overlap sets that are ABD and BCD. +A and C don't overlap, so they must be in separate sets. +A, B and D overlap at token 2 and must thus be present in at least one set, and similarly for B, C and D at 7. +Only A and D overlap at 1, but the set ABD already includes this combination. + +These overlap sets are sufficient to decide whether or not a compaction should be carried out, if and only if the number of elements in a set is at least as large as stem:[s_{t}]. +However, we may need to include more SSTables in the compaction than this set alone. + +It is possible for our sharding scheme to end up constructing SSTables spanning differently-sized shards for the same level. +One clear example is the case of leveled compaction. +In this case, SSTables enter at some density, and after the first compaction the resulting SSTable is 2x bigger than the initial density, causing the SSTable to split in half at the middle of the token range. +When another SSTable enters the same level, we will have separate overlap sets between the two older SSTables and the new one. +For efficiency, the compaction that is triggered next needs to select both of the overlap sets. + +To deal with cases of partial overlap, the overlap sets will transitively extend with all neighboring ones that share some SSTable. +Thus, the set of all SSTables that is constructed has some chain of overlapping SSTables that connects it to the initial set. +This extended set forms the compaction bucket. + +[NOTE] +==== +In addition to `TRANSITIVE`, "overlap inclusion methods" of `NONE` and `SINGLE` are also implemented for experimentation, but they are not recommended for the UCS sharding scheme. +==== + +In normal operation, we compact all SSTables in the compaction bucket. +If compaction is very late, we may apply a limit on the number of overlapping sources we compact. +// LLP: What does limit-many mean in the next sentence? +In that case, we use the collection of oldest SSTables that would select at most limit-many in any included overlap set, making sure that if an SSTable is included in this compaction, all older ones are also included to maintain time order. + +=== Selecting the compaction to run + +Compaction strategies aim to minimize the read amplification of queries, which is defined by the number of SSTables that overlap on any given key. +For highest efficiency in situations where compaction is late, a compaction bucket is selected with the highest overlap among the possible choices. +If there are multiple choices, choose one uniformly and randomly within each level. +Between the levels, prefer the lowest level, as this is expected to cover a larger fraction of the token space for the same amount of work. + +Under sustained load, this mechanism prevents the accumulation of SSTables on some level that could sometimes happen with legacy strategies. +With older strategies, all resources could be consumed by L0 and SSTables accumulating on L1. +With UCS, a steady state where compactions always use more SSTables than the assigned threshold and fan factor is accomplished, and a tiered hierarchy is maintained based on the lowest overlap they are able to maintain for the load. + +=== Major compaction + +Under the working principles of UCS, a major compaction is an operation which compacts together all SSTables that have (transitive) overlap, and where the output is split on shard boundaries appropriate for the expected resulting density. + +In other words, a major compaction will result in stem:[b] concurrent compactions, each containing all SSTables covered in each of the base shards. +The result will be split on shard boundaries whose number depends on the total size of data contained in the shard. + +== Differences with STCS and LCS + +Note that there are some differences between tiered UCS and legacy STCS, and between leveled UCS and legacy LCS. + +=== Tiered UCS vs STCS + +STCS is very similar to UCS. +However, STCS defines buckets/levels by looking for similarly sized SSTables, rather than using a predefined banding of sizes. +Thus, STCS can end up with some odd selections of buckets, spanning SSTables of wildly different sizes; UCS's selection is more stable and predictable. + +STCS triggers a compaction when it finds at least `min_threshold` SSTables on some bucket, and it compacts between `min_threshold` and `max_threshold` SSTables from that bucket at a time. +`min_threshold` is equivalent to UCS's stem:[t = f = w + 2]. +UCS drops the upper limit because its compaction is still efficient with very large numbers of SSTables. + +UCS uses a density measure to split results, in order to keep the size of SSTables and the time for compactions low. +Within a level, UCS will only consider overlapping SSTables when deciding whether the threshold is hit, and will independently compact sets of SSTables that do not overlap. + +If there are multiple choices to pick SSTables within a bucket, STCS groups them by size, while UCS groups them by timestamp. +Because of that, STCS easily loses time order which makes whole table expiration less efficient. +UCS efficiently tracks time order and whole table expiration. +Because UCS can apply whole-table expiration, this features also proves useful for time-series data with time-to-live constraints. + +=== UCS-leveled vs LCS + +LCS seems very different in behaviour compared to UCS. +However, the two strategies are, in fact, very similar. + +LCS uses multiple SSTables per level to form a sorted run of non-overlapping SSTables of small fixed size. +So physical SSTables on increasing levels increase in number (by a factor of `fanout_size`) instead of size. +In this way, LCS reduces space amplification and ensures shorter compaction times. +When the combined size of a run on a level is higher than expected, it selects some SSTables to compact with overlapping ones from the next level of the hierarchy. +Eventually, the size of the next level gets pushed over its size limit and triggers higher-level operations. + +In UCS, SSTables on increasing levels increase in density by a fanout factor stem:[f]. +A compaction is triggered when a second overlapping SSTable is located on a sharded level. +UCS compacts the overlapping bucket on that level, and the result most often ends up on that level, too. +But eventually, the data reaches sufficient size for the next level. +Given an even data spread, UCS and LCS behave similarly, with compactions triggered in the same timeframe. + +The two approaches end up with a very similar effect. +UCS has the added benefit that compactions cannot affect other levels. +In LCS, L0-to-L1 compactions can prevent any concurrent L1-to-L2 compactions, an unfortunate situation. +In UCS, SSTables are structured such that they can be easily switched to tiered UCS or changed with different parameter settings. + +Because LCS SSTables are based on size only and thus vary on split position, +when LCS selects SSTables to compact on the next level, some SSTables that only partially overlap are included. +Consequently, SSTables can be compacted more often than strictly necessary. + +UCS handles the problem of space amplification by sharding on specific token boundaries. +LCS splits SSTables based on a fixed size with boundaries usually falling inside SSTables on the next level, kicking off compaction more frequently than necessary. Therefore UCS aids with tight write amplification control. +Those boundaries guarantee that we can efficiently select higher-density SSTables that exactly match the span of the lower-density ones. + +// include::cassandra:partial$default-compaction-strategy.adoc[] + +[[ucs_options]] +== UCS Options + +[cols="1,2"] +|=== +| Subproperty | Description + +| enabled +| Enables background compaction. + +Default value: true +// See Enabling and disabling background compaction. + +| only_purge_repaired_tombstone +| Enabling this property prevents data from resurrecting when repair is not run within the `gc_grace_seconds`. + +Default value: false + +| scaling_parameters +| A list of per-level scaling parameters, specified as stem:[L_{f}], stem:[T_{f}], stem:[N], or an integer value specifying stem:[w] directly. +If more levels are present than the length of this list, the last value is used for all higher levels. +Often this will be a single parameter, specifying the behaviour for all levels of the hierarchy. + +Leveled compaction, specified as stem:[L_{f}], is preferable for read-heavy workloads, especially if bloom filters are not effective (e.g. with wide partitions); higher levelled fan factors improve read amplification (and hence latency, as well as throughput for read-dominated workloads) at the expense of increased write costs. +The equivalent of legacy LCS is L10. + +Tiered compaction, specified as stem:[T_{f}], is preferable for write-heavy workloads, or ones where bloom filters or time order can be exploited; higher tiered fan factors improve the cost of writes (and hence throughput) at the expense of making reads more difficult. + +stem:[N] is the middle ground that has the features of leveled (one SSTable run per level), as well as tiered (one compaction to be promoted to the next level) and a fan factor of 2. +This value can also be specified as T2 or L2. + +Default value: T4 (STCS with threshold 4) + +| target_sstable_size +| The target sstable size stem:[s_{t}], specified as a human-friendly size in bytes, such as https://www.techtarget.com/searchstorage/definition/mebibyte-MiB[MiB]. +The strategy will split data in shards that aim to produce sstables of size between stem:[s_{t}/\sqrt{2}] and stem:[s_{t} \cdot \sqrt{2}]. +Smaller sstables improve streaming and repair, and make compactions shorter. +On the other hand, each sstable on disk has a non-trivial in-memory footprint that also affects garbage collection times. +Increase this if the memory pressure from the number of sstables in the system becomes too high. + +Default: 1 GiB + +| min_sstable_size +| The minimum sstable size, applicable when the base shard count will result is SSTables that are considered too small. +If set, the strategy will split the space into fewer than the base count shards, to make the estimated SSTables size at least as large as this value. +A value of 0 disables this feature. + +Default: 100MiB + +| base_shard_count +| The minimum number of shards stem:[b], used for levels with the smallest density. +This gives the minimum compaction concurrency for the lowest levels. +A low number would result in larger L0 sstables but may limit the overall maximum write throughput (as every piece of data has to go through L0). + +Default: is 4 (1 for system tables, or when multiple data locations are defined) + +| sstable_growth +| The sstable growth component stem:[\lambda], applied as a factor in the shard exponent calculation. +This is a number between 0 and 1 that controls what part of the density growth should apply to individual sstable size and what part should increase the number of shards. +Using a value of 1 has the effect of fixing the shard count to the base value. +Using 0.5 makes the shard count and SSTable size grow with the square root of the density growth. +This is useful to decrease the sheer number of SSTables created for very large data sets. +For example, without growth correction, a data set of 10TiB with 1GiB target size would result in over 10K SSTables. +That many SSTables can result in too much overhead, both for on-heap memory used by per-SSTables structures, as well as lookup-time for intersecting SSTables and tracking overlapping sets during compaction. +For example, with a base shard count of 4, the growth factor can reduce the potential number of SSTables to ~160 of size ~64GiB, manageable in terms of memory overhead, individual compaction duration, and space overhead. +The parameter can be adjusted, increasing the value to get fewer but bigger SSTables on the top level, and decreasing the value to favour a higher count of smaller SSTables. + +Default: 0.333 (SSTable size grows with the square root of the growth of the shard count) + +| expired_sstable_check_frequency_seconds +| Determines how often to check for expired SSTables. + +Default: 10 minutes + +| max_sstables_to_compact +| The maximum number of sstables to compact in one operation. +Larger value may reduce write amplification but can cause very long compactions, and thus a very high read amplification overhead while such compactions are processing. +The default aims to keep the length of operations under control and prevent accumulation of SSTables while compactions are taking place. +If the fanout factor is larger than the maximum number of SSTables, the strategy will ignore the latter. + +Default: none (although 32 is a good choice) + +| overlap_inclusion_method +| Specifies how to extend overlapping sections into buckets. +TRANSITIVE makes sure that if we choose an SSTable to compact, we also compact the ones that overlap with it. +SINGLE only does this extension once (i.e. it selects only SSTables that overlap with the original overlapping SSTables section. +NONE does not add any overlapping sstables. +NONE is not recommended, SINGLE may offer a little more parallelism at the expense of recompacting some data when upgrading from LCS or during range movements. + +Default: TRANSITIVE + +| unsafe_aggressive_sstable_expiration +| Expired SSTables are dropped without checking if their data is shadowing other SSTables. +This flag can only be enabled if `cassandra.allow_unsafe_aggressive_sstable_expiration` is true. +Turning this flag on can cause correctness issues, such as the reappearance of deleted data. +See discussions in https://issues.apache.org/jira/browse/CASSANDRA-13418[CASSANDRA-13418] for valid use cases and potential problems. + +Default: false +|=== + +In `cassandra.yaml`, there is also one parameter that affects compaction: + +concurrent_compactors:: +Number of simultaneous compactions to allow, NOT including validation "compactions" for anti-entropy repair. +Higher values increase compaction performance but may increase read and write latencies. \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/operating/index.adoc b/doc/modules/cassandra/pages/managing/operating/index.adoc index b7491cf652..39dd508c45 100644 --- a/doc/modules/cassandra/pages/managing/operating/index.adoc +++ b/doc/modules/cassandra/pages/managing/operating/index.adoc @@ -1,17 +1,20 @@ = Operating Cassandra :navtitle: Operating +* xref:cassandra:managing/operating/backups.adoc[Backups] +* xref:cassandra:managing/operating/bloom_filters.adoc[Bloom filters] +* xref:cassandra:managing/operating/bulk_loading.adoc[Bulk loading] +* xref:cassandra:managing/operating/cdc.adoc[Change Data Capture (CDC)] +* xref:cassandra:managing/operating/compaction/index.adoc[Compaction] +* xref:cassandra:managing/operating/compression.adoc[Compression] +* xref:cassandra:managing/operating/denylisting_partitions.adoc[Denylisting partitions] * xref:cassandra:managing/operating/hardware.adoc[Hardware] -* xref:cassandra:managing/operating/security.adoc[Security] -* xref:cassandra:managing/operating/topo_changes.adoc[Topology changes] * xref:cassandra:managing/operating/hints.adoc[Hints] +* xref:cassandra:managing/operating/logging.adoc[Logging] +* xref:cassandra:managing/operating/metrics.adoc[Monitoring metrics] * xref:cassandra:managing/operating/repair.adoc[Repair] * xref:cassandra:managing/operating/read_repair.adoc[Read repair] -* xref:cassandra:managing/operating/backups.adoc[Backups] -* xref:cassandra:managing/operating/compression.adoc[Compression] -* xref:cassandra:managing/operating/compaction/index.adoc[Compaction] -* xref:cassandra:managing/operating/metrics.adoc[Monitoring] -* xref:cassandra:managing/operating/bulk_loading.adoc[Bulk loading] -* xref:cassandra:managing/operating/cdc.adoc[CDC] -* xref:cassandra:managing/operating/bloom_filters.adoc[Bloom filters] -* xref:cassandra:managing/operating/denylisting_partitions.adoc[Denylisting partitions] +* xref:cassandra:managing/operating/security.adoc[Security] +* xref:cassandra:managing/operating/topo_changes.adoc[Topology changes] +* xref:cassandra:managing/operating/transientreplication.adoc[Transient replication] +* xref:cassandra:managing/operating/virtualtables.adoc[Virtual tables] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/operating/logging.adoc b/doc/modules/cassandra/pages/managing/operating/logging.adoc new file mode 100644 index 0000000000..d740aef82d --- /dev/null +++ b/doc/modules/cassandra/pages/managing/operating/logging.adoc @@ -0,0 +1,5 @@ += Logging + +* xref:cassandra:managing/operating/auditlogging.adoc[Audit logging] +* xref:cassandra:managing/operating/audit_logging.adoc[Audit logging 2] +* xref:cassandra:managing/operating/fqllogging.adoc[Full query logging] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/new/index.adoc b/doc/modules/cassandra/pages/new/index.adoc index 7cf4c41837..0980244fc6 100644 --- a/doc/modules/cassandra/pages/new/index.adoc +++ b/doc/modules/cassandra/pages/new/index.adoc @@ -5,21 +5,22 @@ This section covers the new features in Apache Cassandra 5.0. -* https://issues.apache.org/jira/browse/CASSANDRA-16052[Storage Attached Indexes] -* https://issues.apache.org/jira/browse/CASSANDRA-17240[Trie Memtables] -* https://issues.apache.org/jira/browse/CASSANDRA-18398[Trie SSTables] -* https://github.com/apache/cassandra/blob/trunk/NEWS.txt[More Guardrails] -* https://issues.apache.org/jira/browse/CASSANDRA-8877[TTL and writetime on collections and UDTs] -* https://issues.apache.org/jira/browse/CASSANDRA-16895[JDK 17] -* https://issues.apache.org/jira/browse/CASSANDRA-17221[New Mathematical CQL functions: abs, exp, log, log10 and round] -* https://issues.apache.org/jira/browse/CASSANDRA-18060[New CQL native scalar functions for collections] -* https://issues.apache.org/jira/browse/CASSANDRA-14227[Extend maximum expiration date (TTL)] -* https://issues.apache.org/jira/browse/CASSANDRA-17940[Dynamic Data Masking] -* https://issues.apache.org/jira/browse/CASSANDRA-18504[New vector data type] -* https://issues.apache.org/jira/browse/CASSANDRA-18640[New vector similarity functions] -* https://issues.apache.org/jira/browse/CASSANDRA-18397[Unified Compaction Strategy] -* https://issues.apache.org/jira/browse/CASSANDRA-8720[New sstablepartitions offline tool to find large partitions] -* https://issues.apache.org/jira/browse/CASSANDRA-18646[New snitch for Microsoft Azure] -* https://issues.apache.org/jira/browse/CASSANDRA-18592[New CIDR authorizer] -* https://issues.apache.org/jira/browse/CASSANDRA-18624[Pluggable crypto provider] -* https://issues.apache.org/jira/browse/CASSANDRA-17948[New virtual table to view system logs] +* Storage Attached Indexes: xref:cassandra:developing/cql/indexing/sai/sai-overview.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-7%3A+Storage+Attached+Index[CEP-7], https://issues.apache.org/jira/browse/CASSANDRA-16052[JIRA ticket] +* Trie memtables: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-19%3A+Trie+memtable+implementation[CEP-19], https://issues.apache.org/jira/browse/CASSANDRA-17240[JIRA ticket] +* Trie SSTables: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-25%3A+Trie-indexed+SSTable+format[CEP-25], https://issues.apache.org/jira/browse/CASSANDRA-18398[JIRA ticket] +* JDK 17: xref:cassandra:reference/java17.adoc[Docs], https://issues.apache.org/jira/browse/CASSANDRA-16895[JIRA ticket] +* More guardrails: https://github.com/apache/cassandra/blob/trunk/NEWS.txt[NEWS.txt] +* TTL and writetime on collections and UDTs: xref:cassandra:developing/cql/dml.html#writetime-and-ttl-function[Docs], https://issues.apache.org/jira/browse/CASSANDRA-8877[JIRA ticket] +* New vector data type: xref:cassandra:reference/vector-data-type.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-30%3A+Approximate+Nearest+Neighbor%28ANN%29+Vector+Search+via+Storage-Attached+Indexes[CEP-30], https://issues.apache.org/jira/browse/CASSANDRA-18504[JIRA ticket] +* New vector similarity functions: xref:cassandra:vector-search/overview.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-30%3A+Approximate+Nearest+Neighbor%28ANN%29+Vector+Search+via+Storage-Attached+Indexes[CEP-30],https://issues.apache.org/jira/browse/CASSANDRA-18640[JIRA ticket] +* Unified Compaction Strategy: xref:cassandra:managing/operating/compaction/ucs.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-26%3A+Unified+Compaction+Strategy[CEP-26], (https://issues.apache.org/jira/browse/CASSANDRA-18397)[CASSANDRA-18397] +* New Mathematical CQL functions: abs, exp, log, log10 and round: xref:cassandra:developing/cql/functions.adoc[Docs], https://issues.apache.org/jira/browse/CASSANDRA-17221[JIRA ticket] +* Dynamic Data Masking: xref:cassandra:developing/cql/dynamic_data_masking.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-20%3A+Dynamic+Data+Masking[CEP-20], https://issues.apache.org/jira/browse/CASSANDRA-17940[CASSANDRA-17940] +* New CIDR authorizer: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-33%3A+CIDR+filtering+authorizer[CEP-33], https://issues.apache.org/jira/browse/CASSANDRA-18592[JIRA ticket] +* Extend maximum expiration date (TTL): https://github.com/apache/cassandra/blob/trunk/NEWS.txt[NEWS.txt], https://issues.apache.org/jira/browse/CASSANDRA-14227[JIRA ticket] +* New CQL native scalar functions for collections: https://issues.apache.org/jira/browse/CASSANDRA-18060[JIRA ticket] +* New sstablepartitions offline tool to find large partitions: https://issues.apache.org/jira/browse/CASSANDRA-8720[JIRA ticket] +* New snitch for Microsoft Azure: https://issues.apache.org/jira/browse/CASSANDRA-18646[JIRA ticket] +* Pluggable crypto provider: https://issues.apache.org/jira/browse/CASSANDRA-18624[JIRA ticket] +* New virtual table to view system logs: https://issues.apache.org/jira/browse/CASSANDRA-17948[JIRA ticket] +// LLP: waiting to see if they make 5.0 \ No newline at end of file diff --git a/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc b/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc index 500da166f0..3df664bbc1 100644 --- a/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc +++ b/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc @@ -12,7 +12,7 @@ compaction = { '' : [, ...] } ---- -where the is xref:STCS[SizeTieredCompactionStrategy], xref:TWCS[TimeWindowCompactionStrategy], or xref:LCS[LeveledCompactionStrategy]. +where the is xref:UCS[UnifiedCompactionStrategy], xref:STCS[SizeTieredCompactionStrategy], xref:TWCS[TimeWindowCompactionStrategy], or xref:LCS[LeveledCompactionStrategy]. [IMPORTANT] ==== @@ -33,32 +33,29 @@ compaction = { 'only_purge_repaired_tombstone' : (true | false), 'tombstone_threshold' : , 'tombstone_compaction_interval' : , - 'unchecked_tombstone_compaction' : (true | false), - 'min_threshold' : , - 'max_threshold' : } + 'unchecked_tombstone_compaction' : (true | false) } ---- -*enabled* :: +enabled:: Enable background compaction. -* `true` runs minor compactions. -* `false` disables minor compactions. +* `true` runs minor compactions +* `false` disables minor compactions + [TIP] ==== Use `nodetool enableautocompaction` to start running compactions. ==== - -{empty}:: ++ Default: `true` -*log_all* :: +log_all:: Activates advanced logging for the entire cluster. + Default: `false` -*only_purge_repaired_tombstone* :: +only_purge_repaired_tombstone:: Enabling this property prevents data from resurrecting when repair is not run within the `gc_grace_seconds`. When its been a long time between repairs, the database keeps all tombstones. + @@ -69,39 +66,73 @@ When its been a long time between repairs, the database keeps all tombstones. + Default: `false` -*tombstone_threshold* :: +tombstone_threshold:: The ratio of garbage-collectable tombstones to all contained columns. If the ratio exceeds this limit, compactions starts only on that table to purge the tombstones. + Default: `0.2` -*tombstone_compaction_interval* :: +tombstone_compaction_interval:: Number of seconds before compaction can run on an SSTable after it is created. An SSTable is eligible for compaction when it exceeds the `tombstone_threshold`. Because it might not be possible to drop tombstones when doing a single SSTable compaction, and since the compaction is triggered base on an estimated tombstone ratio, this setting makes the minimum interval between two single SSTable compactions tunable to prevent an SSTable from being constantly re-compacted. + Default: `86400` (1 day) -*unchecked_tombstone_compaction* :: +unchecked_tombstone_compaction:: Setting to `true` allows tombstone compaction to run without pre-checking which tables are eligible for the operation. Even without this pre-check, {product} checks an SSTable to make sure it is safe to drop tombstones. + Default: `false` -*min_threshold* :: -The minimum number of SSTables to trigger a minor compaction. -+ -*Restriction:* Not used in `LeveledCompactionStrategy`. -+ -Default: `4` +[[UCS]] +==== UnifiedCompactionStrategy -*max_threshold* :: +Unified Compaction Strategy is the recommended compaction strategy for most deployments. +The following properties only apply to UnifiedCompactionStrategy: + +[source,language-cql] +---- +compaction = { + 'class' : 'UnifiedCompactionStrategy, + 'scaling_parameters' : list of scaling parameters, + 'target_sstable_size': , + 'base_shard_count': } +---- + +scaling_parameters:: The maximum number of SSTables before a minor compaction is triggered. + -*Restriction:* Not used in `LeveledCompactionStrategy`. -+ Default: `32` +target_sstable_size:: +The target size for SSTables that use the UnifiedCompactionStrategy. +You must specify the bytes. ++ +Default: `1 GiB` + +base_shard_count:: +The minimum number of shards, used for levels with the smallest density. ++ +Default: `4` (1 for system tables, or when multiple data locations are defined) + +expired_sstable_check_frequency_seconds:: +Determines how often to check for expired SSTables. ++ +Default: `10 minutes` + +sstable_growth:: +// LLP: need to put in greek symbol for lambda +The sstable growth component lambda, applied as a factor in the shard exponent calculation. +This is a number between 0 and 1 that controls what part of the density growth should apply to individual sstable size and what part should increase the number of shards. +Using a value of 1 has the effect of fixing the shard count to the base value. +Using 0.5 makes the shard count and sstable size grow with the square root of the density growth. +This is useful to decrease the sheer number of sstables that will be created for very large data sets. +For example, without growth correction a data set of 10TiB with 1GiB target size would result in over 10k sstables, which may present as too much overhead both as on-heap memory used by per-sstable structures as well as time to look for intersecting sstables and tracking overlapping sets during compaction. +Applying in this scenario (with base count 4) will reduce the potential number of sstables to ~160 of ~64GiB, which is still manageable both as memory overhead and individual compaction duration and space overhead. +The balance between the two can be further tweaked by increasing to get fewer but bigger sstables on the top level, and decreasing it to favour a higher count of smaller sstables. +The default value is 0.333 meaning the sstable size grows with the square root of the growth of the shard count. + [[STCS]] ==== SizeTieredCompactionStrategy @@ -111,7 +142,7 @@ See xref:cassandra:managing/operating/compaction/stcs.adoc[SizeTieredCompactionS [NOTE] ==== -Default compaction strategy. +STCS is the default compaction strategy. ==== The following properties only apply to SizeTieredCompactionStrategy: @@ -122,24 +153,40 @@ compaction = { 'class' : 'SizeTieredCompactionStrategy', 'bucket_high' : , 'bucket_low' : , - 'min_sstable_size' : } + 'min_sstable_size' : , + 'min_threshold' : , + 'max_threshold' : } ---- -*bucket_high* :: +min_threshold:: +The minimum number of SSTables to trigger a minor compaction. ++ +*Restriction:* Not used in `LeveledCompactionStrategy`. ++ +Default: `4` + +max_threshold:: +The maximum number of SSTables before a minor compaction is triggered. ++ +*Restriction:* Not used in `LeveledCompactionStrategy`. ++ +Default: `32` + +bucket_high:: Size-tiered compaction merges sets of SSTables that are approximately the same size. The database compares each SSTable size to the average of all SSTable sizes for this table on the node. It merges SSTables whose size in KB are within [average-size * bucket_low] and [average-size * bucket_high]. + Default: `1.5` -*bucket_low* :: +bucket_low:: Size-tiered compaction merges sets of SSTables that are approximately the same size. The database compares each SSTable size to the average of all SSTable sizes for this table on the node. It merges SSTables whose size in KB are within [average-size * bucket_low] and [average-size * bucket_high]. + Default: `0.5` -*min_sstable_size* :: +min_sstable_size:: STCS groups SSTables into buckets. The bucketing process groups SSTables that differ in size by less than 50%. This bucketing process is too fine-grained for small SSTables. @@ -152,62 +199,6 @@ Default: `50` (MB) The `cold_reads_to_omit` property for xref:cassandra:managing/operating/compaction/stcs.adoc[SizeTieredCompactionStrategy (STCS)] is no longer supported. ==== -[[TWCS]] -==== TimeWindowCompactionStrategy - -The compaction class `TimeWindowCompactionStrategy` (TWCS) compacts SSTables using a series of _time windows_ or _buckets_. -TWCS creates a new time window within each successive time period. -During the active time window, TWCS compacts all SSTables flushed from memory into larger SSTables using STCS. -At the end of the time period, all of these SSTables are compacted into a single SSTable. -Then the next time window starts and the process repeats. -See xref:cassandra:managing/operating/compaction/twcs.adoc[TimeWindowCompactionStrategy (TWCS)]. - -[NOTE] -==== -All of the properties for STCS are also valid for TWCS. -==== - -The following properties apply only to TimeWindowCompactionStrategy: - -[source,language-cql] ----- -compaction = { - 'class' : 'TimeWindowCompactionStrategy, - 'compaction_window_unit' : , - 'compaction_window_size' : , - 'split_during_flush' : (true | false) } ----- - -*compaction_window_unit* :: -Time unit used to define the bucket size. -The value is based on the Java `TimeUnit`. -For the list of valid values, see the Java API `TimeUnit` page located at https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html. -+ -Default: `days` - -*compaction_window_size* :: -Units per bucket. -+ -Default: `1` - -*split_during_flush* :: -Prevents mixing older data from repairs and hints with newer data from the current time window. -During a flush operation, determines whether data partitions are split based on the configured time window. - -* `false` - the data partitions are not split based on the configured time window. -* `true` - ensure that data repaired by NodeSync is placed in the correct TWCS window. -Enable `split_during_flush` when using NodeSync with TWCS or when running node repairs. -Default: `false` - -+ -During the flush operation, the data is split into a maximum of 12 windows. -Each window holds the data in a separate SSTable. -If the current time is and each window has a time duration of , the data is split in the SSTables as follows: - -* SSTable 0 contains data for the time period < - 10 * -* SSTables 1 to 10 contain data for the 10 equal time periods from ( - 10 * ) through to ( - 1 * ) -* SSTable 11, the 12th table, contains data for the time period > - [[LCS]] ==== LeveledCompactionStrategy @@ -232,7 +223,7 @@ compaction = { 'sstable_size_in_mb' : } ---- -*sstable_size_in_mb* :: +sstable_size_in_mb:: The target size for SSTables that use the LeveledCompactionStrategy. Although SSTable sizes should be less or equal to sstable_size_in_mb, it is possible that compaction could produce a larger SSTable during compaction. This occurs when data for a given partition key is exceptionally large. @@ -247,6 +238,48 @@ For example, consider the benefit of using higher values for sstable_size_in_mb For related information, see xref:developing:indexing/sai/configuring.adoc#saiConfigure__saiCompactionStrategies[Compaction strategies]. ==== +fanout_size:: +The target size of levels increases by this `fanout_size` multiplier. +You can reduce the space amplification by tuning this option. +Default: 10 + +[[TWCS]] +==== TimeWindowCompactionStrategy + +The compaction class `TimeWindowCompactionStrategy` (TWCS) compacts SSTables using a series of _time windows_ or _buckets_. +TWCS creates a new time window within each successive time period. +During the active time window, TWCS compacts all SSTables flushed from memory into larger SSTables using STCS. +At the end of the time period, all of these SSTables are compacted into a single SSTable. +Then the next time window starts and the process repeats. +See xref:cassandra:managing/operating/compaction/twcs.adoc[TimeWindowCompactionStrategy (TWCS)]. + +[NOTE] +==== +All of the properties for STCS are also valid for TWCS. +==== + +The following properties apply only to TimeWindowCompactionStrategy: + +[source,language-cql] +---- +compaction = { + 'class' : 'TimeWindowCompactionStrategy, + 'compaction_window_unit' : , + 'compaction_window_size' : } +---- + +compaction_window_unit:: +Time unit used to define the bucket size. +The value is based on the Java `TimeUnit`. +For the list of valid values, see the Java API `TimeUnit` page located at https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html. ++ +Default: `days` + +compaction_window_size:: +Units per bucket. ++ +Default: `1` + ==== DateTieredCompactionStrategy (deprecated) [IMPORTANT] @@ -256,23 +289,23 @@ Use xref:TWCS[TimeWindowCompactionStrategy] instead. Stores data written within a certain period of time in the same SSTable. -*base_time_seconds* :: +base_time_seconds:: The size of the first time window. + Default: `3600` -*max_sstable_age_days (deprecated)* :: +max_sstable_age_days (deprecated):: {product} does not compact SSTables if its most recent data is older than this property. Fractional days can be set. + Default: `1000` -*max_window_size_seconds* :: +max_window_size_seconds:: The maximum window size in seconds. + Default: `86400` -*timestamp_resolution* :: +timestamp_resolution:: Units, or , to match the timestamp of inserted data. + Default: `MICROSECONDS` diff --git a/doc/modules/cassandra/pages/reference/vector-data-type.adoc b/doc/modules/cassandra/pages/reference/vector-data-type.adoc index 4e360fb547..10d26f4d58 100644 --- a/doc/modules/cassandra/pages/reference/vector-data-type.adoc +++ b/doc/modules/cassandra/pages/reference/vector-data-type.adoc @@ -8,6 +8,10 @@ The new type `VECTOR` has the following properties: * elements may not be null * flatten array (aka multi-cell = false) +[NOTE] +==== +Vectors are limited to a maximum dimension of 8K (2^13) items. +==== Create a keyspace: diff --git a/doc/modules/cassandra/partials/compaction-strategies.adoc b/doc/modules/cassandra/partials/compaction-strategies.adoc index d37e8e0dd7..e7e52f32cf 100644 --- a/doc/modules/cassandra/partials/compaction-strategies.adoc +++ b/doc/modules/cassandra/partials/compaction-strategies.adoc @@ -2,8 +2,8 @@ | Compaction Strategy | | | | xref:managing/operating/compaction/ucs.adoc[UCS] -| xref:managing/operating/compaction/ccs.adoc[LCS] -| xref:managing/operating/compaction/stsc.adoc[STCS] +| xref:managing/operating/compaction/lcs.adoc[LCS] +| xref:managing/operating/compaction/stcs.adoc[STCS] | xref:managing/operating/compaction/twcs.adoc[TWCS] |=== diff --git a/doc/modules/cassandra/partials/default-compaction-strategy.adoc b/doc/modules/cassandra/partials/default-compaction-strategy.adoc new file mode 100644 index 0000000000..9d154b8143 --- /dev/null +++ b/doc/modules/cassandra/partials/default-compaction-strategy.adoc @@ -0,0 +1,5 @@ +[IMPORTANT] +==== +The `SizeTieredCompactionStrategy` is the default compaction strategy. +Any other compaction strategy must be defined in the `cassandra.yaml` file. +==== \ No newline at end of file diff --git a/doc/modules/cassandra/partials/sai/sai-only-select.adoc b/doc/modules/cassandra/partials/sai/sai-only-select.adoc new file mode 100644 index 0000000000..9bc268b00c --- /dev/null +++ b/doc/modules/cassandra/partials/sai/sai-only-select.adoc @@ -0,0 +1,4 @@ +[IMPORTANT] +==== +SAI only supports `SELECT` queries, but not `UPDATE` or `DELETE` queries. +==== \ No newline at end of file diff --git a/doc/modules/cassandra/partials/ucs-recommend.adoc b/doc/modules/cassandra/partials/ucs-recommend.adoc new file mode 100644 index 0000000000..d91fb7b0d3 --- /dev/null +++ b/doc/modules/cassandra/partials/ucs-recommend.adoc @@ -0,0 +1,5 @@ +[IMPORTANT] +==== +The Unified Compaction Strategy (UCS) is the recommended compaction strategy for most workloads starting with {cass-50}. +If you are creating new tables, use this strategy. +==== \ No newline at end of file