From a977168c48d45085cdf0c40f9b9bde3850b1f3ea Mon Sep 17 00:00:00 2001 From: Michal Gulbicki Date: Tue, 24 Jan 2023 09:31:38 -0500 Subject: [PATCH] =?UTF-8?q?qat:=20Add=20Intel=C2=AE=204xxx=20Series=20plat?= =?UTF-8?q?form=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overview: Intel(R) QuickAssist Technology (Intel(R) QAT) provides hardware acceleration for offloading security, authentication and compression services from the CPU, thus significantly increasing the performance and efficiency of standard platform solutions. This commit introduces: - IntelĀ® 4xxx Series platform support. - QuickAssist kernel API implementation update for Generation 4 device. Enabled services: symmetric cryptography and data compression. - Increased default number of crypto instances in static configuration for performance purposes. OCF backend changes: - changed GCM/CCM MAC validation policy to generate MAC by HW and validate by SW due to the QAT HW limitations. Patch co-authored by: Krzysztof Zdziarski Patch co-authored by: Michal Jaraczewski Patch co-authored by: Michal Gulbicki Patch co-authored by: Julian Grajkowski Patch co-authored by: Piotr Kasierski Patch co-authored by: Adam Czupryna Patch co-authored by: Konrad Zelazny Patch co-authored by: Katarzyna Rucinska Patch co-authored by: Lukasz Kolodzinski Patch co-authored by: Zbigniew Jedlinski Sponsored by: Intel Corporation Reviewed by: markj, jhb Differential Revision: https://reviews.freebsd.org/D36254 --- share/man/man4/qat.4 | 6 +- sys/contrib/dev/qat/qat_4xxx.bin | Bin 0 -> 532308 bytes sys/contrib/dev/qat/qat_4xxx_mmp.bin | Bin 0 -> 150084 bytes sys/dev/qat/include/adf_cfg_device.h | 2 +- .../qat/include/common/adf_accel_devices.h | 89 +- sys/dev/qat/include/common/adf_cfg.h | 10 + sys/dev/qat/include/common/adf_cfg_common.h | 6 +- sys/dev/qat/include/common/adf_cfg_strings.h | 4 + sys/dev/qat/include/common/adf_common_drv.h | 16 +- sys/dev/qat/include/common/adf_gen2_hw_data.h | 172 ++ sys/dev/qat/include/common/adf_gen4_hw_data.h | 132 ++ .../include/common/icp_qat_fw_loader_handle.h | 1 + sys/dev/qat/include/common/icp_qat_hal.h | 79 +- sys/dev/qat/include/common/icp_qat_uclo.h | 48 +- sys/dev/qat/include/icp_qat_fw_init_admin.h | 4 - sys/dev/qat/include/qat_ocf_utils.h | 7 +- sys/dev/qat/qat/qat_ocf.c | 31 +- .../qat_api/common/compression/dc_buffers.c | 64 +- .../qat_api/common/compression/dc_datapath.c | 352 +++- .../qat/qat_api/common/compression/dc_dp.c | 29 +- .../qat_api/common/compression/dc_session.c | 385 ++++- .../common/compression/include/dc_datapath.h | 14 + .../common/compression/include/dc_session.h | 204 ++- .../common/crypto/sym/include/lac_session.h | 106 +- .../crypto/sym/include/lac_sym_cipher.h | 24 +- .../crypto/sym/include/lac_sym_cipher_defs.h | 26 +- .../crypto/sym/include/lac_sym_hash_defs.h | 67 +- .../common/crypto/sym/include/lac_sym_qat.h | 39 + .../crypto/sym/include/lac_sym_qat_cipher.h | 27 +- .../sym/include/lac_sym_qat_constants_table.h | 111 ++ .../crypto/sym/include/lac_sym_qat_hash.h | 5 + .../common/crypto/sym/key/lac_sym_key.c | 11 +- .../common/crypto/sym/lac_sym_alg_chain.c | 1519 +++++++++++------ .../qat_api/common/crypto/sym/lac_sym_api.c | 96 +- .../qat_api/common/crypto/sym/lac_sym_cb.c | 128 +- .../common/crypto/sym/lac_sym_cipher.c | 302 ++-- .../qat_api/common/crypto/sym/lac_sym_dp.c | 337 ++-- .../qat_api/common/crypto/sym/lac_sym_hash.c | 153 +- .../qat_api/common/crypto/sym/lac_sym_queue.c | 11 +- .../common/crypto/sym/qat/lac_sym_qat.c | 129 +- .../crypto/sym/qat/lac_sym_qat_cipher.c | 867 ++++++---- .../sym/qat/lac_sym_qat_constants_table.c | 257 +++ .../common/crypto/sym/qat/lac_sym_qat_hash.c | 126 +- .../sym/qat/lac_sym_qat_hash_defs_lookup.c | 490 +++--- .../qat/qat_api/common/ctrl/sal_compression.c | 69 +- .../qat_api/common/ctrl/sal_create_services.c | 3 + sys/dev/qat/qat_api/common/ctrl/sal_crypto.c | 251 ++- .../qat_api/common/ctrl/sal_ctrl_services.c | 38 +- .../qat/qat_api/common/include/lac_common.h | 3 - sys/dev/qat/qat_api/common/include/lac_sal.h | 3 +- .../qat_api/common/include/lac_sal_types.h | 14 + .../common/include/lac_sal_types_crypto.h | 12 + sys/dev/qat/qat_api/common/include/lac_sync.h | 2 +- .../qat/qat_api/common/include/sal_hw_gen.h | 88 + .../common/include/sal_types_compression.h | 9 + .../qat/qat_api/firmware/include/icp_qat_fw.h | 116 +- .../firmware/include/icp_qat_fw_comp.h | 132 +- .../qat_api/firmware/include/icp_qat_fw_la.h | 408 ++++- .../qat/qat_api/firmware/include/icp_qat_hw.h | 201 ++- .../firmware/include/icp_qat_hw_20_comp.h | 292 ++++ .../include/icp_qat_hw_20_comp_defs.h | 443 +++++ sys/dev/qat/qat_api/include/icp_sal_user.h | 37 + .../qat/qat_api/include/icp_sal_versions.h | 4 +- .../qat_direct/include/icp_accel_devices.h | 3 +- .../qat_api/qat_kernel/src/qat_transport.c | 15 +- .../qat/qat_api/qat_utils/include/qat_utils.h | 21 + .../qat_api/qat_utils/src/QatUtilsCrypto.c | 39 + sys/dev/qat/qat_common/adf_accel_engine.c | 32 +- sys/dev/qat/qat_common/adf_cfg_bundle.c | 49 +- sys/dev/qat/qat_common/adf_cfg_bundle.h | 24 +- sys/dev/qat/qat_common/adf_cfg_device.c | 33 +- sys/dev/qat/qat_common/adf_cfg_section.c | 29 +- sys/dev/qat/qat_common/adf_freebsd_admin.c | 9 +- .../qat_common/adf_freebsd_transport_debug.c | 31 +- sys/dev/qat/qat_common/adf_gen2_hw_data.c | 132 ++ sys/dev/qat/qat_common/adf_gen4_hw_data.c | 176 ++ sys/dev/qat/qat_common/adf_heartbeat.c | 63 +- sys/dev/qat/qat_common/adf_hw_arbiter.c | 44 +- sys/dev/qat/qat_common/adf_init.c | 11 +- sys/dev/qat/qat_common/adf_isr.c | 8 +- sys/dev/qat/qat_common/adf_transport.c | 109 +- sys/dev/qat/qat_common/qat_hal.c | 168 +- sys/dev/qat/qat_common/qat_uclo.c | 320 +++- .../qat/qat_hw/qat_200xx/adf_200xx_hw_data.c | 4 + sys/dev/qat/qat_hw/qat_200xx/adf_drv.c | 1 + .../qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c | 973 +++++++++++ .../qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h | 111 ++ sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c | 267 +++ .../qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c | 4 + sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c | 1 + .../qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c | 76 +- sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c | 1 + .../qat/qat_hw/qat_c62x/adf_c62x_hw_data.c | 4 + sys/dev/qat/qat_hw/qat_c62x/adf_drv.c | 1 + .../qat_dh895xcc/adf_dh895xcc_hw_data.c | 4 + sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c | 1 + sys/modules/qat/qat_api/Makefile | 1 + sys/modules/qat/qat_common/Makefile | 2 + sys/modules/qat/qat_hw/Makefile | 1 + sys/modules/qatfw/Makefile | 3 +- sys/modules/qatfw/qat_4xxx/Makefile | 10 + 101 files changed, 8939 insertions(+), 2453 deletions(-) create mode 100644 sys/contrib/dev/qat/qat_4xxx.bin create mode 100644 sys/contrib/dev/qat/qat_4xxx_mmp.bin create mode 100644 sys/dev/qat/include/common/adf_gen2_hw_data.h create mode 100644 sys/dev/qat/include/common/adf_gen4_hw_data.h create mode 100644 sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_constants_table.h create mode 100644 sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_constants_table.c create mode 100644 sys/dev/qat/qat_api/common/include/sal_hw_gen.h create mode 100644 sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp.h create mode 100644 sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h create mode 100644 sys/dev/qat/qat_common/adf_gen2_hw_data.c create mode 100644 sys/dev/qat/qat_common/adf_gen4_hw_data.c create mode 100644 sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c create mode 100644 sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h create mode 100644 sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c create mode 100644 sys/modules/qatfw/qat_4xxx/Makefile diff --git a/share/man/man4/qat.4 b/share/man/man4/qat.4 index c6082f873a44..e8a46a99e949 100644 --- a/share/man/man4/qat.4 +++ b/share/man/man4/qat.4 @@ -1,7 +1,7 @@ .\" SPDX-License-Identifier: BSD-3-Clause .\" Copyright(c) 2007-2022 Intel Corporation .\" $FreeBSD$ -.Dd June 30, 2022 +.Dd September 1, 2022 .Dt QAT 4 .Os .Sh NAME @@ -30,6 +30,8 @@ qat_c62x_fw_load="YES" .It qat_dh895xcc_fw_load="YES" .It +qat_4xxx_fw_load="YES" +.It qat_load="YES" .El .Sh DESCRIPTION @@ -53,6 +55,8 @@ Intel (R) QuickAssist Adapter 8960/Intel (R) QuickAssist Adapter 8970 Intel (R) Communications Chipset 8925 to 8955 Series .It Intel (R) Atom P5300 processor product family +.It +Intel (R) QAT 4xxx Series .El .Pp The diff --git a/sys/contrib/dev/qat/qat_4xxx.bin b/sys/contrib/dev/qat/qat_4xxx.bin new file mode 100644 index 0000000000000000000000000000000000000000..baec3ad9ca32a87113fe185afc94ec264ff5fa4f GIT binary patch literal 532308 zcmd?S3%p!qb@09R+&O2?nGK1_NFv<{Avq+6WCsEX5MUDl(PAB|&{(ky_fSCxL@pI| zCnhl_6epV0knq+GMm39WQ8FsqfSmHNH*F_h0)mb2x`X z(AMww{eEB2>^!rd%X-%BS!+G(dG;;uy+MAsL&Ok~H2=Nr9dEwsEi2yiww09i&t=g= zE`Q4<;qN=+av^@c?eggN-En!~r0C71lyANI_N$iMb=O^2z3=W@7G87HdpbW{ckN5d zS5K&M^>w%0^q#O%+%b;~|LGqc^Nnx5@C_TnlE{X9D*jjWNZy-;==bFP@o}x>{kBrn zU-Ev|xcAodT=+}!{*3tjC2zU>Z6}=<;cS+q+BkFK_ffCF`NlVdIJ+qR{h~2&E|34M zBYZb}uM`kj??Q*;XItM~EmEbd|Bv4AkKe+dYk&Bu>#jX_=(#`tMeCBR+c>)Zb+sSg za@*VQ{mN(G^!D#O{F#f~IdyN~FF$tIac4c5O<(@hN6p=@TltL_PrKFmyRH>?e)cbK z`^THUy6%GO4j1mrI^X@k?|krIdjIsEH($T-_NyBSCr4s|MU;v zaQNpJp7fEAJ^NzMx^MmC+eR<^&SP&`_L+g7KX=8>bx+Q*9{!8_U;BlJe!2U(tNs4F zE_+AmKVJCYIj6ty_>srn^6tiEzk0*nJC48cs^i~s*Z)1&y8a_)zia5SpMCbmyFaz! zb)P=+ukSi*&659n&E5Z0Hsz@)7oWRw&nMQ}(|*?VfBdU{_rN!PlD^?v*Zy$*d+YDH z&hI(rx%c;7yY`u9Uh}s%{KwyY;|p`Ya_;cEzVXfX*cW{Cuf5_g&U*KP4^4l+ynf5Q zmwo7tu3y~mzHZ0&|L&bX`OM$n`j4%i#}*&=!4vlXi+R(Bv*%Cy$)}(E^5<6FxpmI`Yrg$~wXJ`?bN2Js&vf$7`>czj)cx4_{yWy%jtEWX&6we*2Ag?7H}lYkqQM+yA)o zy`TE={7mi8DL4MlJy#9&mA}*~%=r2HFZiq3bN+SX6MKLDKNoGiV6Xqg?=-(O_bVG7 z`nMHp-~G0Sjx!I;`g-HxD}T86kH2yB?VHd3qI1oT_kZUtpZTM!&KZ(l9rqXe|7O$D zyYib_-}~&L_kYRUd#&kKU;o0=3;*E9^B%e9>5tv`zfbz7pPc_UrJv2eYySHm{?K<1 zZF6or^_=4>g-s7V|110WAAjlaS@|;yQ|8b6;@AG=eILE=!52Ta^SV8=K9POXRk!v3 zpUW@(%9YoBq43)O$X7q}hF>ne{;K@D%Dr2E_m0c=4$u1ZC$9a$Ip6u!SI@lXJOBBI z#czH5i@*BiqwB93-utm{e0#yAk1TlbivPUx7dMT*^>?St|H?`K>&08AP5t+8z3spg z-+b!bU--hKL$_w0`cm!9-+JWdpa1MPZ+s$n|FeJl@eQx};%BD6;mvPEvO*~<9OYm7 zf9I%}p}6t05xx?+>Dt;=@4Nom_uqEY?RQ^w?cG;hb*&v%`TxsbD{gwv)wkVkUH!iI zU4P3pH(!69_1^bbzxUodEb9#Gbo=zvt*dXh?6c0Xmz=YBsrCN%-SnOtt=r%G-kaZN z-EsYG@4Jb*XD+lCEc75{h9BVNzZEMB*Z@T%qtA6kOw_J1m zZRc2LS+9TZEw|isyY`9O1h{_3XfglxeB%1~yy@&|<-p&AJ%cUqku#f{ zl@1xx_3f0D#VHLsz@7JcLipw0o+IF2x^;ev|7bqV+)VJ5mvTnZL5=#|T3_*dbNPt& zn|`5uq>*dm$<36F{m>^bz4Qb8GtVk}6}X{1x9l;#$XxMFL-JjP4FU6*)dijd`YlM= zYoukh!MJ`}jv8~_Ep@7QnH$_nR@Un8;Ax#c-IT1?r<0P8>XZ4%JS*_Eez)QedTTl0 z_in34%bu1^EywUn&hlK~&rGvC#*@O_=vh1~`rW)FDo?#4D%&?$p3r{D9hRr#S?id!f=-&3kvbNfB`O6J_r?^zJbx}o1QdD>U-J|&N* zm=E}q{<9W(N{7C=3%x4O{yP?WHOB3`VWG#oWyuxc`yXh1Kc{?j@diKSr`K%o^KzR0 z-pY&R{OqI#@Fwz?SAJ&v9(~dd!iev~1{rg8P*uLLZasJ_^huT7I(VCvD<9rirCMHE zyC{p{ymhdVmBkl*x4+TMPLP+Qv*x&K>lt}$cK>b<`hPXGeD^>V`W9!17tp>iLwx3) zhClZLM=E1F`{-Qmvl{h!T8wOVEnwsejxXd?WiOOxd+lty1thk-ssM}a< z-Tk(d4JqW2OJ6NG+Ihg8F=>YvHR*VlIWj2Hv)*d%p#NE2eT^N^V^&u=*ilKz_h$D$ z>^XV)erox{12yP&Z?||Z&kYO24~$9Sk?xx-a;c&1T^~GuKXt?%XaP^bX8yD*+0I`0 zAyu@UeYVcuY1GY(luGwjWp73b-S8{(C<*O&NT=uHu{Dt&A82Grga2jSX`yZP>^cL#W<_pQ=0R zDS3M#(i4nZ+vV}zkjh{~+l%d0z^~z)F=*0A`lMNdZ}Qdbk$`gC4|<`o8hHF}6Frb+ zJovMom8=o07&Nm|Q9T6w$0^@dfyeNd4tnq*{V6?C1^C^f9M3ydDEhk^A%8Db{!W>c zX%8GxxlB1=9K)rbw3ynis_o&!8vI#QIa2;!2Avq@$Wi2s=0!X3Md5D+171#yk{#^x zBf5SrV4l!_FO%=IztkB3-b~R9_SN}4YXzfe&=omvZTCO|`fqy^JEf~H*zSSP z{ia`c(qeb@`P&=F1^m4c_|o|euGP#=pc4p8lpn@FgDX^TP5j*%8HKMV{{GVTtba8R{uRT$Vh}z};Ovo`vuZCS zd{ic6cJL|in(&p}INS4}Q|8XuUY%$8^RqqpRAzp8wrBFRKFSlk&iNF-W5=92UAETf zJ{LT<8hRgHG+p+%ywCa1tycbx^<_r&CG^N_c=it)Em1j_Rx>Ye5A7*!Z#Fhpv4dl~ zrUQTNU8=7pw{s%>IeuOu=m#eynW^Q~fhiO1qs)N#(vhddeYI5$7kEozw3>`JMN%!* z&od;`a)D1S>G!;x9G%kb?}DGRhJ1ar#yrh#+pAH}2(B44>92sD1Aod?wSOp^YX2DG zW-r^li=VMSt{ZHoBi6=wy>WX3^>lNTA#h_;a!Ydmg=g(ATQs$hV{Q@?uPE{V%uI|07;4e;dt@t{w;6DhZJ z{ff%0zs>FN*VizkcfoJq^X9?2APXR-{YtIkgnGg>$#s8vci6k@eJ-Bv{)}o z7W{(U(QD2Q?Tn1B&pF9TG1GE*vi|PznP(Bb6$t!My>k#crd8gVcc$BPUUYR{^*aZv zS$W6$`p-6c>F~M1%E~))es&Z1l8ml<&>?F&TO9i9=)t=OD{5z%ZyRR*wBNchoM)%J zf%(k3cvfReOYs@4Z-&Z`laieKX80Svp1M^0%2+sSA5=Y`=#L!s@E*oJdgdEsYZJX> z(vRu|!=WB}A3JNOM?Z$UX3%FHo&+yliWfI4U5b|){kQRAxr4wr5ib>YkovzqUS!Q+ zUG;Yd4=;xoOW_uUJNSso339ns`6#V+pN+ok_M8{NXS&-7;{#nvFI!|q$%4N4m*eQ( zt@A_9x^;d`uQNY)y~YvB(pazd-LF3%cS8iecLF!M+H@6%w zo1y#BQmdrv#>u@6XQv0;8jk_aWAlN#W6)GR^ve7-*?f9+KJZPi&IkGF%%}P1%J&_4 zg1=nVyXL1~Za!@|Z)H9~rahlIy)9>FOYzTqg7mA?^TEDgeN{haDsctq zJr6$!eI`kqY~}*|9OzrdAGfJ@=5%)}e3Vyzsg~jyI0KAx^voa0I+M7b@+o*cx_G*~ z1-i~6F0e=Cq^+<0&#V84eG}RlM;Cuywl?6J`L)Jsmv+mi%N7eJ%29?d=UjB*Uo>jK zv&O4#_C-#fKhWA>sr0e{;ik?X=99Kj3d~QoU~g`9NM45Skq9@G9?48rE07jLJ#-q&zdMuj_vF-qJGspOkGaL${>tXc@SYvKhATydPg4 zKi>HFg|NNq%a@05eEacbOWTiu_cxtSZ$Egh=dqihM*u&z`O0f-t{_Ke-wA)CAI|=^ zyRM$)`DNB|_;E%k*)nGl0;mY^p_rs6G16?2a=+SzCF~JYXXJ`l9NxN#P z=Ili7-m%{Fwp589B2T*-@OQ$`ZTSMO2JhSRiOPn`n+N~Lb_;&$R|ny{j$BYip3uuR zU4O8nA0Kp;udL&T&^I#8bnC4!z7)kPmrY|~KrTx3tM6lZme~I5DdKM-Cp{Zpe4^37 zeae{Wz+&AZ4!XZi97N?4J|FLYN!GRzd%;esh3lGOKRrl&>207Nh*QXwP8fHdUUBw0 z=pT&(fxoT-^42sYrSZIgxYz>kC2=6+(b?O=?~|F1+L8G}%^5&0@_o>w0ba6`#etgN zR6Dmz9(8>9N2c2NbpT&*ME$NzpU?V<{m>6R9pLEq-F=KBlf-|F++Ed;=pADb<5DfJ z@gMCsiT^AZuvZ*N%d8VF4dbNk{-zS|X|lhfc&mcDS8PG}Feyu$_sC=KZ!g!yNCobBeeU}x=MdG*16yiVdMVrnEy<`2_J#)G1oAlEI z{ghrN@Ya~u=9=&N#79ru>u-neQ~i5odma6_=%sc@x6;*tj}m^GPTbO0eJwInf&5dE z2g%BDhC4W@_?_Am#fPk^`KzJ(@v2Aa&~uT@KBa)&N)@$3Scr>9Y##a^Iv>|xcXu_3 z6D&slYxKXc)mUw#9~OVv-BJ^o-T&-JqY8aOJ5Tk*ba$!+*OW!8`_UH{hJ`?LB(%I`c6Lte)E!-dqn-P z(y~LWXROobnyI;~fak%M@4{z&rypRwtXuidDcto zrL>->^_T@SxP>s9a@rC)Y| z=Zw~~;0vv{P2{Y;=D#+4;Jx*&vKl@*y5L6H$-L(FyZ_PvPdTIPyUZ)6{6xECysYw7 zmGYt*^9cRme^vYH*!3kf_W=618~e&aZ(?r_Fiw}k3!b{daq?|=dvu%%^jJi@0{DyN zMmF0aeiJ@V%X07wo??DaYyG~w3Bz%2VG zhvCaLroWnb7z(c!K4+goTsw*X_7);NV@z?vxcjfGFR-&+-LL7LT0zevk1r$V9(rab zcy2jq!OpNww$<;ESTB4D`Gn8<6F**lrR!!Q_ZxKIv&DXw!rxSS5nr!>--YOJ<@=Zp z#v=5;3*Rh6@A}lc&ky&RjE|sq533z%67Qz^V>nj+M154Cz3xNFqGt98^hkBTkUvsU zb2);uss31%eHO#{w(i%A*UxRceL8SDw0ht_MB{L!P`~Ip?iYsyzG2_OhYqs93)fBA z_6YF~ABx8m4<@o*Be0R~3yT}*GGY)dYdgtoCU$Xx+**(XrJ1R%>nTBn0DRfnTyKDtGyJ^zRSao zNXc>C=rf0LPN;_UyV1)n6FYw9Wv*{i?|8?qd(3wk`bY3Lm!Dt+b@W(2cCm$@dZgs* zx=eh>8sb^%H;w%b?2c2hSCF&BU!fjyhMczbmpgk%-}gJ>wSX_)mzSB-#WU3osm>c} zBA4Yai(eDjc!t9|M7@|kkMqx_V~p_!i@VcB-d5{T(~I1-`SB+4uIMA(weA zZG`*G89CyF`!Y+aQQQNz(Kv5^B&T=&#V} z4}C6&9L7_51izAJpFyx^7qeg1Qv90O`)bcFb=M+?hI3h<_NZZD$AV?zIqTNd44He< zJr8;4*?A}3Gkc&#`vrf1JqDi@!4FTX1f0z8=;Ako_VcVMP89EH`xo}op&D>%yuU)f z?eV3t+NZyEyeBHpxoH2gr<5<(_|?s3N={z5x3#@Wyl3IwaNiWZIa~Gctc81n?G5Oy zc(V9QpB4U!*JasOusymFBe;ABzHIv;YA-eTUF{l6?IGel*k52Ovrw6-Y4yo@=SdXKPP2N%iupL+gi3%KS%8yE!R8!n_>IT z`|;)RW`xRbJN|g^ z`$N#HBNt`}$I&yt9?pk;=wIVW!0o`F=e@z*+CmSV;6(M#BhDpwmu7$;%2UT-eQd*% zQFxfY#-Er5aiqYj<|f8r+T%p{eTDc_BwsO}VCgFqFJt0QVcAgmhyTayzx}}w?DVrRFYEd6PdH3V8cK!QBP^%?jL{tmLZRBRc{7q z`GCR;JjuME)5@W$?&FM~Z_^!8I+wAF>a;H~?@B1w;k?&sKR)j#fuk700bOD^dUc*P z@Dk%AnP-BJWS%{Ww>ojW7$3>J5_}~6ji1+_DLnSrdAZdgSLqwLYCk6Pmo?(A4#-1R zd_=#We#!uTJ5y|eN8pO$H!;4BD-yra-;=n<{AKvd>Q~bdzy=0y#t>+>CrD|$!AO5z^)509mdXDmM)L%yHS@d6W@PE_jb@0YKvRW@-9x;F9 zv|b&#DiMF!z!QEy)ClDu^7p62{{ECkYThc|x8b^!@!;#YAH%$q_yTZE0sb;rXwNgM zm%x?MdUfVG1^D6Hw8ACWDf6jUg%9HXGFs1MyeYuHP4hLFr-whK^{|tY{&HH+W1dr7 z;zcy4eV9ic!!1*qsd?MzcM5c3Uh+Zs-OkBXC(sZ4oAhI8KP`oe_`@OUf6(w?6#In?1XTi$~xInv6DC+a8~HAcKUZe6#ADj9@_B$?XShS zCHzE3@CvcZn0+#?5$7M~d(}UrUhGeZ#sfCILOdXbv!MH`uJ%tnphlc&Ts%O!Ejd(6 z$=n=zyv6?M0|#8sOp#y2I;3=-%QPn_RI8zqci{uS6 zPD++Ln$N-flK24i@n2=s(f#6{g2O%rexcgw;KL{oSMsqVP4X_Q`1wWFA8<^3V8c*= zKcMk}K;ski3!h}PU-UNdfj=FhKRM10{r?_q=cv8K?~I4PzA4lj$MfD{+|K)t4Ka?6 zw}<&9{ShD7G*n@}lk{g4sLwiKXnerIuYlcmK}X^Pn}(WdA2F^AJd@h@#0P4MZ}@?6 zh!5Zo14kaZKtDt6bN`jvudHsQT`4mYKLUNC^lx04^Q@sF>@PuYhk6XJ1Rli}Gjl>B}5BzRybdWoTJxl zpZz@aWPdC0n&?mMx0#cVFDV;=m6N-c!Z#lFfl>b-L-755QeM3uKH&FIPnI-p(gMFm zd{uPdZ_qX~qlv^{ygu0q|Q~W?|n#tz}JjUjUs8&2T?ze7>~Nb%1y- z@SV%JE?TjxHL%HHo(cb!4pmpI6q$A6DsP958~4{E_pbEn*sW^URv}WHe{FOfs%~Wd zNq@az{x##r{pIo-Dl3+e4|C!we+Tn?+!AR*Us-}*QBl5h|2Fg+GdD!>2bqR^!QV@o zVcevs__eWPkAYv!tJy2$x5fA^Dt@b}SA*X#Yy2g_Z@1#tXZ&O0S7vIS=|p@CR5#Y) z^BBKfUFPmEZra9g;ldT$@aIol73`=I|BCV3ExT9hxD)#uk6(XaQ$zJ;biO5*AFx&o zhW&;1L}#B(9uL=pB6xL}PXWBbf64xFTJZ_r!s|VjJzq-hs&L95aJjzVZ&%Vt3 zu{V&r1fTgXd-tXqVja)Fx72s*3iK`btnbiqlm6_0y!Tgw&&I$eg|m&%1NMq?$gh$8 zRo1vP{A@U18`64-{WU}5(v;(TB4$<#^X-k`n?p5zp9WQmDycc!7u+d9K@YGN%C--L2(>kAZmR}Fp zXIxKy9{4R~R>?*mKJ7kWS>TEGkzL)Un>?fI+hla>tXJ~vkjd^=qF$jwf; z9@x!LFX68bsojGX4A5m(+<7VZfDZYW8J9H&Q!nJ zIyBT|UPU{y$HfW#-Ol{2p&`}*q3bZ-r}`IpP0LBLk5Ss~6F)%L#RhTPWyFsI?3;6| z!7hd8f+l`DdM9^yIF=T`4>ce@WiJGZ#Ho+t5jnY4YMGrJbjIn;l&mmSFct8ZVnXWQZW6!M9V zzXLuwcKf?zV+Ax`P_SlO>_f$TVilL;2eNKE|K{#PF5B>T_~-c{Tj|{wof{h`_u#X& znf8BnZeSb&&*|ROhM<;_kE9BP{Z06%E0x*b%*mQmVeNjC?sKU%`>Ui8NFfzCivN@m zxSZED3abM_PmoV=sEU6JKkO5YOXtIQUfP9D^mlaejqa8PayO&usQnKE#QLfAuR{&+ ze{RGlM;HG>wpJ*guX!891DD?@Ta?ZxQg+D$ylC|cPgQ8YrbT{+At#@IILgO3|8SgV zX#JboPqWTH9Oh*hs^@I@G}d==LCi0z;o%QJ%?|tYx4DY{OCIC-hr>L>%<{uwJRx@h zap!@~el!-Vz0(68Ls~<1jXPf=>zFr~#(%;-(f;7yPTcvY-WHejXL&0K^9(PD>SYb{ z$PoOL*l}`xE6g)IZ~^qFsC@Ch%DATDPyLp-T%p|3I9gne?Z9@qru9|6;`fSwb4)p) zEGpMY`yTW_nCBXe=V<#e@5h&g_pLGgjel=x`&WQ(eEacbm-b`f{SD^>J;?jS+*5iu z%75ed>g*fcwKnuS`$z6NrQ_MW574*gx5E6voUV%jy%;w9F*fe}c~m)iFf{-_!Ur)Q zlZTfb2Jf9bJu5Ygd}o}SUEROWm7Tz&U1oVi{{*hg@+x@|@K3pl9n3zS)-yR5TISql zP5GYv$qMTu$13#POwumxeo%H+HP5i>ZE@f);Mf(w=LtWzgmuNlA7CB#C}o$iKUXIoOY60?o=ZJNtJy!?r2m0? z;aA`~ll-X$`ATP1e2q^fajfpCE6Jb2&X_K#xq~Vv^(DhjN)}Id;ZvaLYmgtovmN)Y z&m3kvvHkG(59d8+Peu9NVm!vrnwzx)B?nv8oY;|92&^BEWYFxj}Wj(Q*^wVrxS z?A16fSyK2K*rns(yIA3?F>lUo>}dkeaaK64`bnDq1Aw9bC(>~BJTGcI&NJ|53~%(A3sUthWOj5!{F;^-iB_^bY~C!r8NHL zLU+{jt{{xR%^S9K9dJ(_c7Z2@{R|z(?}rN?s-7yoUao=!a>M9!dJMRc?^G;&g{RMe3ccdX9SMsGjTSkshsARXxJKh3=P} zM}NSTI-h!V&4!EM>(zQ5tC!X*pg(zYDA4(0e>u#S|5W~EDf=8P_--lt9Obd|XoUM5 z$vmtjgV8?0Qs%n~bUo;Wc2er0s9u-sM*hib)q3zn*HY$-KP0c!`J%5+1b@&!fzJ-{ z0=zSHzVz3gFZHCO*L$>{&Uc2(eBsaY7+3o{pL)P&R!e-eXXu5$kU17k#!@%cTw?Z zG4G}9w-i{1%Gd`k_!Q6e0^ptmuU!f^cquB}10DWP?B9g{AE&3L1HE5{?>!+)7rf~? zQELWynCLGs7w@~=5amU8?20!H1Gn;*p8IX@yC{BYG@7CLd4^ag5(iQEW4swN)ZW3k zI1xFvsCQqLJUjSQ_g$dVqeU`CsTCZozIYlV{RbFs_rot%PVYZZI=uA0BIx-LcboEWvshvA20AF#b&5c|oT z&-276x41Rj_<4QL~VD-jIR`&nuXz8c)+x*ElKd$HdzN_VG zR%Tzfxcp2NzfsTSdW@q6D)6M)SJ;VMjX7^~rP_zGJm#MT(8GbN1`dhm@%wJtL8snK zQ+Ck4mWw<$*a3|7&*iKyH@qT zq2t)PK1F3$%S>wCb*ry^x(QRn<22{NPgTe|gMAhB)X!$LKK^2e564{#9x^Aca<|*i zy(|m8AS>r(Zol^FO1g~)&Lb}j=V8uUsB*qMbJ4X=TUkBlPJEsHmoul!R`4}5BO8Jy z^S+lnO7NP=vyPg&AK_m!Y%~7UI~RJ4E9Q+0^?X8dex7>7t#jHQe>?04zdWO5{BeF% z`BSFh&vVWaxK7ShWGCaYa*?;l{0;xkVUu>Z{FQ93P;b^noS)$L zn2uSe7@l9v8o?vO(Bb4vaP!kO@V&<5o(zN}SB~z<$d$u*Jk0!gAB6X2 z$n@y^Y-Zj}ufhBaOJ{mD=39J2bRKqp;MbdZv2%Odra4H$lh5F3O#;Z1IW#;$89iq;`J~tdcO#a~pVQ&}qLpQ}5wQ^2Owg zXumvjI&sv1`1MloU}0sSZtdO#AIN<{c+XTWU!JuBdn%zzN8aVp;fAVR{7skkM&oYf z2l$$ODCQHdANZG#M)x0>j}F79Ir2Gmeamw0PWQ1(*oVG@Ii0sycXfYzCi_T?pO5yt zk#GD;`Za2QGmIR`fYOul?v>&EvNrf`lFu2G*ssGWj`_HwFXW@cE%viw`lrFa?#pMP z_kfRFErlMmYtw&8!`bVTS0ayw`vYCJI0G&8!V(`oVP2>C(LR`-6KG+-EsEsZFub+O zzwLd6Of&M^V*j&yNi*0NaBgZD=}b2IGua2|C%z4zj6c_qSqR;M+c=kTA$Rve;@c+U z$LASxO9v_|wuN-}cQBuJf0NkPnH=_YjrpFde1yImXJ2Q}AYZjl^=f60#FR@4KFM&|Bqy3gf>ym7^AZn&$t)FKNYB1H8HHQ+oV9hxh13Im7BX0-jL> z=v~UX4_i@uU+IMFcE*nGgPT)kM*#X-1@4Q(|C)83yVj?i`O8RtFf8=^Lb9)&(|v6l z`z>E|ND`1>_DomL)$C6d-GMs8v0mza()jvp99LvjhO5e~@El0O=j~@pO=HLsPJu&G4MSKzM$hI@O3ME+j5h` z7wl1&HU32W{=VTRc+XUbqo?JuT>oy(dzAM6+7qnXypQ#t^1I&emdyl# z=Ft}W>5umrqyDL3rp@|+AEIas@4<&j{&rE%ed_#Tf3x>oYk3#us_ow^J zP4-n`&9V95ncM*Ph_PP9`QTkDUk>w$`^%|(9U@LaUh=*`^8_lphU;Pfp`5kp74us` zEci1!f$vlx&wn=;&*S@Gwqai|T84%)aga$Mkigr?GoEm&Q4%aqDfmu-sKUO7{tU_#|`W zXj=1S)clLq%f=%bXH3@R&bsM5G7LXwW%tQ@ZPH%?$O0yQ? zZ`jDWQTzL06FE4I^TQ5&9N#C1f?eS60_vMzug+{?FJRE{LPpDXdZq-^7d z=y}op_vk%iQm!rZntHtp_LfcAXgx7Zza_c(J&N}qsJ~_bpX#Lw{#tC8iBWw*=b5^_ zuzr6Xaa+WD=$c;ssOuw#CG6F`b{J3p`Y>?4t{?x!1m3tF<@NjNFZn&O$MDMz3`g|) z2aPv!&t*&f>kc0}|1fNYzlZ1C6Sxhd@{j7by$(|(Xepk9X(IvpyifC%SogDY+;w&0 zCF*CO$JlrHr(s+7nHtkZsvW%IdD_uEAWqjvi{IDFHS2(%UVk-=USE9C>ekcnX{p?D z)voAKy99jo%z+*jc*v`s;hESaXMmRw-%%drS5q7?p&yxGmn8lZc2SM@i5@#xZ9J4B~4e?q3GOd`#Hfbk=8KuoM@LM@V&g1LZ;7rc3CH9J_y<+Qo z>=o^Q3I0Hnan)Y2>1R@VrS`n?`O#T>T=loJYPY~IIoTcAy-OtW-_>pbZ(ZRzHo8;0 zMRXob@>Gbwt2|c~FZgQ>@+NbI>c%?46WbY*Yhh<7zeM?#z2_dPtvEou=Y&hc`)rf` za`NO#wYMkshaFTWjuQ7*%ryo!HJMl3UuNln>WTs6vEOnv-=)hiTdLpaKfG_ZO;7g8 zl^=@&{{rWDzSR2nH;7-IP_A$Fn2+K9c(}o{ZMQg-naAXIOa1o&U!ok68DmwVZqnnHRkVBKXL`%aqiO{u%L77?`j_E=S=FUziaDz_#6J0{wDiX z#Od^$luLPX{yw7ilbjJa`yBP#$?Kwju>RaDj!As-R>{RryD0P*<2rSX+57B%_pV&4Zf28dKSUo ztHOQLkd6soi$xr=$+&Z(eA-uw_g=+2;~fj{y^42N?S=^Nz5Vz>75pK^JMuL--tEFj z6@Hn3_a^(G{5}rvy^446-^SCVct@{T*pD%u#_N^L(%%l=6MT~nVDCh)i4pX?bUk-^ zMjG(xHvAM1x;=p3>0q}k{}TQ_aboWG(fK9vf*;&7f|iwjH@v4)GWFTvz4GJksqNS) zZtsXg{KUadMh-jnjZ+HoT((91{-dW6-&KC#9JJoUts)xcyFF-;50E7;$^H;>ne1n! zX0+VdgIRVVg?$w&pLdPB896M?Xy5NRWu(Tv>xl zdVfkbFRQI>pvVh}zAWtb7}hBxioZ$as)`omC=vj zi+iUycMf{u`#zF$=b3)&S)fhiDpPh?ud%aQ*4c^AD*Xi70^doVGxW7`4{Ou+aj{EDGQCPiVWf;GU=(v`3 zB}mCxnLVdJ1O8<9VeTVgo&y8e+iIUgc%GeE%X`Z0^Wfwofp@)cX-k#(WWEt-o>Y%R zoVB5TUH$wK;ANlA)jZCGK0VyO<$HRb2>RHQ() z%3egDKBW(KVoaZ|kUsFe;?ri{ljyyP^kKcJF;1I4=)JumeL{Px-CsEX)LTi73@-_1J7gA!CpS%!EY1j z;79AvI6AOz;_7*(Hr^(Q$E7uXhWsaZ>(Y2!b1WT*#|?&dXBdxb_g8kghfVnteOyIe zV>>yW8{`}ydMTzukH+JCD#dgyEzo#(XwRKL0zQ$yFb-#w=m$Jcp2ymBM(nY260HZF z^Wjl>&PA(NJktPAlirhQy?z9`9_yY=^e6XZGB3luYQzNoM1RNkWcKJincz3Ofc(`% zy00$bK0X`}`^R$0NS*l@(LI@|@SaS%)BdWOzxuSh&dy45PiDf8oqICP-ydnge~Z9N z13sP~-IHl8aMFhwia*{%r;NUDcFI%3`sw!jqV2){K1m1YHM!ao{a%N`Jzsu+KAA7Y{phn!Kk>QpxSvPzzz<(ap1h$Ze(AXL+T992 z>)tEC|6bMa3H%9v0soC79>0&L!>&D8d7M&vDy!IoOFt>rWJqr9R^6lx2>`emHO~ zK#rJ4!k-;IA|D*_*%!b*P`@O}D@)!RM(qP5I!>R2@p1|DyBDmQ=98{h5AB7f=6`kc zoopDXlmDf7b%`?`*)4l&z`JD?_Haf%eBuG`V*ye>zwy~k%v&<}C0=+QD8egwLK^4p zIdN5O2Yh195c?G9(_^f%>Ms0&e;4NOt*?UD4!q>;XZAx^&J7@c$Xi^$gVzRl#cq%B z+B3h=+8D4u7~{1^@!A@L*Nxz{Lin_eSAX4z$9}=9#%u7=5#Z=mypDmhgI9a~h&2(f zz2*z9=Z5(pjUCFIZ9lR8u@Rg4?eER?Mf;EA{6(hExdXL7qVGL~8&MW{M_IXTZIk&K z&R>tz6%X?CTI8Jb3V%J)WIrQ3H=ys`dll}KmfcRdEYyQ;r)JDgJp=zH`m8_G`1ms| z%2kaUQ$N0!zL>WHwW~JhxxK3DlSq%X?+uOphBzHJ@*4}JBik*YyJmi~gAjPFk#^uqk!q@RrT zgP!OL`$10QdCD!bWVMHrq3c^COMV%8-T{tog)8(AWvk*Zo)`D~BiHz+%SUCvLT@iW z60Wx;;wwJ$FX?^94fbX7+y~cWkTpT5AC6c2qwk7&={w~v#j_RWWu}XctKUoNDOR{Q zfJ)qBy4l1|I&G<|dCt;X4etw}9PQ)vdX)KH*J;HUd1r2&MZK5%P6E)831k;=b%pn< zkViYkEX)O2$vGgyu)UEg`Fsn%+*ns({dM4XpZ;z!WHpaOeseDL0WUe@0N-`soU-=s zp*Q0Gd$s>My-slobBU|4FUq+b-Jf+H8gbDRss{r0OC6q;+E1%|0;JMSf{=b6BA2NG zC;X!K*f8H*sZy9b0Nywi=`wDqi+ad^l26&A^?TOz-g2kg?Wdei{AHJfnT*>5B(O`%WAy} z>xhoO5_!>j9&#JPi5$oE5;(D=`sS3!@ELSn4g7}kfd!tF4+@L}rc~Z$=E+wMfQOXj zll=~#C;0xy6Gs-3T_{Z>_8qkR-h^XfwR<@bt?hdqhBb*rCfgTEx6 zqWQ`?Zxmg#i5ryM8@u0*_^PW+9F6$u7(Fq2CHE1=aBs>!y3KPkveDSRZJ@;hO^w0;1{SwQs#QD-6hHuxxPan=Us%yQhEXta}mKySi9pX1x z-}Fg8ncv}Qf{hTLyf>Cvb?W6^>;~G)6pQr@{!Z5Y66=1T`pw^`en-h<-A9ke9LM#M zOPRvD?-J)yK15GLws3tA@)$-c%6E2Xmn6RvUwlOEhgO((i+saZy8k!+-j!mG@7w{0 z^2;eUP7Z|&PT%eWcH7p&&Ha+x<;I#Js*3&#-s9P z4L^vd&PVxc()qZLDPK;UPkq1ce;;c;Jvtx815W#t8um#1-A-wLAJBec#Dou$dFIOA z{lesPgP#rKjY<5roA_-2o@4sYzV5phK5IcY7JlEy{@3rKHYu*{6TLTfS;73_GbZzz z4t}5?->)TK8@jRoKZ1TB4t-$6qQ5nczgg*-Jdp?84 z*|%DFCwi9tV?EUAKltWIi~jf?U!<2J{hi#4WlVAH*?qd7D$)cFdEj?}7dN%iL3zVVB4E8IQTY_*-f} zwd48Y?k`3Fi{U+8iGFC)&HSPAZ=v8~htNF9D}JDld|%2j{gQYU`QhImfzOhAiIRId zq}y`#A(s=+(FN28Ke|6|v+ib)C*&}>Z#J#>rFiUb=sCIuc1jVw0{+M+`Db0POL~u0 zP1`Sb!gzH193B0FH_7TfR;+v44t_C=Xgu;yzB^*UXYo0@G4H>l@aTAZEY_*GKk`w& z8}j8O{c&%JqxW2=^&DM`C(DrD+g{Li$PeRvU*kM7$@fByuDil>aEW}P_n%k3Gq`t0 z*QdPRUju$7Z%_QW4jwZ7?mh>*M(eBm#rKD#@|>ePK%5tn>AE`W;^P}zRL67z$9CS%RCirnR6ks^u0{OYsQyCnCmwo0@xK#3 z@lrB|KVBU1bl>`R;z3z)yR9&9IN^_kj_?)u3+Y9D`qjKo;3z78xWt1r-W~AF1mx)x zw^IH<9y;eNEB{;L772eOe3#IZ`yF2#vB1wndWPdcub+;%%yS}tv_`6|tI8k1pY#WR z{B%U|GD&~k${)ykxAI58Q~ASXUD0+;)xyBzPEOyv@`n$-V)=wWey)DQtL2ZRy`lUO zjNzwA_yfP{!cqDULOTOGCiX)duSw!Ii!a*W_pA$C+qg&EV*bfKt&m3*P@ZRDr|?Yf z6HmVP9?v76Y2sI+$NS(H8-3Rgzcln-vxq+=W3kR-SE4`m+MIJQphuuG-vgJeYaxr?CHiSMP)W z_2F6$T;OLCxY7#Oz$?HdbHjY8-U9l|guM(mJ6a(>aU5Lw{RY(&S)+2{D0;THz`ZK) z3w(7{_igk$4|={iQI)OXnDP`L#gsKw-bG&}1We`Qx zO6-f0o{#SM3ke^HQN3`~W}HIB-3NT70{M`p+VB5Wex8XO$fW$7e4l<&IUC1+Qvy^`v>a%WYJ^_4K zB95T>BFX+}u82KR!+$qbpK$&_<%;nQqhXCY)Q{~6L;G!lSMAqSKb?NzSLv~w%`NI_ zJMfTi^M1C$PSa7VLo64;W@3jPxp`Qz$=Z(V02J{d0g!;SgE49Z@M*fTp_hN$2 z@&566zDw2%Zyx*wpGh1agO1@j%>)kNepAnb-Vbz>j{tDR1^6Gj?h(+mC}(KzAP?9* znh%=Na!bp#U;udA^w#gzP%qyF9ayI_hI5+A75Sm3jaHGjWPR!4{sxnI$9gSexCh~Glk`?9Km2B1{`se!!ewBIXeUKjf zXSmc?y&sSd$LhrS9Nk|jss2;`$@mUwwNjdjzfuKU+*#dsL;syQ+Jvu_4@KKMtV{4s ze9t8M@0?Kq?yLi!k!Pdr>MAGEc<7^LqgC+u8s0O`nD=KX{5syAI{0eeTS;yEgHXRG+Ky|5;t-ZF+uzcvQGg3LKA* z!pG!IJ@Txb8ncfW#gS$0GSPgArN!W?XWbBgQJl4%mlN-!#@|6R^A>hfM$RhME_t@5 zc_HX`3pxz2;Gs8U_gxak9g&}Xj@}Ox-S=5g`43dTMrB!J`s|C)6y5lSBXE$3;4h2J&k`hDRW=+oDKNRNxUNN z-_ZN9ee{a4BS5~`U58yy|^_!Vr@*S?^zVLL3`xCeaA&KL3;vvDC zM}6dflDG$w$oh;PS!4N|pi@fut%jVGWW^~f;op#tlwU03lCY1y2ae?ZtgdCj4(LcX zalESj*3lYrYxT3ArQFdM?SH(=d$2p}od2#-kNx{tza;lElNdIcexW||EHp*)f_qiJ z)W_-<=Oy|@!)5wC4D`!NT~A*@zr0KBFPW%c!ts#zcaAE5PSh{&7_CP7Y@B|1=V*oc zll1or`sJOYP0b^5-=%iwucMEOS|9vO zreBWjA3I+jt*}oT`@>?Ge=%BtuJJtqJr%wq1V5t}LOYUm=5e)GbpJ&2ERyg0e!YwM zVnh9T3%gn48yB$N;J0OLSML{Ce9==|o@KpD_(KY{===j+g!!r5`>6W(*#1}5^*kE= zm;JBZ?tlXwCh>E+)XzDT{&oDXP+$Jn?bSzJ%`=<~u5N{E+bh5o+ojlF*om6|=-xVt z-!UGp#O}ubdVCZ)+*dX2MxbLxME^o{(lp3qLguQlYOqYm~> z+Ya_Wr0f3CSueV4ZTMULFXhwN|KdCt{uuP`_+QlLPhx+l|An4ye?P8>{b7 zBuZ#=QTN(E$9Z{{(#D0G?3e=1G8G}`a!_+RcQ{GRw<8TF@{ z=<`YZuhKN^Xj|iOmC*mnYW+I(kL`awu6}w?=zpPK3a0A`dEx(M|7$diLrlicJ<89l zpTCCxb=xR>+VQ^{VSJ#&cePKbpPTq!+#rNs1imNtzo-v6rHKDUx9Wc(56St6-eQY$ zCd3`o{{kN!KdEuA($i4B3p7tn{V(jTj{o(EQBl5&{4eZz>}8EJIN^BMef7~Q^Ef{8 zzs9`3Z!}Z88$-OKmatNvG$^Dt`ny0llhtf*YlPL%H$*8`qO{V&!P z^aKK&_+Ot?KQQsXit2xXhsoO?3h!;`3H>kb4bl2F>c{(&FXMkbuJ*$*{jY)<+6##t zG~WN}_*;?x#r|y&=IO-#SK=q7)c*pXWBjlC)o;Q7x_{I{&yDlH61|51wLQXLWBceB zy`0dSc8UKLu8SQ$^gpM5TVfxz?4I7+?F{d^Prug|G{A+bHvqwOl8zZZ^&{@Xd~p_gO(Xw3VER4#P9J@h}u z|Jpfr3Q;T?(0@khNF1;2N7HYWc^`+L z++_XJb`7O}G#=~r7j?Z#=s4!Rukk7!Z%<{cA5DFe`jhlmEKnafib_Wt_?3=~PyGLj zsz(@akJ7VE2h|(k4a9}#KMg(qVKR@lKB<0b)Fp1E{v6Me>IZ!}DILoZ9pn1#@1i97 zWZZX7&~G6fosJ(J+GEG^quca+@Hh6OSvS%Du^-Jj%g5F3N&M+~*IoGdvo7}VgncR# zJAq5u^6K}4N43L3yL+C5c}g+fGQkC599NR>q36`UB)+2iu@-)EiG2D&=!!@+Re#7? z;syMB3Ve#**F3R$lkbg(@d4_a#0RLAFX{c=*fs5T8fT}UZsJcabnCLh@4`u5T>T!3 zfLFB-C2NNHg6tp4&N_ZMj+*8ra?hHp_Xg8poZk|~59A5ct;1)WyC=Rt`*>de?f1`h zz9$=g_bsdWBo^S#VLebjPxgaiJ>SkhU>;FEiBnx`rP}!{eMX2^XsiSB=MzN(Xd@eQ?WvnF}UtgpE_k$v$B`4|6dw8gqRA)loF4b|hv%D+f!{ssEc z`Ze=OGCChe_fyILtN?J5A2T7J#D7ND3;5_&=99>Cqu^=ce9UKceLU8DxObwq!x5d& zB>5!%U}zU@iQ-5fewFzozb!rte){(xe-U|`?4CmGk?>w&L;2EDc_6Py)XrA@NqI{j z@vyvnnD6ds{?D>Pl>Z|ci+w`!B)au0_|{*%SNjK-B5294{V%s%iI>;mcagZyUg z>K2dv>t4<81x}48SE+xW=0%2a)bJh*@~3{F{ucY%fd?2e2mfE!>k{*o_F`PrAKI>2t z*BX~c)Uh`syh~;!>n!jw6^%nIp4$59vsK!6i=lZM?CYoWetP1KnHKPgT+;7(_!m>i zhh_d*BQOryjBk?9t@RD(-!z^gg)90!rNh)a!ux2%yn*|}IQO2tZ1*mHE{pdQ^_L&J zrl0frB414{-#x&%U)Ovw`hP;7tncLIlHZV*$Mo6E%h&X&>uULu@Sc^Up9Btf3>x%Hc7U{oZp;u!beK&x|l&sHQ_V6yoWuN7z8bABqHT@6k`Qz`WmOrff zO+V6cYB~9#KEdzz_37m0d-}BV?fW1f*SulcVTT-6zl?JsVf;8Le~r&x?eg&B4f)xq zpONQJi0p`@%(~FrUeCzay7(PD60i8V#wVZGakLpKh6?x8@(MD%qA>nuO zihia0xyRC@%T$j7FICYKInt~oiu=ZTlQ<;z#v1bB%uDkQl|wY&j(hA@2dtyHRN1Xp zuy1nQyI<9~(yu#jKYP$){dm>$mi$5ZHql?5d7HU|Ztk}>?>Q?z*7c26PEOS)dUt_7 zZQZYvo`V+pLFZlJuUKER4~TxAnBO^D@eZGL@NRMM2zEwSUv!Uem)_$GF=9MpPpuBB zYF}3`A8djr#kUpW`_M-CpL?dB-%v&V*RoEach4($JHG{f+fqVeb;El2Q! ze(L3=*=vi&DX8b@`G8qh+t$_==jo=mjL?s9N6525Pfx2fwpDWSnrR!J()*XuW8prw z7}ePaD`9@Cv#r6p)N41kTkM~o%>4lMFi(c_IKW{zCmp0;nfdMz&r<$N(fimK2lz$4 zHs2#}7`n`;274X++XanaSCjct{*p~xYidLEJ7!NztH@r)>zPWPfsNe1rV`YlqsG^K z=4FK6htPe8%83V?Iv(RdfBcJ+4tn%cp5g8a7-uF6ZjJt9IHVXv-^m)6Pm!9ojC)Jz zH*=-CuZgml)*vqqI>qrgS#Y4W;!@Vv6D~D(z!y@kh3iF++4)8AR^D)ls z5YNf_IqqBx=TPteXXtm&g?4qTuCtkC;D>lIJH~n+f3B#0rtYu7S2~}VuX>fQ%$MP- zMBm2oPCQNG9q8F#dDV5m<(6vlQ|Jeu5r@Ss>9w1B4k&?_{LNDjy3na;qo>fnS6A)T zP4HBJ?<&x1X1@&BnitObjw*TG#lLhlzaqb&%6>fl&q_;yl&PbDQ^J&}4A{+bc=AGFA$MV;!s(!`Y( z9fUts0yWR(34^DJ-w>X2WIpZt$_v$v9`lOxc-!a7PONc1gVHm)uRQmWRsX)Ckd>2Y z+JtYG6=d0S@aOD;ocSDjOzS+M7IgUGDXTj3!5$C$ zpI>m^@*Mqj`j?9iHne@!-)y4~Py4*PB|u-y=NyoQemm{+vX%MDe9i$`{Jwzx;e+-$ zp!s|K?G90>GW*%I@9N*ncLgXf8h;L`<{YHmt3L-cz&-!$_h|nX-vKCl^pCu1|8>r3 zE^@+iMxAp&?uvuxKc&dao&(bJ{VnC|D*Z5D&U=REfEI8Ls88e0@)p&bZ9H&ZX{$9B zA71!9p*a_w`G)6O<$6avw0L8ONw+a!^!BAao@*qJRdT8CXL$4xfL=uG692UaWkVQH!zfo0>f z^KcZK5Rw#glogS^yRRi#BDNeC*n;jxbQf}Lvtyl1$srCK6Zd|;x9)wfsa6L$GiUgx zsIL3-`}_O7-}}8kcOE<{?k;jA!M6tph+E zXYMf(PhZbC0esh2<$e_TU@{K#bgnIB<$Z4GaOATk#xru9b?rT0W*z&R_Z?QeZ+GqK z8uCl-Z!u1x8@&0F=LOzEk#~$2@_lWXT}^!k)@$#_?-4$^zK{25Yn*R5(9Z~Qqi*QO zmHTPX5BN=K`r-Qq^`n6k&~@BD{Lp#iW8s(FPe!cw`oTUYVw@WXgAcvV{A%3O_Zc!T zvOnlA<$B~-aLyv@squZS%C5%9cG2T6H2C|DE8vNJ-atR#m67*6>cquuKY4dT`t?gc z@YLJHlcKce`6^^=_1aSIm(!+|ACD)K4<= zn11STgM1X;jhA~C&`-hTK3kdlFtVHXA5$^$UZ!&&jkFHQ~Wntjtvf}YKeT;uFb)L8uo_dGml8!Sol`au8Vy{h-3b%e)(fUKYHfM=hRGf+zTZ`d2?1*u#GA zWqkO`;@^&M@z1$%b~W)=tDn{3QuDs5!GFdrfq(3Oga3@+|IxX08U2R%H$N?1wrM`* z+}qHP`E)5sGm53<77@S7U^5^*c^&`r+;2S%Kj&($FH>7bw4TlhP8Tsb)Msn@IM8KLVs zbeTS>#D0|b44#8t)V(hJmIdD69e#IQ7y83~6TFw9L%9z+0bYzBqQ>c7;r^(m6UK9! zd}Qx~D!C8(XhwOPEPbl~aK0f@R`eO`z9rP}hvnBpKe8?(&ussWLOHjF`;FbgPgT}; z3;Y3pWu!gyWBH?7))!(Y7S!c|XXfJ9f8soFDad>(t9tgo^ojEo_+z&H6W<8X9}HQC zOVMi&ofrM{NRfOzwwG^^FUR_s)#}S}_z(IlYI&L16INb!^b7x@w*+pU=&z=}v~aWX zu2b%FPY~aV`P@Gw@Tbx~`OEOtEk*slR$BNrzz?_gA7gz2pYMVHg}wrIcS{+$z`2d# zb1Ppvq&?$ikwXIyi9PBb7P~Fhg9#rVJ`a9d`GYVv{K3DjD)tNaU%FXO9lOrZL0c9y7dVGDJn_lcv z_kv-@Jm`bE^xeCvp93C_ywg`E&v>b-ubd#vq&!_mAjZe;T5(?lyw!)4GMt$J?lRBRj=st>DfZ#KQ;oQJ>A1< zRJgQjSbc_ZsiUDJKZOMTLc>GP6&m+lDnswx!@(%}E8R659Ap&Ru2G-`>*DjiOYk-A zqR|HJyN3OP{N1*Dtq|oUZpr68mja>d;reKm{Hm^D?;!iI?T!}04ms!G^XD%qe(xS` zj6%2Q-LHs#q`iAElKJ5WvD<8&yLjKC-#E7@0LLS|9@4V!thRf=hq33g^HSGjnm43E`CjTFE{>d_j{KDN-}}z05V}^IfGhh*@Kmdf30{Ug@C1Cv8rokPxRvqC z`17)#5VS`gs5eI97a)X?45UY5 zcLfL0FDYpk@Obl5_?Yqdyzi1iuPhmiMm^eJzso;Zl>Ic@y|xrS_S1^bdoC&Vw`8zB zT1L)azsoyVWZkCS(WUTlgfFlZJ`VVO$zWsD)APCqi!yIXy#V}x>xh2mk8=Sz2Ovx{ zrR_!Y_ms0sRoI`L9~>(~_c0yT-Z;-XiPL{1lzjx`=~URSrOO`hFe39b@gQ{R4JY*7 zr=V;0JBKc3&ilaMj$`Rl#sNOp8YiJo|ANHZ;&CjU$~efyYmQ^*xAZFgpo4|_#rtj1 zBbE;x9Y2R}sI&BJ4gV2dcr>su_6ab4_L z=|7CVU$#g0xBWKcg2w9={Njyg)qw`{Z>iU$UU#?um4Ia``~Fz$-)_06?>AvD=s0Wl z@I@W3Ra=P%i@#~@=q(epW8U77b{uQV`>i$dD~8xC-JnX_YXQCQ}+^dpt%kZ5Uw;>^roQrWqQA+vQBpU`lYT{hA1kvi^!}1v+Rk)((N`j$`q5X+AL}dE zE1YRSkG$IzmYMH^*fkM(yo=FS6}Hc?W^3ZFh7r`U*Pa^XD(QjMLNC5PO$)?m;E<`bzR{! zRm%E4YhgNquFc;aX9s`By(DgXtbzX<>npcsnsr%yrN%=1E9&bTyVcf3Bi;plB=E<6 zS~~4tWk2;!hv4DKs^-TVr9X6I`*%wJ;raa&{iK#m3qBr$|CoQJ^e-p<^V0uhVt3g2 z6Z?twVbZ=!+Mk%){_68-%$R?1rpkPV9<9B)G8g@eGtl>q^6Ab!xAXg_UCtFGZV-S+ zetxts9A9z&F>r-|56Q}T54|*u=-$UFJdgffR3jN3Dam!91CpiB! z@5(!J-c$lh8+Zr4PujW2TkiLoxI6cLh*#jxnm7dK<|&B}6?DBnmyvcQ zzW4j7)elO%^k(Bd-RAshY3Iqf zv?HFgCFd2-gus{j`1{!J?AjqE_sdqh(fTt*_T5$D+z@zn6Bnux-&7w_a(~R$-^vt5 zQmapjKdOq;NH^65AJ1uT-;KBJJIG&@b6Du3{8<-zmAnj(ekLy?GvG&I9l4w@209PY z%YK@?i@u+zveZj!&EHUxzajHe4ks+LUvr+2 z#6gNf(<9i2hCbD9m$(CTo7;*)4@>x zupYs0f%=Cck8E8bGj3Yq0wv^ndJlOc$n8GP`Pg?^&eeR{J5_(WNIuuvYE-S0#wG9J zfz5cld&|Vv(1Ra@Zs2R)O(i}7pGrKj%y&%(T25`BE;IeT&`ksSHSI0k*!IxPo2&5~ zhjI^>&M+_XBX}Bot{~3;Tm|}#?~_OOO*4)?|5g-Skaq(+3$@gm2PLjxL!Ou9J=k$?%n!U@1;^>=@UcM(hIAOD`F{vQHACE#g452J^w*i~Ps z;iq87s;|KJ$RBNIi5-i4z>c!#iME}+pP!L-W$eo#;IXv!GV#qTAi=nd!`Qlwve|_Wxlux=f`2&;eRV4ASfN)F2QRE= z-Hc=Ds#DfompFJOwfe}MIJ?1rx=>~w>N?3dBLe>#@)>_!;?dY&UFr?+1R2OU;EQv! zHwPvpKS=$q@TIMP#(Q9X6?%QCr>5mk7lCN$=L{Cf2jCu@6a30FaxJcBvIzV^_o*JA zI16z)wGH_RA87j$x>y{(x%$D3>V3pN{sj2!Q{L}RM~>>vbML`t|E1jiaTmPY1im8T z88^#%Wq(b4S^h{DLM`{zx{}T_%aAwrLLlp4zf~n0)aPFXk>lW7Fm-eUy`cF~_?G_I z4H;=4fbUy8eXeNXZ1s$_tElJjrqrRe@})!ivEGI7lblmD@niG6EFECyrPh$om6vgl zGb-kzCVng*CFc>$zf0>KjUO-0KLB4v@o$^-0{|YJu92@Kbmv3&HgZBW(a)%tx}g)~k-QHe{%P0p zx6~2nu50-N>N)66-2h(d$jN20PvCCwV(HN0C800%o6{xH=f{{w>UywFda0YBXCFN! z`90v*(sP^6TVcMn;1_y!`f9{kA&2D;1X5?Hy`TH*vL32kOZ+J<<26{v0^_A+JP*6w zjA!4Oat7IFiFI_jzzcHfW*xxG+8y@26-W1L7kWP|`FWPk)eR+$7Yhe_zGC5ke0g)= zqSOm(@?%V=H!&Z0wfM2~Svqxuu6MDnL05&q+39n2oY~R)=N_EdztboD3mhcR(uMxI zfT!@A@Qc99$Op?WU3y&F-5~TtoX_CT$|JjO^xT^RgR*WHzSjF5e}eY095it57W%3H zSCijj>9I@b5j_y|vB~4I`<;L8!+glUq1<2ZJL8g9|A;>(dA2r>VpZwo;7ovgar>O{ zDsYMOY@=Yh2Hv6Xuz^08daxrY`P6!ydX3Qwz)y8WdcRhVSo_zmH?QT0*ok+LPr>+$ zyrvxKVSmW$caS4L9N3kP^F$>tZ=VmI#126Z0KXruLa$iAB(clXC#Q>yzkvNUluJK+ zrX;_cJStrR}veq}Lv>{arMdqVZBOPnlE z+^mW`+wFS?YRKv3YgJf7UMTz-=#@Oe1JfSywI1@+i}+19y6V{q^Y#6L_E(ZR7tC9f z_CD>wqrKPdmV+vM7}wQuiGM;Lg}#!1BEnwn#Xb#?H%{$8PS=S$>-qx7QR))P`wnTz zPglrA88>9!MJ0bMl6YA2T*l7Vr~KcaX4UYyFLedy{603#{$;-Bis&^n9`yx&f4WBd z1;#75v=8wcZC$Ni*&q4J(yuIjAnQ@AC+{s6KQXS89oNO;UbiQ92z17J@%s)=0|#qQ zy`;hj={!T3(yzsMwN!;O;j`A`2`PI}z$Bv9}Z>pno zOzJkNIrkqPhwoVT$ItIgssE)AWG{v(?gP1z=o9?$XQu=3-`=cGZvC7{+wqcDQeUa_ zgA3Hdl5q`v*5!BL%kN8dT&CrB+is!HQ0^nS0?+W_Y2=~mWPE{}8K3t8i7SiU()_(^ zfcEmcY1bX_J9KG(@0Q=|$?pZl`_hr<@n-*m{9Z|ZR{_7%-u|xS_Y!b0<2w<*2LiWd ze@D-s{GRMT{Im1Wi)z`wh4;2x@U>H?K+wE#1UG3`9` z>LbnjENSsuMGkRKajXHq3>2fVN*pci2GvOBGxz?iJTdX$EA7Yu=((QdK6Avl?Vf)W zVp*}<^$wK4Z!h=i>cY>}KNh}BWxd+tCCL|Q+Bw$$=~Kaj zj2lV3(rJgk8sOL56L*}(^V0$It_5D~%f{!_2G&y(JX{S&Mn8dUKWgN677U z)h2wFe;7M}{q}vGcsJ)?>QlSqK4wnvQ`Yok>MHD^J`np_3tYipR_ZAD{BG(ZWHvn- z-E~Bt+xUm-0+*3j&$wh=+>AVQ<$h^X9?Ja|{Z9DuAE4jpZ{}6$s894=PV(4CSkKyK zU0CO4>NzOvwe&IWF^T;r_n27RQ=zWw+P&#y?XF>cUQs+0wyTvgZw>RM?_+OC9-*CA z>V{Q|LMJ~q_ovJ~CPF8ni{6;G-<+-^;cQ^Pu&j@c&FA_aEBsmJfi>I4)-7MjRItxlx3kTIGh7ALg^~Cq?32 zZ7OKzzEYSKIpRMTAhvm5>=1BJ>9W?}YIj}7v+O-^)q|X>V}A`O{1*68=96V#bG}A1 zzSc5dLC+`rMf^0BdAV@pX}(G~MzD9&tgFg?V*VRBJ?(>M6K{c&!8`cbT`G=MgkLJB zMGv(LKYHL@KB1%TUY9rx?Q-Cc!CqQYGVwO`?P(AC>=k=N@=KyK({(;&+{T=HP9krF zzT!L)=$dufIK9mi$x5C`k#;6eB>l-fi|%@hJYu$1ciCs|7n=0D6h1B!KbCm|;`fd} zIbD+PesDD8Gv6P`{arN(9;#Y?{L)MvFs$QVV3pt3HPo|p;xMKi>nGoX-yQFlW&+|h zE$xPLjqjYP;lBX~iR0M%(oSzF3V|Qz3w4RRbG}fQbExEAkUu@mer#U8J@0GVU*voN zdiivb^93dLw?1FsJ*m{HZGwkrM(m{p`#nWJ_SM>tdoG$Uut&Z(4gTyo0iTWLy{qUu zXV5!qa(sHsFL;6*=B;b~__*;QANYi2&X{+n~^$7gDC9&g|Ki2mzz5xOwv{j4Ddc@&88b1)G1HyuUO|9F2TY zIWLchi}rFJ*&wbWaZJ|JHdM;3eo*Y!=pVHGXyLH_`?Uk1+&i=IAb+Tw=6vVxet4!L z`T2jZ4ltj(gFFQAW$ltq>QXn)dAz?9d$`-%7U;UIW}j+*gL_xpFTCTjKHstXXWi!B zRq^lSe8;=va`W8hj?4Po$I@ALbD8)k@IS0Lujby&9hZq?08a~_ZI^X?P~9o_u7KNS zKjs~pKFcWa%xy2veipUUhvb$h)TH?{p_eKO8g`V9+R zUasnTxUvu6XY;($($3|(ZU=uQ`^e5Tz)KN$M*zU_Gczvxw0e0Hc^Se#M*qa0HjZHU zGq;(04ps6=<^CD;l#z2jpE#q*bFq2E+*8e(xK+=L;vULo-~;?tZmxEm4VlN_#lp2s z-%FEykzeAXw`ANA;)W)_#Gc=ENq+J!9S_sv7UVrOc2QUcz51LZnf#G1fpZyZx|4hq z;=2jng07hwaCvV-)2D^AgXhI_pZ*U&US`?k(@tXC(gGzz!y!cQgR6F&_kLXX?o1 z_ch;biQy~fqa#UPkj>`?zT9JEopFD4!wjoT=teD_K`vf%-82z@CYKg^0p6Cb%zp4& zm(BS6-L#ASjXCeb$UUVlIgf*$?Ri|6oW~(wx)k~Q=;_!$vG@Jfb3R=oJ|g*t$PG`< zr;*Er4czy72KgcQ1X^u3)2^Jfv**}_4e|gvnA3kY!_4@@X zQ`PIU^TzaQ=Iw0f9^j+MJA)r}7Tx}AG~#52|5I(m^f-51!8 zPrgNOoqc2XnsK{jKOfP0QST>NKjVUrc>QMFtlq!WU;3q;XPF1Pqakuzk<58aNS-C~E)8sji<9x-oQU_xI1GzgRv9J-s8{Ax9nr zj@FMJNA6U(C#UHYyv(Q5+?rk3)tg_w^DN7J7xb#-j~AZ<1w%76vAd0YHgsX>%F+e$ zIX9rsov@1{=-JRiUg))o-ZpuNwtZ(m_i`qHW5&o$L$7xI7QY2hJMfmoSNuC?z<)xo z>W&$g?(!a`NB)jkm!%Io5BL6a1K4--=~Xr5wVBuQpPg5=b5G|4#1PY~tLf0vtJCX} z{|NsXx^mjNr$c;Q9WH9Rw)ATCv>mri??>pWxt|2CO?r*tYQ}|M`v#6l+|ez~c=*GH zUgzNFeKYVi0ahLVv+z_|&W);xJo4nc3AsXj3)Tt!#Clu(tft?E^tRO-5;qX}YUGgR zb1R4J{?fAEghXB#Y>pA$lSIhl)9sUPquoJJT2h2L=4Qou_8;IbTkI&py_3IHVnM8zX-#e(bmje%^V%37q{8&WvEE zECgq(&n&#nI<3Ad3qK0{9aR#0!_vWU?(Usu(TnkUkM#2pUZ~1d59oHy_(=@s;1_3# zEMtLn4CfyHpUzg~_p9GEmvw~jb!&X3kl%{mK)$=!57=3W{Xkvt2J)sC`+@m}bJ4G! zb;(=HMgQt-&?f%#zd8$k693NMC3dm(udJWiz2D^7eVRNw^qTd9yTyOj_tm3cnJGih zhW>58d3oQ)?dhi|m=U+V=X2pO0)w6ZrDdPiwJARh)wI{0h7f+R0 zSK^;%#ov91bPvVe%S@=oquU4hrnDwta zm!hX$OZGFzfBx;W?p*jr|LaT<`bW=6-uyqiZi%sn7p_}U{OC+bT-x~0@A~~>(Rbj% z+5xt1NnYxfU_Z|BpZCn5r!Bq{{~5ale018nCB|-;PtJLux43ReQ~nId6R*)O5 zbxVxBxp3W*;D4X-u%8UwFIcz4==BBbmc(|y|Ct$qgMqvC9}<7-h|b%z__uiud8u2n z2f8)mD93+lrb@lBbkl#1^}pnivTv*ZyL28Yc~Tp&58s&EzxCV^-dGS{sMBc`5tY2mSSvs?D>=u9JBKSSr!2aVL$=D0;V9%}|SSR+})$7eT zgV?cbSKwbJf3w%a&w>8TdN`ND&tZIf-{0WhwzK%BU3T>j79FqKW8In9&$8|9x>JIG z;jfr3gWsR2{S$V|JJvwgs{#9>i ze4d{1=jb&PZ%E`o>D8Gk^kL+Ql?Ok4-IC&eK3gN7QRun`eA5%0s~|@<*Zk*3kgM0` zH|{;np_6gw3VB=9er*yzvhdEgbML8-xKDElig~G2U5S6muxX3M7|;<>=51^=`=J`?bFW4~H^)tCJR5TO(O$xM+^ z6Z$$b(?A}|{K#?U|I!Tm=$d`?i{EsK|0eT)QQ}N?{;qT_2nAl(u#*d%CrUn_jZ0(& zP66X&O7G|&WCgAv@@}4ga7})vevHKR8Nb1*7w{80W&9fXN3kCV`~|KtJev4O;E}G* zKzCiUANaFx)`&;M@Cg5O2E6>V@F)u&L=QOrshN`aWu;RyA@+vxBjY&KGW4$xd=0%> z{mVIV-@rEHr`8X|r=?w#=m*;_BXvtIvc3iMpoKH~smU+zmak$s`7a3F+PsXO)ZKp} z{NFMAdndl>OJ1L)+xqh}0pASWcBDh-7W`j>Zj_amTj)lG=Hg^m32t8OnxB90b z>%zVu4mLROdGXI`uguhSysD`O6Fz)p#$~(~{;>7LR0g>NUs`==>A>!1emtb9=d4{} z^_*RQLE<6Aonkq4xcH87kvVc|aNsQ+kJIa(Pp8+^k9`B5C*C1?6#6b`ybuSSr$@m{ z4*IyZ9_8LEaF6N2({`bi>vmkbU-W2h4fz=fUq|1ZY3TeyeeT|qdik$J&ZcJD|LU9W zTsd3*hTzN6Ybx#GXCvpYL9cCkpU8Vde|9|;Q#7y{}AMU-sn@ zPn9^qN0Rg2IqUzH*x3u&r_@3D_F0$lTJ*s}aRM8!O!Bx&^)cueZqK}cw3i_5jhXN+uvo-93pSb1G29H-PB_E@jQS~L{FIIu)l2ZLpiT>tZQK#gK zRggzL<@#7e{Ba;84}*NFzH(5lA`kjy9{j%BUgcgK@V9s-ZY1?)`+ia`k0LjMewl~- zv1Qn~%#&SKt{)0%-zWJ7jK904^HpazQny3&BXvKe&W7A;3LC^f@?S=8$$h2j%nbYK zTU(C8h;`=33mRcklOIL;V<6|TX(yl^dT=v&7}!5fk-P_A?lt*u%K4+!6Y;$!p%eB6 zz1)4tSJc7eeU*Um4gENqBHnxBeCJm3E%;qM71b(5FYo$&Gb=-d-=X_l*`DPV4uZ_Mt6VZRn)pq8Xdsmbd$I91;lbU%5#r{C_Usqo8)#Q9G`u>apA5YSszuzeRE8y!|{d4q({^S&$jwqC?Cfd}xH{Q%$j;|G7M@fyR!Z_I%2xQ>bRuYi_o^^f7<|8OQ?{>N&- zgE}VCpY>d;e+&<4@BLHlTj2qHj6k28gg)eav+|=E;NMlO1uqEy#dL8|=QVW7c_{Xf zvlY7d2!10=75d{m>c=y{r%lEg5xNR4OMXrahb8ncau3Y(@6!FbKi<+mCH z4_(D{L4Us6{tL|?!$IZ?lJ>3hr8m{}d(P_iy3V)Yx<}CPp_8rTt%mT?9SvP~L{Yx8 z`HpinJ3(HWrK7HGHUHU?Zm<3H^bqTIp%cef6B}t};kIl-!_C%BaE1hKoD;_T4l)yO zBWI+49lViWF!8p~kKrfj58STRKPCNz&Ky5I@%9|JNq^vWt^NoLO>ZltJ#bs7y;9#J z&l0(JD|Kc>PTD#x@2%_ihq{JqzTE3d>JJ2qCr&{J>|64AW;Y_QOMJf-xFeU+TRr3{ z<5_rZImtORaTwtj@X#uMq)wH{ZEq{`Re(5Fr%n#{<#l|G{coo30P-NUHK$hIpTPOn zV!fw!9Jt1H2gtJl-m{yiI}k9>t<)VrpE(;(s?Ye?9YxNELh!rE)Vtc$tanAbU=+O5 zE;tCE+jfmjrry=2X1y!gMIwJ`=N}Bn`!M6Zwu!m}$Xh;jy(`+)nhgO3*b|YiCs$F3{&T+OX~^fSM*pu z(YqFI>9r!SInNb+UgTWN@mEb87WkQQL?4#rckl>*roiLMV=BhuEAK5YKZe~E>$jHn zt$0VSe_&!K`cm{}L(Y-aO%waDBL>OuhHm3|jh;-_ztFs9o$t`zTX&T{F!QqC2)ioQ zPyX78odP%HRgpRZPDKv551nSTFgvOzdRcPqGf=#EsIw z$o@?Kl=R;=zki~a{0$Rh$UUo{rN7t{rhk{7KfxRK4)px7e3kad1KYlJy~#dnw@&Ou z+Qs;?_`Q0+vR;oo>tw%bc;apR+IYXw=803z96fyXer3HL=UuJ+ll?|pCKjnVdid)7 z%KE*B=FhK2S#Om%MazC$*PGsio#A3%C;IdDiH+!o*#5e#>DiO~E%4lRC-`|Fkq5x{ zPQBkme{R$IH`br@uRvF(zoYxlrL$)LIr{Sst$!br^^5&Q|9a9tCH;@h?;qP8(jL09 z^R=#bp5DA2y$Qc2`>)(Jv8Zhh-VL1_e({8_frF#v^W1eb`82U3g1aY4`l z*#06H-ihB5I0WyX*b5#kolF0E(tja25!1P}_mcLl>unQxG9vOT__>M0(5t-f!S}ZO zj{~` zkHX|d^9&95BdJp+dKY@n>-uBx{dgTYNmygc1m{iU38Qz!zD?>~QGZP03~~K2^5Gir zqwuXuyJAD%k57qw0}rz90nSyrbRHynSK5_`E6!_2{Nh2$cZ|Zi~1=t}{r^V*| z#&yTye5~MCCJwW1d#<*DJX9a}nS3cHe~x@d&Q-R(5$L$W5cf2(r_zF7_SJ>oeH44S z%OQ^B@+tOgN$e%gV?yXr_R$~?$3ApjG-H>larmT|)H|EsUgAUI@5;E?C3;-3Q=8*< z$+#u-w$wYz!OzmJgncXRT;fRc=O^9;y{tdc5PwRI3!j6hEbVI&HxT=Tx(+gK2ppyE z*&e_WulJqtYWLSYRLrf(VSh#3V;A_@{h9IYdTjkS;6eP6^P2T1>T>_u(yQ3tjVf`u zP0;&SkmrWpll(dL>l542rwiZ<`MhreySq#7ZvbEF4T3lL%HR$De4w@+|I^@4;wHN8 zW5S;g)HbGsKOd+ekL>xq<wA>p_OI{uHn3nk!@nysJDJ>t6 z16w&4_`HUXroXt3ptSel|Dvz`AKN~?6}pyqs*U?gJNPc+@@K z9$&nj_Q)UG-jnw5Vb@mpYLEE)MotJEeCWl<2`i_%Y#qT^PDKB1Vg>72fbJHecgs)E z`*o|2&^^YN;Xi{1%YT-i9O0*v$PMAAK=|pM`PIrv=yPxYKR@9okGL)Vpy4O%tK2H$ zYV-LCenDR?xGy6I6F<7?ho?8;N0;bl?c((!_wXk*9bqr9&$)32iDSrp$&H&!pQ?>Z zeRllmDs(g!RV(<313Dj-^{(5a9uK(BxCuY(<;32we)Q}H{O1bo)|$ zbB(%v8=7_dXjdQgp~G8Fcn4X(ZFh7-*rDt8ZD`i*qg`VZsP?J)lVTsz-aS}t&i_IL zKH0aN`zQN8fjsB@7kIBcme|J@ZfWX0y-?!b17T{phYCNAO+Z1z@k?I0P5dyymzlSveV+E-r_fiXJ#p9PCiY@4#eSH7U}A;%Vcesu zVb4$(QqQaFvBAGGuZx_JdGAT=^JLx{`UbzVrG1yod*z+xWxsXw+6IX;A;+EIGdiwA z{H5uKoxIBL?ACr~5`V#u{JhxJZPLFidNlZ~mRnCY$XiLpabxSZ7Nq|h^ZTcyzpMKn znm7f0#qmMruLB?B_lmu_@`1VgXwIMbCA9aF_N{($vQPIvi5;52t=OcK7QeYo>}k$n z5`6j3PAtM+jrXe#i=C>T^f}++UY7I+j&{9i=|6|g$QRW8ll`hcm_QGz$IAK~&c?}Y z|E%<1p3s-=uSP@Neud08XTKe~eL@%M4N>p{beZ^{!58&tE}b^{bdLXB75Y)JABJ2&F53Pn>A!7${{#-vf1cQhTzL$-V_z#}e!+ve$I>SK z@0s6UjY4Nd>`Q5{lJ;q7j~(Z!rW{g1zW8jN{X6Qr6WBG4O$3%aktfxvem5YSqs}bz zW1Bu6@ws7-v+`0hUlQkaUlRPsb!KBdwUj!_*tc2qRD?aeNpX%R@!nEXUajxb_PLxB zN^_7M{+$$M5G6nta1 zwWF`ldn$WUjaJ~tOrQFU!X8W4)zN^zQ};shTKK&066@^Fo(x9ei|$NcAo{|#S0LS0KR)s$4l(r9Wv9}1pGwg$q0UG#Q)mM zt@QR?iKk`c{0q9~{Nd!pM&S4)az~LTF8zy~!UU+~2Z;v_$ay{dXY*k?W&Er4WeYr_n)n^+N#vNv`Ka_q4%+_R(*G;-*O!Um z5uKjcDRRF7zrY*PpLN^*1?i6+$k~_z=`#7&X8yeNUp{wV&Hic>xPv$D9q9Zz z+rC@cA4}SEj!!&F;z_`7Bm7U@ma#H&(bOgCN=>MWho2hPZLZBsoKoDUEiae%?&G@I zhCXawi4u6{lX)z>llKLp3qo(nc^~+KK5YMm@RihUj=rPMC1d?Sf9S*ZZ`1TKfBr;2 z&>s4*?bFi!Dt&ZyJr3(H#rYZC&3ZmIKVvA@c&j#!y({%H;6v5ktm|X*9=Jd9R_!?N zOv)#IM{<9{|1T5c(4XX2HPUf@OM1Ugoo4LWPy5`{0FT)|UH>0CO|5!E;%?CoBpz?? zS)#&pUQApU`YGx{SMkG4KlDp#)iM08EfXegSN~M zJ9FXs&h$tA`@p*bpZeQ^umm0Vg`BTqcO4EjpATw#hx|u%j5wh9SNIhb?kz}t=VzGr zu&3<|Ti@B%wM*(Sm#N2zvXppONFI>f^JBfXovrVjlKRdkB;Shq&cH+M>!F@uIlr3>qf05dH<6b5&a68v;}(%K)Zvo&W}D=pBhQ?qzVmGDIO~n;J1bgq)rJ9xayuGJq+ZWUo{M|&(a_=!E_tl^eUEg_r z9jlTr_h4-us|@l+-oKUnr?INYTO3P)Ll1dY5#I+QZ8zm5&#K7$CeO;|2`TO!$-ST} zOR1OTDaor+&o$WB(z1WF2%qgPc?U+2)4U@j_jKmhhiPOcH!^NNbneSKn(s<=rHjxX z^wHal*QcH6I^Axw)M=m{rH?1a>1X3GL&R$%CyA>FKcg?>e5a+1KO*aD)-Upwkne<$ z<-56`RchutNqw1!zdu!~RjRa?`Z908SIi{!WnRppSC}W`xXUNE^GEZ3(u-O87pXV8 zgx^Ex+X*@+4~v{oOD5k&jtE>!v^R1q&R-LG74df?uT)DNA^W}WCm#80PFznYsi#nM zCl8ZnX6g%(he@6prGI<2yAOQ#FMvsBja$uGS+A21{N zfMxo}bXM%2+y@wYSr772;41I9+j!c$g)jMF?1Q@F2F~5G{u=u|2EWYXvjx{* zEssLaLCg9(W&JX4vi=p5=mWcc2mI(d;0BKGcKvqUv$w5Ll3$WqzFgzEy}iMEJ?NkJ zE)AY8Bfr{(5GW+?aamQ#lj zsvB8%1HIZ#9W3Y)M%8s-Ah<1)7vy{gx@*XKsAJwBAE2E&=7ZD?iuG&LUYp&vR6Q;G z&6aBWilUF3vffN!G9Kz&`N6&)&B&#fdL%D+XbJ-zo0^As4#E z{;UwEGj*}s1V6};bbCNuYb^(Fo7{;XDs>`&w{36h_1gAg=d3=F*i{zpnf7{0^1`vZ zwoGEj#rMGsUb5{`YRxeB{DqDJp`&QWZ?lI=3l5H2<}!m(~;i7C$A8ALL`7q^SewOMSuU&dGhOH|>!R%=wt z7Spx(A!Y22rD4zND=F0**D){eo?Jxz;dRt8FH!e6t@e+XWSk@VoHZr<50cdbL}x19%%cv2al9s^Fbv2p;Mw^->nO>H=?}5B_T9 zZaZ@8*+AkSIphVOONc*~pflhiaY<>TD zUuK<_K5SgW(ucFI9#yNw@ z?s;1&mmc@TdlPve>x8a_Kg92FrQTUn-sZ*c@Sy7rtn;L-^DSS_|D9p>U!)G=)@mg6 zOWQY;(Sw{<4WnNJ^kdq?9+AA}i^yl>kMJAo>sD{+^L0yq-Likq?X%)fu->e+3+ZS4 z3A>MkPD}7L>iiL3*D+7%akvR*LyzROV7C!}6*%zOCiIB>wDsy2qDL!-Ej zJJZVAvuUvhJ;pKiU@!N#qat}2CD$1r;qMxM_$kOKsUzCnZt>yyN02|MbR#&5T#3(_ z<-7~MX7!YDY5ySnBy6q772_I{F06sE$Dn00PADO%rqo>kKSs!t!t~7pKjdLhR{r03IdcE@7lVj*9 zvrfUM)Cop!rN?P!>7CERkAV-n58k7B=+wAUt4a_5?kVE8y}j&*^{HOymj22Kes}UP z!0$!xRv^cvrKsv7ua;K*BjBks9deF|KOu6;M~({~L$NMiLw*v!GW=uZRJYXCcZu@~ z-YfX)(w_C%_X(_A1MjJU<;l5#GI<`NLd^ZC)GFx0#)A@jJ^F<9Hx}^Quv`EADfkfi zKF@y(U8(nRb=+*@{TKF~T7kWKBlS#?i`L&&dHk6o^u)d2=Zg$y>T46PRL?*+s+W4& zF7}71r)~fK&-G(x+d1DE$-vj1dM;8biAyM5XJ6%z+a7TXTVI=c+M=IrT;i+ZceBn8 zCwOFX9KP@EXZ@+9zQ4qK>8~7hWby+0T239eE7hbf6!9M;uox#8s}$!iUg*Ux++`{?~>63Gm+`aeUy%y_HW*5)aErU4DL7 zY2sKRA0}Q)o$pUiF5>Suguc|>x98|~KKz!-ro5RRrIXThe{W9|2Cm3=(0=b%11Bt)%_zL--WK~_#1+MCG=SOPm`PnY2hLRYlkozUr=_IfOWoV~bakXz&pD>6@)sxH#$GgZ)xmoVPXxee!Tu-9 zkB98f&{baA11C#Yo&7uB{@G!r-e?;*a)CHYE8W;Q+Hz1?_nF7YyY@$1^*N9o%em}> zyi?C}Tla7O`0FA^Vm-pU5JCqF$VEr&AQyW2XV&kw^qbIY^e2=1((~yumUI69J-Lr_ zWw~HELj12oKT*boPkF!O%adckudD2j)o3sMBI(zh5C7~h zCeiQeCVnp=ACupom-Wkh&w*Dv9{$Z=Xue%wyi}g{5q7`Ov#h_5b@*B6nSNu4f5p6V zp1h!(JJGx^8`E?BE0bfu)6jFL$UX3~aQ}&`(cx%7OezyIy0l$}%-?Q>Ndd~jd5%K$owv+YxI`r@0 z-OQitzx?`48Tr<-f9OQN7t^KQ<1|FirQ}>v{E=E9dP^#%h}{Ex?>~P^&SlWAteQB4mm-s*TSATf{Irx+Jz`yP4J-job&rdp=_v_>x6T$0X`mbxJ_A>vewjp)~ucg7A)sRuGw>dY(S)8Fnpquzr(OCCcE_h#JIx!8OE>)W09Jq0^Y*}v}o zZ|qd+!S{Q$0}a+)V7!R+E#?zDn)h(4hdlh0o*nwU^;bF@*Ijfu&lElbj{L6uL4L0t z0)E&t-<#Z+?8DCUZa)|Ar~h%{%Xf|cwb+fRyr+(MS+(f8rXK;`zBf76rWXCAJv|QJ zjMG5g6|mzkev$r%i8rVAeRuM0;O^2NIZ6M05&M|afB$&!?)%F+BtK@sak8ufx>#ua zvJSBa7TiaN^oKspJa(PqW?Z|!cl!zbym?_G^tr%!#kKxU|G&PlQS*t0_sZ08?NA+l zDunKp26>wDy^?&N&3)?of4MQO26NT#pE}OGD4wvAQad}`jJ$^;d?Nmz^N2h0IP$~! z>-7DvUm!m4$=v@TpTh((#p{LqglVe#mz6AKx#V=iaH&dMEr&jb-m!<& z^Kh@VPwcx!@%dLz(|(T6zr=nH7}wa(*!8?Sgq^T`+mHV8FmXqbN600cw-BFiEbYV} zN1r?X+mnlQy#7bxm$=tM$3FL`m!cnC#&x2ezgz#c z^`0mM^1rh#48d#eqw92@i~7*I zP_dr<#IXbBkH>MZOde+)kCfDyM_gcg?#f?oXTK{p2M-tNpZ}N&!K2FI_aWWd`mswG zr;YmE`W+(kUQMcy201fuU=O67wi9^>|5r&NiXEznQ(6R5omT;lhjidQCg zvi?WPBa-*WJ2b$Dbw9$oUD0E}7kZiV9@8&O9d46%>$IJeaiV{m+%EdaTQjv+*6nM1 z4ZJxGS&z%RQ{WT-a8~42+^!dYH-cSyxyKgLuUFpDanvnKYF`Mkr_!WkOrbwkYTixC4Az4X{9hrDRNKGYOBX%bT~xl*L0PdoYTCcl50LmO z-W@XnUbz?1U|v%PrR`SiI>vQ2RJFg`xxRYj3G|P8s#>dH$7K7DllYH(>JvK5wd5hREg3F94t+Um$sec^XI#SjO)mH``3If- zk9v3YF<$Ty@X7hw(vtszj8EJ~OHRchMB_ zqnM8jpSFp9ukgFchd|D)nmW#VEIq{Ef}e(7B46&oS$fF}y@b&1|8RP#^-t~m?@2H9 zCjPETFAd(oV!tuHT=eg{DD)C6pE?B`4ZREqz0{$*m|lLMn(uBk`APLgP5Qhhy~w*; z(4nE1?*5a-yN1z6KJ>!)IZZFDr?-rJ$i4%;s1;NDVbY`<$cMatMh z9XkGMho|p@SUK0H>VG@Ak@JS$4yZg^bGlHU03R?SADAM zN)i4sep@0>ecB->dUjL}v48A_;PDao!RS%)nmAvObBX+)N$hie7W$5aIBHz=U9(Druyp@CR8lULRP{sJ{(D-Eh7JOR0c`bZ)2tEVRD@}Un=<7sp zLKhmJ{Cz=ux_3_z5BTqgPvl@Z{Vwo{9MpQu;uATzNAMZlJ%ycb@i`#)EQ5y_pH^=+ z<4-Dw9AuyK@hR~@?0SRGdvw$4D%y%DrMk0Tk_{?d1;#c)xk108)7XO_4 zHV&UI`dQ=xas&CG^fJ<-=RDEce~yj1#XN)Puk16Axe?CJ%Ji;*pMv9`{HjF2LVc zEZgOazQmp`%6pRN)BC6Bx6ruP`upP4C-0xyh}`SNUuS*2i}3gBLMP$4g6czFYAzV)&=(= z{=Vo{)8Ec#*ZXqI__p1(K9S77_U0ILvjAOO>+kfx|4sbuSvUXDufZ>|AK!x?j~*X% z-H)DLgnT#ty!)_nqB7q*CC0`q!4bmgQ}`~FA2^fmCb{ulB0YiV`3v(#}h#H?BT zVf=mJgVF;+2QQ1i4?oA>#ovd1o_0xo$p$;PTR@9Gj&SxrkZlu_Uq>S{0aE1`zhY@VBfYMb^7mG z_4(%g40#6udj7=JPS*QKwf@-v|9gF3<1y&Oc{KQUr}2xN&+~a0I+O7u>>aZ%wO_F= z=;l1Mg1i_|S;N zTX>IozpQh1{ZLKvjl2D?VE^&Yy!U9|i^v4Pv&=cTeOF1wuQTqtI&nzYRq6rMz~7Lc zTYZH5QR9yi#~7V@8+%vq0sXc4e6qea`>uf0Cj-uzbtQGQj9()6tl4j76YpB?(eyGl zwHLa#a9Dq~b^tm_;L)s8FmU&uKX!Vz{Qdjt0P9k>S8Mw!_&sLctmJhuFZ%b1spI5- zdE0`p41NE`PPH$9Z}!*AV-@Zd-hQ#se3zs7cj9OkPEUwkY}Pd{4d zJbn+o$FDu(l1JHt9RoeTryscqJtlF|rreC=1bu{oxdHH!XRPz&&l+iy^A>AWeJX$FB?w zYx`TbPv$LsVQM#eXBqZzP39eD96rrBHjmKG+qPIu3_Lh@-d#Gc)wahEB<~QtV&=^( zR%Zr=#a?T!b473Z4^I<+vipSJ{%~qzyIPmpQ9Cpu^n_ho$DT6lxkdi2lE15K*rBog z&~gr7`^D!@Mz8&0XSL%ZbQzy7^$dH58afW($~_9!{k+5lEIs_RpNsdu^t|}5a?Wz~ zxl;XkkuwXOE7^G#zexXm?{uyd_uv0c=SmCpx92^{xl+qGE$2V2>z{wF6!&kQkC}5N zJHMR|JnQ}bH=Hm1`J0Q7KMUA<*ZMp5*Yj_l!oJD(IZ;W>$>7B4?b?s{i>bHCkIQ}P zFTTd-d+-mRYrt?V=dUY#_2sXfA|LSXlP`SjIP_rrBYVDP;Teg);L^^2%>DWb=+>Oq zCiYy>f3Czigtq4xhXP4s@?GA0_mOw`Vxaw|0`?qw!k+W4e}A!YpwX`Wp%7xf0tfRR zdvb17d2woEdd|6(`5vEJneXwrRp6f6nb?civH#?E3tvY)JpLy1oP1Be_$GGbBK)6m zv9lKA_rd@8tB*>2%-XL>zE@KY$N63&htXr#mcu4bc%B??ygKD1ayTt=7=5VR2Juvp z!|utcMPeWL+D@{c1&Mp1FZuLOh9i@ugRS37jP?ex?s_IrIqK00#Tw9mT{b_)8%w9gNP z&3BHL%Xt)bwNv_*#62^8HUEV=cD2|Sf`_7qBQ#1m!R z^rOBS?;LUdA?G_0d51e52qd0q-_No0DHr`2Vqcm0@B`Q{>k>Pi^j!eqr*5cgyRfi~`=)RuyTU$iygC~`0)3|Urg2+d{^(+!eP9OACjT&pyW@3 z2e%Bo$%ird*4cLIvppmJxBC9nBJgAT9S)oK-E=?hNx32Ykhi8^M?3FQ?ZO`v`?Z>Z zUlj5le=h&+w@x31zxo7D+&3!rzI7VAXHf_C7GRsR&i|t6^d`eEHwzpQ_m*E|9^sev z@Z#!|0!ROC9S^uEYQoX9xAbb^NdEZuR$tcqrQ4Iw^tRCL`cxlrKIp1l;(4;JZ#Pb_ zAU@e4=iGDG<^R{I!vNu?I`}0XVCI>R-|({BA4t}vSQq}iX+Iyo;g43INY=GM)>BN@ zGY7BvbK8*%F&#ud6uX9iX;JPef%m@?{T}j~mUycC{VA6`Vb=S}Jv;CENV~e3q3g(z74O?wd%DJ8Ha=Qc;*}=tIu^7 zun*$?ah^i;AEsD73vT+whQ93jTi>Be-me=5pvX_+@z3e*=AE5*+)Do31qSYy_23VZ zXZmafK6CukIrt=@Gk;e2*7{>A<8zM$`I#+p?;1MAk2qSTE=hh^*WFDQc-Q9{7MUqG z<)YLJC?VGj-`IOX+@H)f?d>3cZV~70q7R@KJAQ|ZUqx@tA0NJ9Lz$5AIlnXGTmG`+ z^Bz@p;IftmvRvSP z>l9$DKkR;MC;Bw@Go$WvhoL9@$hq%~gNI$K9!SOffgUWJL!L++MBaHt^rC;c*VnE- zwCasa+^$vMh@J@}&UeY5ItqWFEJjL<>R~Y4PIbmZk$3eQ&ut{XVS&6pJI~^~=)VvD%f$J`&W%E$|F}PLdVXHtLj7&reZg^B z{jZkw+c^J%`)KvQ*tgWTvg`eC@V{1kXAC{OfPB5y-|@e0{0@G?toz!Ze*HM{nZ>-n zG}jNJZoyT4kghiZ*zKr-?9^;z% zG6Q}T)-l^;K8&(!&8K!(i(^IjcLRE-PCO>|AGlA2ynbZ~egX7_|FHhtUf!Ep91*`~ zzP=x;u#R5qyhX;{%-np%Du^kxUa-5g@2vkgXhMP z)9YAQiMU)Zc>s$0O-``!9QwRd&fR^1BX+J&yx8$Scy2rUG2aX0Kh75qH>_kQZ@0jK=3VKo|{Rd3j%K2wxDKlCVYRsGc0gKzn~ z?-F=RFB=X<`8|EZp5P#BvF#fD)YtRViqHElDe$vwI2s+H{SABkgY3h$d##`PdVWgr zdCw(<-?ePGK8oIRKWfc_V1Sd56$ndMxhhdyF%IrNqeQ|OU~1KH}^Ra`nGd(4gLY2TblGnU9b`T4$ZQ% zlDxqL9`4VbTg18k4b;000ooXGZs=`5>NVBzgLR!I@YG-4?tPwkyn!or^9O|=6S$~f z&~UZ&Q>DL)ooo81r9bsVTKgw(QTLwP%X%Lx>G@YmfB4e&&q#kRCAIcfqh;NGg|rWo z_E~BFM$(>lCIhi2xYuL+@bK6<^lhiyZ)cs?_YxPV@%#Q-b<9ol?(9a+K^ox8yhF9J z_qRUut=;6OZY@S~|B*Ttei$G}w>1L!RO8rfRo=JM{EYnAO#FcUZMwbya$_rXt;uss z>Tg3w6`W&z&H+Dej?hEcv!)Jfek*mYPcTk&C-RAY;~{jwxs(~dV{=G*#x>_69b46o zfny0ha38gCsGMS9#xB{4oGY=PH8t*!@aG}yZx6V)=&Pju%mnyo=~oZ&bpGWl@+v2*+Im)BK!-wxvqb#k;W^*X{2pWBWc;{Ce=W$IXLj-sPw z$*=v2U#+lC69+-AeoW+QTI%ywh?hCxzdnavtZ(HW7jiE@98wQP_*mdg-A8$sz78G+ zk8rMo-?0`xTGuv*UZQ_RPy>Kgr%{i^>>ix=ollBX(*PiDor%69(G_99oKUwtY2!z%z^~oINEh8^DO!T(R zlWSXspF9FTn>;z{lWo82jk$7_@lQ$Iiu#!)+6^}KHt{(v7kUSYTcH=yL(mI!f!zO1 z!9ym155cn<^P6;M@Q@KWme5mz$F~wZA}1d_H-`R@aiCA=?zcq#cgQ#)_Ohr0OY9l7Xt&h12A$T-Lg^%#6mY@_~Y6EAsL5Bg$0A2iofkbdxETz^yg ziTsp)@J}oK_ijV4AScpvll;fDQyD#X960p-y$U_>Vd8zsJDt2AHv<1|RZ$3^$M9Ou zJlF|Jua9xoqOa%VgDLNe$Q$q~@)bR&#_H9IkDfO5?%USo)Ze_7A(E!mwrD#?-BH!^&#o1FhjW9Qcd5|`IegO*^_hm;FE8u5A8M`GJB+th>>cFL&AWn8 z@R7agNN})<8*bV)){4E8QhZ|XAh&Mb6^;6Q-gLx2I6^$aw0mu>*gI*(C-x5G+`Owk zigcr$cW?yzyec-Gr>%{-(%)NK+g&Akg$Ot2loPhx*A-EvIC?p3%qoTZS>C&3( zB~j^{)z#FfEYA=Ug3%q93c>1FR`%5yiVY0SLnA!pYJ`~7gOa$$RF@R5!aa8OH>LZs&r-!h9)t;cd z@coE>lhV4#&{Xc?#dJ=)?$<$B--omKRIR6x=HiBsJB)M{R4x?y~xi9e3^f35qxy-yvH{GsSyF6C#1 z@EwbPbqnh-cB@bPpAtPb*qJ4MXqLd8_`jyEK=MH0AN}^>7VPDXzJG6)eWi1WFN8j< z`_#9e>SH{S|FuQhWpoWve`U44wbAzPLZ7j|tpedsRlS1ziaua;PkMfaic;T&AN7eo zhTV1U!`RdC&&Y2d(|MPm^SO3P;(p~kaAp7g9}n-VtM1>QiQFt{f9dzNKGJwt`}HdO_mQ26 z|1JD(v)`-Yv={E*C)DF^A%`9&`}eeie*(N@Tv0u-Cj@er6%)WhG;sndc zg#~HP&m*7WHgcTuxTJPv#V_anK;(733I<9XUS^I`NkqrX;E3)& zy1$@#GxZslsoyB|@jun}Z|tY$jZ&WY+i1K^Qhs-}ywL;emz3GZ6?`j)`o&W8i!|dt z(+%&pslcQ76S3dF+k#&Xxn|(ne2=T+PZNSqS$-dvz1+FGe%QFcZ|J)Um*AhZp7JU1 zk@acbN`3Iy)E_JL8>{PrT+;Qca0&iJ{IABoY~DX&kFllo*5-{; z9{v-Jw^5I`y3bZ0Cx4z#97OB0KYywl`yrEB-&Z1jhx~I5;J>LEUC`nAz%d7YEu>r#yu?nEd~0TYV;RqkF8Yo3|CQxEe|Q;w zhqmDOQ|J@z@kft8CHyzyKNWwW<`ekwI?8M`|pTxZU^;5^FUlU)GiidGGoo(v3Pc5UJO1y*n8?ig9@eb%Y*jXliyyC|Jun4%Kg?ysBdwu%Iw#k!G3E&=l?PF*Y|-heG+F0 zUe}ImTuuE-yy|G})b6*|jK8v9JK8*I_gig;xDDA~4(I#(@3%hM`0PH`?e|;<^y{W) zTR88~`1pj!+25f?swv7ANdK0{l##lz~Sl?7uvwre|8m@26jL{-)Iq*H78d zTM_)X)cApR+9MuH;(n6j(6f<~Jl%&CbZw{6pP;uhpyMoZw<(SMkSN6{pJo)#(%4MC z&`!!pK27ABYANfulcqB%4?b4%XiIq;{j^zK+yOr}^kL4`Sc1m|^g6yzzrE^LsnB&q z|6>~SdLN@j;v9VLLyn$~&`USNi0ETZ6lZ&Vv)y>K#B?ipVe+pTfDc#gqUgP17e((C zyD0d#fyFZbg7yLUrCDD~{LaX+%K55$IA4|1@fX_uX_^FF8-;$| zX#?niz!iSVc%phE&P`2RtLx#%zIgh#v5&^DitMJ+^nrO|H)TurI{ZFS>?Y!&?hdB! z7rweaUFZ&wbJ8yBACvm5r>T$KQo1Xr;jnQ3GRd$1HE&z7PEGCHGjqR1-nx0l&m?vf zdVq{m_}m45>}^P6XI*fEcPD&=C;kxNJCSoyiu*>Fzwp}q7I8!Fx4C}-yF~Kzrv?Ae z;-oH-8{<=m5;PYchBWvP!I z=`tP%xVz}#9(XZ3K)+@j_d_Rnkjf=?hznyizKOL3B4lFWWwFEu` zc8DE_z7U(6)%nm9l7H7nznf3FEOww;NxTtsDdmdTk8|BRIe(Q2|+*UPyOmwD@Qz$xm#N!rb`UQ&+uMyuaveVE@(mg=^F`>Y$(9`vdi zo$mSj_p#1rGmjGVJTc4os`F6dr=fgoQjYw`0(WoWKnw6W+hu%+@u<8RpD9niKkoUF zcqu!3SDEt0-XDc;TEDwY?P2|=jLQ;#GVFZN#B8Z2gMHPJ#Xpa})|SQYWge4=!?#&? z(_ZACFn$ShuCh5rUZVhg&{D4-0H2yuZ}G(bi#KAQme@bBGD`;b!8cQjHQ!W?oWpjt zo%#BKhlnp|T&&M&n>g}ju^(AC6W?v@Lo@Em{UOn~EBA-UxYO{}+HsrlN6)t3MfZnH zlDHZ6A14A2H9+}@r>y=3}A2K57c&W#hz|0pl<;mqi;<>-$HLReAMiBD$8B2 z>l-~Q;+2 z-wDYp??aDKd6QS(ly7Lx*Ujh`{sMnscB+|sMi^&vGx@&P(#ZYbe*r(K)X#!P5j~sw zap~`F9nbc&fqD29x&PrVpYv57_5$;tnendJkGuwsR-Z5M%HUt&N!(Je3m<62PAM~A z;O6`6=g2uS7k}f#RoIz~hjB?h|G4Duc4U2lLof4@avt!Oa#G{#eDkm@T3anGI zIZOUh^rj0IdE3A@H4{8b!+*`WGb4W^y<)8QR`@9MjXw-OXP;$u!QJC3mBmm0KoRg+ znX3o-ct-W(f)B!nqwyJhFym`zhM%p{@wIFB&D>{$T}M6qMbLRuTHEh0E?K0uvA(KF{6`{R!*-`N zKKpLb4;!4uz<2Pk(1GO7c=7ez8$l$5iPW9MZxy)+3NnP-!LVBZ1_#(-j#^Ip=VF+I4S%l_}l>Uyjd;LdQwd7#_tKfc9yk&*U)u> z{J?v`3;dS14RpgN&?P0_SP8!syb%7ucq{x?^AGs#1b^y`lkly|eKw`f4WLsHzYy;O zj_`~1;zz0QW1%PL)xh7(d#v#7UeVVWX92#|SPsOmVeb8C7Cds8hrrFP;!&f_*9Om} zem{DFmOHc~`c)SFAcC{u$73~opzDYqcd%r_|4l#j($BDZ24AcFgmS>-OE7rQDC+|r znEP4O{l4};ne{RA9fogO{+RV~1b;mEn8kYGhcfuX`?i7A3^L+l>>Fx7T7vwmJ6yH_ zIZNXq&?A-xG!nz!2R&6 zXkWmHGa^~>>w!k*t6%!xUgp#M=x+_#vdS^j|#pp{>1Ez_T$GyE`jCb`_uB$@V|uA%b?v!J?!DC-Xi7TFP$ag;6y*99P@4x zxHf9NF{Fo@`)tg;I|-pL5BeCtO0TD(uO^`{U(*-&fx4e1G*-$f9Z!h8g8mZC1ABgI2l1fj1I$C~ zEn41J@ez2*_#=EYa@^o!qu^r-yGQds(aTbyJ`$}+i33_A(CeI?_80eHy*f+>`R3*E?6dM30Pjy5I$TTjrMs zuFYzG$K%L1Fty|iu_6z8Urrx@-?Y()xz|kUXGK3fsn^fcXZ_}PED?BC z^tBqj)BW@PM?|k1P46sdJ>BS?4MHDj;#Nf-l*M{~9C`=uBfYax>z(MEBIg|R)=2L( z^kMYQrU?Rv;d-a)9cULk%l8gEM4p4+TkrICi5*|nJHvO<9qitJ1b@2ls~pcruQUA6 z=ymYJ`7=(6ALYMB@2tkLhI(g{=$-J#s@`coG_YFmo_GMEJE4y<&uDx`uQB6efi*ux zANhv(t>U8B!1qjfgFmJ`dJXeoAA2C)w|^gc)~I@?8w_;g|6oO5s&}AwuG4zutJFIm z_~!nHzz^sDR=qR*xBK^@ca5rdJ_fu|w}sx)Rs1*dv{CDwg2(RPi5_%nJvCgJ$CV=_06Y-mu z!O!^}CuRQL&j!%nU%lR`zBkY=dS^-b@UaTqgnvL6rl00W@5H{HKLfecTG2cGpNYLW zQtyPGJh6XSuK@iDi0J$Fw7j7|Ezc!x{H1!Q(Vv=yzC=#Qd|mXH=zcr1-bQaV@Tu4G z6un2vE9yo3*wCZl#})n`=^vx%t48mLiyUZ^_MlHjYw*DMAB-L|njfOtPAGrO`c(B! z*6WD)LqoeoS>92>Cpy%2VMT67a^A>oBj=Tt^U#gVClEQmHq=uE4nbA#H2RnD*^b+* za`4o8C-;@O+~ao&z0>4Xs_LEinOuDz)ic81jNWPNgc0M6^iE?Zyjs1}(Aj8yB%{X~ zJH@P*(K{=AT;PTMWAMzZo58cHzAgBOz8&fF2EPnk+f+mkk=_}Xb0$UcPRjbACmR2A zq<0$os8Q$(dnzt+e7pFc!#J6ddMExD;95_d&aOOuwn*=cOFpn+@@O=@vvNhyO8$cv_~yxgOWzGDsIXRG+nBmeWOeIK{J z&l*JD)ukNO3%@1~(CTxv-f0cu?>zN>GWAy{+5V)y!@VBF5eDeT=&h#yDfW|JT3^Po zE6qODXzh&HPk!n6WgNSF{M@6>qjod%b#_uo%m{^GNufBYxmZ^{#&Y5nAK);Yei z{)ry&DW0o;x@GkHYTB2%A9&(OLfHZ*@D2RX=Sr0oBnH86RbTMB^Ta`uz(C)(UWw0$ zz8ic@3O+J!!LN+S|I7yjAMNNK2;y0@#tHN9z;FQ4?qR;yB{<%W?A0anDyCxm~@#qj?a%2Hl-K zh~8q$dG#Xx-EqL*1MXJPGAQ!u%)sl$57>C7o86hG-^%`e^xeRJ9PmdzR^oap_im`x z!ONNDX!mza=KhWX@Mz=y4v(kJ{T(*<6^yvQgK^%3UuxN7XFt&np0q#fZOAjeM)t|F zJX3Gc`=qgP#22K|-)z_2tKgSWzWdm?l^)2*y&dE+Es>Y0F1oja_*aR;8?BxiccZUT zjH_~QNBQi*dB9<#>-F#;Uo&taPf+#Vj%MLQ;Gwx^a`j+eT<#atd7#+$n>5%b_MN(T za315Uw5wh-c#L^e@9kJUDEvN2y)4@XmRdb{Oz_lRE%mP6SUhVpe7ilheD2dd+~>wR zKVPVmJcCN^cWbsw?k-RA#JkRCij056J(uOz4&qO4A|K)o2fJ)qcwdHUyGqALanFN} zPi22Ty3Z}3JQHsHzVb`h9j(6JKQwvfn_88Y$4%U?_AvaHfpX6vXk=wN278cu=HAQJ z_v-f9kJWVEVA1YF%=@hHzP5zaE6aJaus`r~<{zzVr)SDBB6|CzSY!67v~{Jk7ArdM5ek{VJXl zzcXv_a^e%lxjJ4eL45BP#xr{TwbYmUAXj>l|EgZfdz9bm>U%@t6FOJF;bHJ;1U+V6 zr{~2D1!AA5GnKc+g-*{@*)0K2#ko_Ld6Q3Iux}Xs82XfZ7#P2yPw;4&;Au+}esb`4 zB>tpb@h83l{wx;!j0yf^;2WpLAIbyQ)&}s0@oM~mz3cm6@K3ekM=Wc7(tQ}b9V>W{ z2Ci0S;b05;-RStUSmWi$_168do)-xJc^Uj!B;!Ya#UFzIqKZEk40cm~l=kWce<)wM zH&yT_N75LJe&%UCX7Qjf2HOqqO&x(h#QD7xe_k*6X7Fe1nHlb@1U{qI7ky6nNr*oU zQa(rd3jVCP=?&}1(=rl&mJW7+GLq#voW&|&-=)TEjxj)hcZll(Rzd7I| zd3)qu-h`A7K%3~kg!nl46Gj63BP)1Y;U6Q`mwP^Fr#)KVk?UH%THv0LbuYfE^`$)H zj+b;?Op~lHdO~%5kv|T0f3&{jAwmAgI%QT3?jvqt^!2@3^N*41q5ETfuhMqO$n|~a z;7wtFHS2rT;40?Yck|UyEoG&&P43A?2~ejoQm$vBz|A#!R$%8XB-WPJCkyD^}ErZ@pC?Y@bWlX z0^jq{)rfmHgUliDtR&yvm;KH;wB(5UF^9GP;ree=GG`$Ba-OV;OTQK>N^H;tF7e^O4WO_n_2v(kT7@C1FY z7*|gHcH;77J;HHWN#vVnsWJ833*;ehqPYCjC%6~rVd>9Sr4q+)A897un0PE14}9v> zgo=_{pM-CxmY@3BHu^zXV4%40f{)k^P$z zyEm=R9VGj8f5ck~o|=2QqWT6;)oO<$$su2qbD;2-WWR&_#9`Ja>t^g{ShbF~iOK#~ z75`K$gI|sFnlYvQWtL@c68%PzGSmkjf*A7_yoyQwWjY_lU-F#jL0(+_T%P5)`h`4! z>-S>0nA|5#{V}4KzbNA<0mqm#R@$q+XUEFET*nl0FJvfW;AO@g&HI)l-NbW7-_Ml$OJuPd zO5771S9J}~C8fTMKixB8fUod>zjIBit|z> z@-R2TpBT5Y7iv9Dwen1Zr$XQ8Z{6}N#MF9u=3?p-@>DUkL7u?vPI-EDYNI?E&sh!V zU%;!uDm)|h(kJD2+T)Z{o8&WAjSIQA%u?4kcxN01ZjAqngFP}2@0mg5%E+hsOVLO8 z{hNa=qDOh(623{jza2a#zq|Xw_n?39nACR%qH;Cu1^dJ<8rlA6{dzwN+xLz{{XL`o z4Gj+WBe34)9@tUvxc&D1&pv$<@t|?$?{jXl!2w>Z6Bo-IFI4f$@V_Y=%GTyAai#H% zUwV2Uanu{j$K*bj3jRj;kYrtj-ji*#%lBjoy)6TLE$;#GlO#8$i#vVlMfZLs3llA3!@p1Yw->0R5zj*pcTrD5x z#)VJtTz;H(%=eB|P1Nf;WzPpxAlMTR&_?{}f!hHtf zYMvGPHtW%zYHRE(fIq2JED-plY)w~&9!IWgF1;bg`WZf0i$`ODmn(Ewq95=km1&c4 zn*O{7kypZhM1Jsn6nt0PXDO}6-M3gt{zgyWOs}fu9Qug*KK5xq`w!qpEAx!{2d}sf z54^T>dVZDhwPrQ{Fylku>30<$AM%$PKBMeQ!{1x449m4$ub-9nCf}yNoAv^kZxMK% zYQDWP-?;kOSkB{0c`NwgU?1?=h#X~I{&_5T4SoJ|{p}C>fME)XN|5Ed*U%<%U2T}u zCg)<*rdi?h?#1En`L~4c+pY@Vdq1jt2YQd_*Rl#f&~jq?d&BSdeKah;oAJ1eBicXW zi5)EdAdm9Uq5NJ3o=tYK+atat<$40|I(_dY;wu%ny~ve1JMBK~;}4Md^0d@1J|_0_ z8DbBzPL}Mu(+}-wzjuSQ&v!K@t?m6Yq#km>^b4PUT^wJdZM*JjvsFqHvh0d4r+e_zkL@Ji1nXDB-)i*@)1T2ZOn=?&q1^pggMuH&)H$Np z0UwouufUjo*N|_J^O4Q+JN(}8owH>ez{Qf^vrH-lzscx*QOzG~`>E(_W_{Gk#*;Sf zMe{Ly%#73YW5!v*1Do;EZ|%Fmi{F>esffQ-?ao43@3$!M0r)1zrczsE{&87n@Wjlc z%0DQt=`-RVnPRt(A4m8{z`lv_k1}zADg4ns@r=Shio`<*ofqIUR`FkjUnPZqxcH@o zf8?t44*w`?e_%xKn5CTef}P6m^b7y^#o#fpE8-v3@9uNLcci~P4tj0GkKiA_7;LBf zDD5>1{{Y{cg@2%ragGK)85jOh9$bc=A^K_=eO=(dv&ui#;YS94BKR2okre&`9vQwc zM)=3WRsDQ4{xJr9eAx++_bSwPB7DJ5*dux@^fE#2Ek$lm(D$CRMEZOv_EjO^C6=9_ z?LfmHqVLI==RP2GUh})LTUtm!B7RZu0ll4m_@C%2=Go$UzZ|s0PIdJ8;TBK*N{>hE zCFER7;QdPW=e%SNd}Udbbwc*5RMHQ>yWJW1l_HLXMY?6O)jAj2$_Jf>cTE~yW;`v?tD(ZKl4|mA#1*Lv1>rh5dPC0?} zn;x4k?98B_#!IDZZvn_-C6AAcC!0K}khi1hL+V!eG@oO(+GA6%-qv}XQ*mim#!Wv0 z_vk((?m;AuCyu?K(hvB2x6<*UcG~X>{X-AN8`t;wl#iDjz5g5iUY6f;;9vB6b-ww; z5cBYz7GE;@CI)Tyw$hYDc@(^RA^r* zE7LM$)vLLpn7FK0v`)!5c}f&`5UtbeZ|R$G;PU$a&wimX;um6>D)b)VRn;$a|5-!$ z$-qBtZyCD5PX5}E=w-&Q^XmOVUlTg3^$Y!4_+l1%(&tZDhbHn;!ROUD=)~h0(TOMY z1sp6Hr^9~;0TusAy2&o=%tD_jRa*THL19l|b3x{a~t zUEo*;9haFW`J*⋼(lr;{w~Ds@w|zG(Kly+zAD8t2znoG0)a`NibqDa1dx^$WIgg)5&x67Z%4ttm z{a(-1&wkJkNY4=u<{UtOpgikr5k?~IgMT&UkJL|o(1*`Yi@WIqM-s}zfDt)t?lZ3~ zZ`$vOdy5Xt15c%0<}2l<6~tb!wLfGs=Ml>p_=vhq>+?=C-yIDKOAsP>ickj zuca1fKGdkR{aV}akL#B_=-YKFuloz3P4J_4M8}v>Ln{rJWPHomaD-!CQtT&SaE!44ux1dv86^ z2S5ESbeiUzr1)Lom$E*?`G6@ulhCjO$g^7f^@Wa*OAd*5roqo21*9dH{v-NH#%HKK z!Vj(B9YgKJ^E8*)kKmj;=fWRIL)TVtx!|$MGZ7;Xi{!to6FHe9E`oD&(>t+)bi3^5 zQ~!7gc3iFhcF`k@980dx736+yQ$N|PW=!v|@-_Am+C+|4$Nkdz*5cQ0JwI8$Q_kc`WnvV?|->RW@vHO8@nQ@TkZA-d3KkTuN>HSr|00x(O9{L_u|4)R!yu$oCr?(B0 zqtX1T<9_M<3?IKw^S_bnZTNC$-22l5`{3i&r?maEvcc7M%xL)W+PKOdSZ2w7NUZF@ z?2=MGltbLCj{8)0vFtB3ZHpEOkW}U3o4YC=y&3$v8>?wp&s&C0gzX>kL=zy z)XhB1{3`QtKRDDb`)!OP&$KPxX_x)evHHHQ$PX~BQRNP_#MMIlcM)- zX5R1zOUf0H<3bN@!|8$F1AZ@l9Q}=OyQVJuU0UAbR{`H5y~JsbJ1xO*c)7O@p;uJl z#d~{FjninOX6L*!+%cWnC@6stZKUOG%k5;FX7da_*kOREO(SDBS#!R8RNc=(r zejnB^Zsk5YgnuGoyMgR`j?qux9_=F-z02U8vVz-1PMLTf^E-a0j}Bd)P-U^hfWM0N zU7WFcu&?B=e2%$IwA_zPZg}cVdSyU)EQiiupx$=*am7TZWd= z?}+&pHxFSyRp*NX+-5xWZQ=UXx9Rn@>f3TV88xWrOlREm6!MF0_*IIQx`XYbds$tl zzW*y*fWIa41&<9LjkUb|P&aff^MqbaKFr#Frb<5o=bC;3;X`GK3#jo|3EgDH4i4jE zEGyrm?J?E(IzLPQ6~BRP1ts`qyb=Gt#CfZ2Lp^{?@{t|-q8Nbk^A!S zZEfeMF4oyYuV}Qp1IC5_cYR+T`4QQ-sk+``x2&&Xf44yUQQjKt5Y3lZ?>VWgz|p~u zsjuws<2PB~m$ex9LXCG9H3s*J{X;?Y65tp4@$rLyVW=fBBAy`Wwpe|h)MtF)S-p%; z#%VKeo1d-ReM3y1er(wXPS%5GHhDwFLSI?NKUvC2{9w><$LFN-Rr)N|H$LdIZXIgJ??)8RmWJak9RDbAbI^ z;~#mo^`@$ezjdIm>Q{NS;~XRY;T&=%@()+|YsBY_-w^uOes%b{wvTG@r511Tqoqz3 z;X@Hzd2$b)z*Y4L|53;_i4TK*EV+l+?4!`X+RM5_U)C$dNwfskfuSQ*H*sL5pGg0r zp98XfrvF<6{+a(woSIT4@YUFD0jhOfI;*d_hAqQB)-H*&<2{M#k;B(~t?94PWh+aD%=Y}o$e{-I+c4`zgM z8PUGDj9C0#R1Eb=RGD3l@N7CWFmD0LSkkAveC^4J@){}@Of49+XiZ$pZ4 z2b@!iqhD!xQ2fErJotUI|JW$`{qyiCY2T*4w0{%pYT0g2nfXb*jMx!jzwAFgsqGjP z(Vi@Fr}Et`iXJZg?MY*YMD4Nv_@wCLqqWz-L6to|z-YsceSoc`POSLyZFs-cX^jTiOZ6|x4-F)`uFAGt6H9vxrTpBqJIJ zq%}hAJi@J7raC|DRw~Q8{6GA{KmHs3uKnrf)?B;z;L~6DRr%6{SA22fYjQt(-_7s1 z`)|MU=6C$)!7nfOI`V$+*Y8?4X5r(B*cJOeY43i`swa-W?k4v;V=w;DSH6Dp_uljO zcP?FXBDFQ)9{K2$kNr#2r*41ib#rc6eZ?j3T>OrnD}U3!XWx}=7bTznlQ*9D<+5`= zao7IiXWV(-uu(vX_ zFZj*7Km7eRTkRWn$>LRe?%9wY|H9aRFI@AM$Da5_?D_-O{&eF9^6y_$IOD>nZ|%Hx z!&6V4^*7i5$9JCiiy42r_|Vl){P6whrJwvuzwuWK-#hE$6K+*+*mn2IkKZ=-SND3a z+4Z7-X7oixBv8!Ke}x9c<66#n0L$Exgzjf#L_I>=*Q;Tyyy0|#W9(d%h{^hUE|It4VJn@}BxhH$YGfU?b|Nfk{SA66* z>#psb_qKO#|N87Nw71=J_ifJ||84N*?>zAHn?AO~|9JAyXYV?5@rG|b@#tGme)cWB z)0dCA>gMB%ui7&G#k;2e(-+1bK6>udZ+vXYHTVARnqc`0y}$goKkga)?M>%@>xZkq zeBF}I?bv+J*z?ye`Rbar_x|Y3mu>A{e4-`z#Mv+WkMX~L_}I~okKFn7A0NJK!@R5h zDDf{p|Lgzx*&DyM?&P+?A zx~snPrL9M{bzV_^Zs3FW-~Wc|&UpPB-})vfOH*0uMgArKcVFbxjr{Smgapgqd#=r0 z^TF$`z4hkz-16aT)?9ndHP@!Y5C4z+_u}`wfA!5Dc2u$c#S?Ijs1Mhp^dv0;w zwERM6!37KEzkA-IcRO#t<+@ys)-QA}`NIRgH5K;RCQpad>^z0vXUBxUJJEMHdamFp z=#z<_uk>A{!z+C!Z1)wv%Q#-CJ`vfk_+4btE7d1l`4zwOx$@n#^WXU-Z^t;vmo9KC z1u^mz`Br~c-oo<`mGe;1d4U#J)%6p%?8$j?pZkwIudY8}btEq==O9`F73(jOFQOr* z&;Pcl?pVLXqc`~Bch`#d=M>)q&Yy7asn;Fv_gR&OvOecQ+<&6KK%BhErzrWeJb_x2 z&o??>mX!O&1LE>iZB~%wFqxdsBHmrml*DCcV`=gzFiz`a^Kpmsb{wf}_XW=!eIASR zd7M9y)ePt9NqOS5z3vyAk4w9i@=C|=`?|f+zds<~WnQDy@73SQBRk4>cbwAa3sj8! zO~mV)JTSFS$%7*2m`SUvdYF$BYn&6$RL=ud@}x)}VH;q?vSI#Vb8mbEAM<-{o)j`L zhWUpLyrc3-;^GC~Ey4X~_Gegcu7v5~yaj(czbScAS|ty%GI@(lo}8uOdYE}|e#R?M zj`ISspu{|vnmorhww)}?c%N%KS>!xJv_4)J_ibg#U#K|O#5n{tnY_5tZyV<#G@jOT z9wMRY>Nwv99=6wk2QofN_-;~}a}_UI_1?BLaI3R7d6YBf2Ws==nY?w}M?t=HT8ZH2 z>Gh%Cm~+O-LY?yE-I4cb{HoshL*{u=9SdjsV6$>&%=Tqo9o%>103V0@js#9=?mGf% zkIFeT;HPFUoZ|~zI%nwoi)!|5bNn3R@4SAF@A6!5rT+e`J{Mcfw*)?&wW*L)uan=) z(3$(fw4zOahCf93Ve&V~fzX0Y{sqOk#De4K(|7l#rCw2= z--*wfomMNzUvpj%6g)YvN4+f59<^Q}yk}?DN>^8sR=Ra25$-3|ic4p)1 znW^N}Vcb8iTk>FU4mjS;JyRa=*z|Ss!Hzzj*sL2fYP;0m&3T@f)c1PJtZQdc>Ab>= zisAeh6r=fS@L9kINx%0zXu#AK=&^Fv}QGZwohEE@8feOuGQzO zdC%=ES<0ED^3#3RbK5#qp#I!**17kVxZmTP{!6w5;A?yn>kEB5cC0T89&LJHaWTj5 zi)lZ@crHAe+gapXcvqnIG9Hy?g#y;e3KsR3d768Q&f^}j4C{WL@FU?f5ntN2N)-ej z`4f0Jb*bw>d5C9xw8w+%B2VG*9O_ zHTBdq7ydG=oyxd_OZs!PTN#&oX+P`7{h4p<&yrt#8tcfqRp;ftu^&7z<6s_d>@P8X zDF@u9wdPk88K)VCEx(uImG>#r3f;$j;1~6$tiqcGZgmd#H>S}e80T>ZeTjF;+ey3D z@v>FimMY^*eHoyHY=X+-Ck2czKugJG$L^xp$(@$*R32fuFOgA395@`&NmZP;FkDU!q=0 z@PqP}`lEguxXZb6;8uBW6a0{Ke+s+@TPvE z6M9vWx77LLegzz(dVH@t?omJLmwv7k{m_()E%LnMY28lzxX1XagAr&x&`6AopqggwKvDS()l(&0FOJu`DQwyCzrgMv!ExwTUPF>e%4>9hmjAoKN~u7 z;5#Dkaxa0;RsHm1#tD4F_sV!&+M$0}wx&42uAHnZdSixliq=;h@pbuW^NQ}aEc09F+T9uSmU+eWHr6S=&@FGa z$$vSonBA6#zb*_)I~4e`$YH$nv(?pkd{U$t-{n%?%H7yczwrfbAo^k4E_%YB)gsBe zO1@E{haB&n$OTvE&U=4<8akcdpWSKmirkiabh{IG%YodRoS+=xP5f{jp9zl_%@{ zV|lV(&&V^KR6mhtP%rwnmaoViPxu=8G~5Z_GRV-s|_k!}*W7J6M-$yY!RhJ*ppD=%*r7IX8YO>+O50Nf-tgt8hXTy^Dd4|XuU1V~vB2k^ zKaJPaKT>e=HuNZT#ynJva#_}Yt5fLCk{@)g$?sFRQ}avZ1cDcJHg;tmz8Tq9lne9w z%!W>i;3rq6^h3Ya+=9M02{xjC=RTc{!v8AyZaUHHfJd!f*Cl!#cmaqtpE#4DmCE4B zwiNk&w7k)F%N^)-I?s=iJU_rCk_&Yr7kuchuHVZupZ0~=HO$8ry*DTG&M|Lm^m$K_ zc_UA(+^0l7GQyXi_Y|2obT?V%2Ry7`Yd_<*rsTqTN57*_d`ixhSLWUNUp;U5Z~;Eq z70N?z9Q?YBol}tq>23W*)^(||k2CNU#vA2l8cj}&_x}KW<><~TczmESz*I*kA5@1{NY zkN$ot{!=G;iD;+6D*oe;1N}$*r*6$wb<~l22ZCPVhr^z`MZbQ9ciIs>DR|i@1_W{g ze8z;)_iDy_wY?MTv||C(8cQuwB4=WEKl+acpvO9~dt99l<3W9o*c8utK7;)|S><|t zX@`Dt*x?a9+f&F>WK8l* z1GlaNQ^ig;_sb&3wcqTbHOszT%(31c@N@;9=u5!E=u4%4MW4;X?jwEH$jj1CKmFUMeaEWwXU6rcjGOvKuQ7UzKdu1YfVXpL*Ndaa|HY@pp6ZMA_$q!C!*Y>6 zTdUt!#?yu05jd(P_#xe@{%`uP6MHp4o?GRA5WZ!`Va8?juX8q*TDFwX4=VVpzSj@l zRM)2l-$#K*?fgxDs%D&~|LS~^dp`JP{MMWB-`Swu-S{JY@O$lNepVFxjOcLM+U}nf z1y8rbKgz7refUv$e{swjZ$lb?*&^h72K>W+C;5YtI!_OJ@hs#g^GwM7c&w{6A*1a8 zxi9al^p|YSxDS_kcIL?EL%yBXaO42JpR)^Akp~~Y z1N~1;V~3gY_HFP#;8Nk&xx;$jqV-Cp{Q{FqQ~ff0aB`+!?5miyw%~ctzjd^jXP)(4 zSiCOvFiF& z*TMbgey4h0qUu5YA2#ls9yj*wH0I}5^ERrUVj=J1&N;W2ew2Ey6F9Kmu|=!WPQd<$ znO_A@%skf>cDU&Olh>&oW%$Pw=GkA(2c=rCN8cqM-^8`<_6&LsPl1bSW1cSXGWVrb z=J_z~q3_RrIBah&^ITPJ4}ZwKgAVl<<<#*Uc~BQ-ruO;?b@k@rotw+hS2Vs;?Ms<^ zckYt>SDvLd%aeY$$dh^BEzdxyPsy_Y*~l}e)TiYMECPAvu{R?8S61Ozza3M3@_VUH zJt9xW`SP7rQMfc8ePuNuPdz*3M2mMvQ%Q{)Bphxi6 z>W3cTj|(!M?04}FJwhi7@UH+5wJ=laPK#dXoalGJNBm>uocJN5@6mgF0rXhJKf{$N zp`SJLYw?%HDW~aC^XHeyo8O3^Of3o;B`+a) zO5d{(TY$Ysy#}tV7r(2KV++>y)bGo%{t?})-nAiIccbqy-xtR$@HQ0qJ_)%da9!jE zy}(m-;s0{L^QC21@|MMxl-<4HbA0JaOl|b8shjk@V5WYj$OFb(soy61MyLzXy2NB% za@1ecm{upqyE}<>^aX#TeXp8ztd^hc0%Ym0DECS747C5rQIWk`jvoZy)Yfo+Ol>U} z^J(T|*4y}{rm_Fw0xtsx^zw9& z(5dQOp?rtuZtx_Ff4Hg#H6IMb-pMo{%rgE+uQ2k}(5K+F__fs}@QC$S3$pNW@L*C_ z<58k5>+TKMf0)F+X)k!N2s|hVoxVIC7v>l|&snqrQ zT{YeoevkA_NDn>d)bR!XReapcw|1UYzUfUl=yT6%-`_P#UU~DovOMP?{JP5Dy}1W{ zOZfZi_Obq^oMoxk9n^eixwip2P6>ba;d4_m@OQzVjMf8U6VOM&k~n(lUhp+B;c-Xx zSMevI-o?H%a9ofN@oi>4EZ>n2%g=l+t-7#}BYdltFCjnRr;AJO!|?NXSIOP$(f;Db zW$Nh5wGUr|FBrZCzU%T8{c+Lk%T)^e+?X%*0GD?5Vf6ky`!GAR$oU!DwY{o3;9smG zUhi`c+RSrnF2d&;K4$pWO!!xk`OQRsb9tt}8|n!Q4uttTyM7uzm;5sPolMMrN$_gI zWS@HIjdKq=f|oP(yJ}pdzkg`1-&lr^{cLV#h2GbiUDW)pbEz8_`)}G(_u}I@@OVj} z+r4Z;R?C~mCfJuA&oj@~vc_+t2gKCtJ?($*ezK5<5BQ=t3A{va0zYDl(68VxtgzPC zhw+!o4tgW=W@870^S$2>_3w$IS6QlaGVSG&%joM)8h#SBXW|?xaSxGv{X_AG1#9PB zw9k%@s_#_gFZxZE{e&?g-dE%&c33ZXh#rW32wv9Uz1kYaT_h$TN4Mig5xMP)-4)?~ zm(UCRwx(WU!fZ#at0%0>RwV^q7<@a098_~1t=Gg5GrN4`wdF27={{<@fU&*`FhpWxkPQwfEQCjeS5)L>rMMa z4p;OO1D{YoaVGgeAq&1u5V=A>oxaweO?*jA;%i*=i&?s6bXKv;7OpDnV1AC+Wr5%^_K&_N#9>|F|AH6gc;(*uX=mM&{tWn}W|X|1 z9LadcEmRLsKh~9YcY)7UK2d)A!JtkBOKtUhsZMQpeHnd=_Lr2@UY{TuiEn|AROO3( z#lb9i&?Wu=(W{DA9MpJ1xg7h$75gK&;$VRNE#soS>O4*qc7T^@u~*BY4`X+&sM`Bg zKUpU9lU4R3?mpvWntuPl{HGTG7OZ{fx_x%CVs9bGJ-Kfe`^%Gk@zLl_O)3WxzgD5Q zB{}qY)?-Re%gNZ}T&T}aS?Q=Y{0$X5E96fj>;gl#HGGZxHZuAohee zOfPK9hx8uF^_a+YS;v{^g)8Ja_eTE2vf}Rs9^K!7kJT%G zBKYlrcde0Jn~Of}CqQ;DCOkoErT&T#r9AUl~n5p10PzS=$*S_2Um66uPVBk0bTt zu-yO8_2an2`#=@S3f3J&|AMdR`;9Hj{pi80QXiUy-sL6L&t{=_WxYRn% z`<-^Gkcq1$YmRN$2OXOG%N(hfN8WhY!6oA1 z)~x=-KKQ7KV~M5tU5Ovm-gwYKd`kS29aBr!SR3|bwcQ;4o|E4b>Y_E)#(hPx=fm=i zdR$4$UvlR@m8^WfXidw;eaM5jBmKg6q#x)v)|m?=-Z(xv6Zk%9@6~L+^Rzwu-$KW- ze?c^|_FE(GCWPZpEU&}w1O4JcJ1B5A@Qmcax5N$!Sl>m!lkX$p8GQ1f(7m|8Z{tK5DIV^5CDELL3w8Du*2NJgjM%U{?lvNXdOsvR?pPJn?6R<)@ura<`S~ z_dO$426qa(cfqic=`ubI=t( zHE~RedD|X-1mF|F*X(PmYMg?GM`Gdj>`ulzGF~D5X9pGQ)1l`00pp9}m=gB8JU;_H zbh4iOo>x%=(>8JebY>?h7;J%>nzOXA=(wJQE9nV-fgG0*)odyOn#pfLk>ltbCu?_gS5JI-C<-^KXi3yWXbQf6MQp}yXJZf_V*cW$qar;DB2%e`N~ zbCya^&$2Go!;Zw2V^4sG!Y@X^FWxr{ele@p5r5ybvjl#WUzoPLbNCC-68NF7EKCDG z#(9>&5B*i($NH<85&SlD4;cLngWp#V`tYBQH1OkT=es;h}+F41yEAqPp+Pd73oitJzls&f1TFVvFA# zZL5xZvcUuM{6GD>s>B249y(YgPHB|!3E)ZsznVC~C2KA@w$DW$uk4$-M@3Js?u%mo zIjn=(hwgsJ$>&}UPtx!!zz63IToc5*}qVI2Wb!g@gw5r$;ke- z4}C`M^Zk*q{qXw={avZ7JmEW)^FQ7DG=H5oFY9g-el!uex_Ds9A&$#FTS;Xc%FO(-!;fH19$2?EVlj?fpesEBM=TqU28h)OZ zFT9sFEA8L;fvl5>GpNK%SKts74;C5!{8Dyj@N#fS|L~wxnk*gAB`IxsI2w{7sxJpUhuFMh9M``{l{|4%5N@GBe>J=Vx6V;5BYSKiOX ze;QMq8!iEFg`5yMGCQaAy}sz*5+B34FtyzuMknW#N&UtqdTK3ug&gfKg`efYGqZl?sXiLg$>fkOM$RjX+y&1gJgAO4$GB;4a#sDd1KuJ( zb`{{eb*t2-40sbOb1s^Aa{jqT=t(g@kx$Y55*tD7JaTuU;ykVFkA`vHkIl@X-w@~B zp<+QA`H8|G#u*y7t0b|?Y5J|4n+WfFjV*xB zWZ-*pPD1#Hv40IbYMu(#^e{MqVP`!cfsX`c}J7i&ZhV|3}+!LRm}w0#-=p5pgB zQH$&-FunLC#4EU1JaX=pX8x1@sZXHMivMaaqsKOi1_9_wie^)+T|! zd0(HkHiH-F2im@li(ZnW{`gg;oi2KOEA&nIZn?J@e&8%_by~{EH&&qJQqSzCXgfHr z?zm;1+CaHGZe5n$kYQZrT(!s%=9>__Vtfgkar?kC(!Xr22h%V1sp#L1iNBxbgmTu{ zB@>XdGA^rN9m-;tI2q076Jp=Fz~9(+G9Tg7trfc@cBnwSEBci_pA!9US$5qam$IS< z3ml_;07vW!oB58kE1c=jN5wAb>{UA6zLR+S9OIUGi1(!d!%XH;qTXig ztD?lKnt7CyhZuLFgLweYgxCiq;9~595%X}z90E^A&LfjLl&wGIJmO*>vA&E`+eejo zoZIW)D{w!T`*K;AX}8|5wEx^Hop~sS9X@fbw##^icEh>EtFX>u?-v5%g3SHUGJl`- zmH8L+{K<2`{G)vzwJ}>e|K>wpOx4fM2EHwRkbHirFPU>tFI(q6^Bm=x`oM)-cwn#<36G6()YjQ<3> z+b;YfPx+1T5o%TKj`YMsSxujsz6`w>f41t$>wUa27L$AkO^dmo6gVm? zKjjegX7FQLmpbRkvgm!8DTm-6|9)=c`o45bb#w*iJgH>=IQ;#%{2tV)sa@`rCq1QJ zkoN-eO7?ku3x8Voj3>d%cP*hmFRq%F(q92Ox7_K6prdI^-1aAZ?Z2CTh<5+Ivk^Ou zewL`IPv*7%PXB&fey823OVsoy8NcCo+jU$i_sUN}U$>dPSr6jCB|a}M^JX6LnO&@h zz{k<+@q)C+xXpU2M&hw4&$xpff;Xm}`H$&#)}&YJ{bkdRp(g+l?c3X(hoDQ<>Cz7C zZS2|_yqmVRZ{d@6y_&lA9~KH6t8u!-*#MX7dEMNCLyWIY%QM#Jyny|UYTR2w-R_FN zRNMs;KaU@o^2C2C@>V#+yLG}(ZSMVy;(L>wUKqFQ5C@TEKcxfy<-v!n;+#V&q3Y!x z&UB&@ryG}c^PIDicEL;a+ML@19P1}jkM%PBVgJoPRAPMXe9sb}TK%3r#AuQZdI5N2 z+T(um`G@FdwDu&vc#lsUu)c?rC*por9>1W}^8_CmPl@rQ*}s!{2z<({*LlSAy6~;} ztFoH~Ka3uLXYP;>x?yLC+(^tPzSjo-PEp^J`tXAh^|l5@iDQW4KYzojVRWgk*ZTjz z-}L(Lf4|81;$PMF#QO8I-Y%i@#&0T#fBW~`8Q=dtbnwDWi3`4$6Fnhw!J%wiy=!A| z*5*=-c$n@rPrCSjEAf)YG{{Ydu$B$-Y)Q3 z;>%@y;$M>S1IILZ43GzA|2#3lj?Z`ucy+&{;6Zos_3t2#g1C>UT$kQwXFd}$@fk-G zDm#IEK|zf1V`pU8N10$d?OrwR=1k$E!cV$0ZoUA0o<;vz75slxByHoRD`X4dwQhX-+vlaNL#)gwN{YCZLTQ~gClY(E7-VG+Jk`g_%ww!zU zlSSZU`kml3==~}4Zt#25ak%lBk8$oGOT1;7{zidgQs7tuuB=-}FXdkCxR?&W1#HXrsoX;Ne@t>q^PtFlTx7jB>;2V`Q{TsPnT?gp)C2{jt zVmF{4jbCMN2j3l@J8Z_^nhE1vYw1?YIp{Y1SV*_$LAT&nWt}?FQ?k7E`1%|`{GI?k zC9a-JLpSil#4PA4$2vymENaW;q#XDq=P8^*Qs`^-GufBW*Skf2RrR5l_OnLU*Znxd zenay?p76Wc4nIY1%#i$1Np;@ZTf3hOSf?5IokebR$o@Zk_MBAbX6SYDH0DQt z%ABj1zoH;~jD0b+Iba%*-sea;_>QXRhfz}A-`u@@yj53y;Jx;} z=M6&iB)Nf`fUvpu0yl622e<(W$lZytrZ#Pltva+xr#lYq!%S13!?aq)I^(_=5g|df1G zH@x)&|MpqroG0^lfoI40U##DfxA<;3Kfb>5mfwB0g#Nfz^I51dhPhTkTSRjs91QT}R}Li8nIz zd0}mXa~S)`>!@oKu}?k!zOJ*-Z)RQ*e9`Z4T^)r!vR${8M|tj5DmWcLPeu;6KNka` zhs?02j+Dd?V4sx}1)`^2{A|B--aj7vIds)Kth%ELv)h?-`n=OJx|t{x0(Oy zt)si2mGS@e@$%jnIlVekM+44ftI-D)e|OaTPXjmPs;*N2yx)HsdZezyJ)$3}Rlui) ze?#EY0IufQm1pMns^N9_=eo|u?1vrU6@2Kl$2cjxTA!;QIvop~Exhcrh1bQ;_Pb8- zvT(Zdk^GM1@D1|nFLt|G^$**!_dZ!?xm%qtJ%=2G{=W2FsPXwna*lO>&$;he=ycPT z2mbKcN>1JREbVJr{(XNp@4?%b#v>*DJudmN)1GzzFKt7AbshDR=&{gK|JLj?&mq?b zwmQ!|7omrUeXfjrDn^m`%j`TYTYhU{gboHccNdY%gWK5;`EH2# zE0Bs^^YGZtTOpT9!PUB+j?J&fFa;jG?0f-_yRkC2rTo=@RctygT_* zeRpCv>!wZ#@xCGO>XH2ed!&iW%Sc>aAbKhBV28zS1>6GmQ5zrJlh_wKk8``H&T*b! ze^%%B@SEh_tUs&sZrbb1F@JhjyO@9c z_2f2V&scx9b_o3Q8y+en57wX6@f!G{@ZW(Sr}AG;*Px>`eKRqzoJjWhqeyHC?jZhOTV=iEqxYM3P@->)y21 zo3QO!|EEt^#LkiMhq~9&6YKXbwf?(Zb%Od8vVP{D(ox6t-&+Eu=s}lW|2J~f<*8Bk z@RlL!CT8d_shi09Yg$k0rJj#ReH*hLH4lCZz&rYa*dsE-?DJKAo#L_6@R{PgBy^Th z6Q1BdTT1kRj1&FkX(jTu{+D9s$*QNkT7`IHY3~9*d0*;MH?F~-tcZ`EHB0Y%juU_W zbi{c=gkeQ?0z5_B3SvT@o+WW{c9XE{~F8j-+ z{ftjP=qc>WPV`r&={B{~n!jhA>VH37;>9NXh-J~I`Oeq&3FFu2*<;;S?v1EjF9odI z#EGdT!XK<}xS~X!s^K8{Emc)WyP5KxCwdOQjcI(;pYYSnbs_oC0xy%_uo3wMobo-) zTiQ9nU&}cii=!v>GP8&8qo%x@u?~G9Q9|6W%&D@UI;wUkG`z!+0bn$gyRA~`jPeMdC2&ILXSB4^pkbg zz0qSo68bKyFyecs_{?~n{!M;IUh-Lsdi-l z_$eycXT-jn9b)}H>vNL7H|bwJds_M(_LbBjLLXVQJ7zxDpjU*@?P~Ob8t;w1l~td^ z9ts`EIjhWjbwcm^E4OYt`D`6K6mZr1s>lTX&^u>fy488 zCo1c|8$HVX>S^fk&KE>qOTHIdejQw2tNeZ#OQ1M-NXTe*C4Nf zrj0*Qy=UPwp2=U={$Ka7sQ{s4R{`@Fu24Ui zbqu;me8n*HD$;&lS;sk~`c*IbmB@*n3U$^Prx*Ro=ewSgj^A8)>objKg&yqrX2$OY z_f#|NgR*Z;uiLvv%M}~HVfW9x?4KU}nmplxuG=ehe;+dX6m>8Z{kPP+%}aitZkNQ- z7jzyf{3?Fj0R2kpXZX@CRyyC=!o- zwCYK=E{&}dbNRY}j2}*~3z$4Vp4J5net5c0{hQ^;ari6iB+o*Z7uN+;7i8c6W?C08 zdB%U?jJubr3z)3?@w6@=&++uSfXRE?AN!wv&$wmDQz)zmuvbIxL)86iOslIjLSB8G zSBo}L&jPvg>pQ4t;di&y3u>(kl$UcU@-JUR4=)0Ldu}xQM=C$fbKFw@$kq+A`<~dT z0_5=E^0T#rW$ZZA3zfPa`3=}%w>I@GTj%F*yKc`&{YYESgm``LfS*x){2mlKwe?I= zyR7YhtIjubd$(%4Y<@0w*>j>FDJAyI3ro=d0-guJUnKfV{fcqmmy>6q8^z)b+~$-3xUf4_Ln+zKpi8g zKXAdwd9@aK8zR5w1W8?i!BFq#Hvcm(`JW;emu+#?E`e7Oy%(U_IuIS}egt#IWBi=X zcI{WmIfs#3;B~MPghlpG=Wtv_Z=nA`+bxz>;!)ts`C!htNB_$_ZBI`>^rLHRHmf>&6x9UPj#y54)o1dC(X7|J-rthW&G17!q$G zbw2|6P8Avf&x7bw?9*2`(cE$9qgnUkEsLKMI-`(D8dacZXSJL}c;9&(bGyBR*0 zJYL`cy>kASbzO|d=J!LMI;O`<><{y|Yx@IiZR&p12F5GQ*VJul{`wd1I}3;d@l+EI zm(2@|7mov<%jE_3YWy+&GUOBZU9^-urv}eAgq6MJyb718?z5rby}oSRL+|y6w7t9e zj33IA^Mrj5UjUcpbL;!o_LqLPe;n2PzM<_2w$4#b>KrjneuVlTCoc8<#qtWePrhG9-H*fY(N63nF6YQi z=XBg1zgoZNSL@MgBl0U^p49yxugQ9nbA__|l&xcA_2v=eSDF2MM4=z?`>N}H_`ftB z3SG9;{g`9&Pv@BY)1G>ge>&GA{}lVsT**I$pL-)+ui%#!kbkNibz%Yer~ImU4PBqc z*8L!&kh&kp4_j|&)iL719!AJBPrxy1)6ha#oiCFCgrH>)4n{Lh@^-PM_Ans;aO zq_7jk*druQsw#W~TuQ*hIqO!T%l@0H-o84^TkID{W6rlXRZIKmKeyN?{}Vg>O;vxN z3tShIcSmdSgC_Z(Ca!b?`b$~tr?q#CLj}2c5?9KvUC{PR=s&gNC-BHi z-W_;yHpJNHfRA}k-PwON?y;Vhyt{nyy@A*rv*h)LG4?gd{{){S%L4CcjqeS7-#5Hf z?LyDXx#WKq+0TrBznTBJHP}-EKa&5+IG4}=^rItn(bKU@`SL6D#dvM`pOqw!*5rTY z*Hjw2>*zry|1&T7i@xAV;3)b}CCO)61{_&ehWh<-?#d6bzKGu<$Cd_gvGKWvj}R|5 z|MUMH7ko+{Z51*@K3;quK;?JFL-5;@N1H3&O}vUe2kJQE?D>1u(HJm`2{yT!HSAs3Mk>m`3y@;}K#E+U84S7WK;p+-}U&W`6?v8R^# zUfa3#ci9Zz7dZ#67QYAf^b+SPPwcV9UE>~c=(Q7HKf?TSt9r4^f`?UwdyZG&TgLlX zB!7qBWWA*V@xS1~#DOlD6M0AVI77gb39KTI5;>H@Me;K%(1(dn$X$E=XjGMbD(RQ` zINDwjh`&na?e7u0OLkRMmvjE2RlA-M`}M0HXQ$V@TRCZ6oYrw{{{zy06}f=lN9=Rx zTgD#~+$(YvzfOw(gS;nx<~}*+ss~WBV(b{3I2S-Su8qfB&Hh}HeL*GrGx@s`7xk9e z#q92YzxK@vL*SKD6YmIP*?*IKolR?F;%_o)^RViUT==9`D9QPh_@Va*T`Uy;d4u-G z4xx6KydC3DP}YC7ocL|vO&%uq9ucpo;~DW2%pq>?7PUS=(Ed&K#r{hD}q`(3kN;wg^-ryWT?s3ZG}OMe@P+bc|~>uUFHqklOYB~A~z zF#EV&*ZY3&W_6DupXOa}@$V7&V&ic{-g?lb=nXOZqS*C;pQS6CS2->JB)P|81M;Z> zf857D>2dxMf1L1<(>OHFIMfM_WFEO<9Ml-i+EFi&fAP`rB6fuK{EPd?<7xR9#y{e& z!oFK{fbR*dH)K|wsObEY)lxTtb?rN$B(LFp#0}~^nv&dGp_D85Ekgg!IhzlZ(3w;En8?Lk59&0?Vb6Kyk)A1eaXa7(2<>2Gvb@qXgMzTMd zze8sq8+WJW(R5fxR@T8Zm}hB`z~8Q8)rlmJW)*of&~N{{j;sB`Cyx7H$E(oqnz1Af zV$GPgL#S2MVGfhJ85iZz$oiq5oU@g@bjA^VNf49jJM7aH^v1b@N9fEc9g%ZUexB^p z=!>H7$a<~5GnfA82Szjb2Em_=BL|(iezJb^ch-MI_WLv+s>Awovi?Y~-&+)-r>z-F z>?tOn%jVBWJ&c;rx7~l%a{h>THuRnm{}g!4h+VY~oJ@R9t35S+R=oX-n2UuxthoougrY%!dF%FxYS>UzR3KWe6`(`x+dTw z;j4Z((Lb%;l-`q4`?T24$X_t{2oikQ{4}wn0q3UuNgW%nvfsDelT!SQ=nt2}$E)Wn z)`4EcIugFJ>)0)Hg?vcY(fFLm-B-R2@AKoeORb~)-^Jc@xplmHzKXw~`AYH?UG%is zn-l$K6?RqC)t;Yb_$n{;W1reX>`(1lNouae`~cLLDb)>N=^kzD4ct#>>!2YM)EiYvI*te+(rrAb>s$ z|J(a;EPhvREWP+#%=S`3B=YH{P!-s8tZFO$E%AZrecJQ6?3iNi_`=;W* zmpq2nxWg|9Jwv}w$Z!4C+;?Z@k@~fx#yI-k<^9_3i=yZJ)BW0vAL@8q@)&hIlzx63 zH^~w9udL&}G!8$0vyW%^%sz|$CV5hi&*ty&Th@ubFX!uT%5$UnjD7H&_#ye;6aQ|r z-(=mkea7KA#xE}8>T#0ywmv+#l)Biq{Y4}P-KjQwRh~h)$UZFtvvn>3X^k?}pJuj}|9NHic zzIpz1`;MXSGDU5-<{lUEZ}!jLtncmUQ3C36bo<2{f@6B_D<3Ms>j+#Bey0AaZo}hh(elC4qj+!$Ov95mn^mX!6Qa`}UqJQdWT{jUp)BN;%`=75!f9PfO$NdsX{DbksT75um zsABJfZbzsO&briq_{V^&@sDN1KUU-YeO^8Afx{B~^~8-VOz-JP;;RR-lh=IUWBW07 zI_v++i2sXzvLpEGQ8Eh~8sHN-na1B@*OPJ0y&%o+@zC976I=O{Pd-))DvF& zJo+N{a|}ISV!ekqAfTlm?=Rc8C~(@VaIa*dzf;Me3E{IulS<^A+(&GU8O*XE~> zH%utY@7%#c|XVNCxAoKPaoVq5lQ~B^XBJ$ z&VgwiM)hW)D;qCl^NCXV-jaWmmHeZI$e~1E&M&}Ee-3b)Jf76=-l1K70e<>k&h4-G zd;Iic&}rNEZho2t!4))`xuXBG`KYn^2x!~fbM-RN*{Tvzf3xW7y)aWnQs_aXl=qEMo z6ltC;egNoz9ZBmuO?w}Dlh)IW-qazE5&`sM2O+nRL$(ex_rbnxBIeoSJLK8OmeF4kI!*sB{{C3}{iAPtz9@1q`M&AzfBdb_ zvnKm~)0Vw&d!BV?UD{QVZzirFGsu1ion$v+mqrdZ?TcmhLEvWW)2;sg+^t>YMToyY z_|*wlCm-_4y;8zE#9qT=N|3dVXEcRpZZz#vxGEqwG%u=6O?ZbXi&Q0_4 z*R&mA;qqgNKXv)B#GktSSmIAzfxW&adZvp!#y{G(OMN87KHW#WK*;*Uo>pX@ZS|h< zvz8^!5c@TJZ~fF+=NR8}{ zlk16_K%ZE6o%pNKW7iWmp*V+LC;n>um+SrbV2S=M9IZbn^;cK!o{0FaQ`|(Gzdv~A zM2U4?Kjt6u0c)GTx^cJGV~15zhtkv=wRN*4PF(D^ne%t6qcX45-@h4uKYsJ;@%LAO zUyqOcg&wZon#7@8zcukEUB5N)C(V_5X2>1!_ru5S{(j&W(Vy76r@=2H{{Be({WWi& z&-lx!|60eNac!-%4?fN;r~Yddyj@%K_wk!q9uU_~|7%_JHt1+K{=N$PMH7DS)_UKt zhaZahNL|1hcroWuD~Fo?{$TqA?X!KvOMv%F!0+9%U$(|eOoLxt;sl!he)XP-D$my6 z(&F#8e%5c#EjkiUzPVTFx`<7=T#2-wdI54dq@PZ5xq~03S#QePS@DZEfY-`M$FZJY zUCEpkzBK+Cdrnn5%C)^E9pAcR!bdLqLq>0|n`f&R+h^z>Iw!uh!~53umwvY6q;^wN zpJmY0Ghv@sV%Kq-bxo*0Bl}%aH^*J`RPCUP+%ex}$27X!psD~U9|3nRXxJO|aqS^vuaIX_wHPbOUS z!B_0>3qCx-e%bCPs{V=SC71IPy?Q^H`@lq%_?`b_ezMYknuv+3d&PdT;6G2)+1J|q zoz?#&`p)In|C92QZC**88rd)4yR+;!#1%)+vySg?`|`)1Lf$Xg`grubqORb!FGo)` zINzk_m?)Zn-|RU?-G5ta{k_@*?bLx2#5W*6*uok`=Naj@s-*s2 zZ6aX*Ss8eHE9fi6Ze+*nRo*8jLgu-M_sF?v?;n!$Lw*4J2J^H1QGf4~6ZF%mzdZ9i z>M~DTe~)d_)ZhE$L?rq(^MOy4^7TD51%V549Jo+_4?VJr`?O2+FMnrU>{ow2;iFfO zq9t*Jxvt9LJr&@1nfA8dws!X^U8f+kdaL_}?P#T|Bm66@&)gSk_1o<-K6qjt8eimd zSESB1aQnRIM_I|ouQQLetdI5Dzvlw*7lOwUcEc+Ee_KD%;;+4a;;{)2J=);U6}!F6 zNBS)~g1^t}bKJ^$f}dgV?jkqLJb>rt#Ez;K@;*k-ZoYqHLWx|hqnBa_GW~(S&rLM= zzC(Zc65oUG6n`c^oBD~LoA4xjg!#nmGacbC^%J4%CjKJ%-r^xE_^bUm{3Y#OX%D}& zwQIqj-Oq-%j$U!H4!#ZxztR6g1^j{VYnpdbc}C6$su#Kg-rOJGtMk$&E(ScitHVhA zhpO9)4zs^B@$G*_+Qk|#^l$FH?-u?Q{5SdcEBd@;&wqpEK=i1b;9uxg?!7OA=N|4m zbBU*rI+u0v^YnP&<0;_}b#2@G zCxvcuJ@8|l^_c#^|5Fn+zVFZ&JwA+%sOXR5AR_gO559W*$Ud1$@5xH`w+e~@mkIZao9!p zZ9VM*`Zf7knPt05Z@`{x>&=2M;0OLpd|W~BsTijXpYg;*pwwo;r@Zfo&uC%-JlTAJ z>>%>E0z6vrDe_qGDe|}uy*lMn(!a%LQR9hbbY}8{a&u3(x7@~jSdyF%TJP6xez&FX z<`bO5*H8~!@RP<9Mt?uS{+OOS=Sn;h?~QzYeP#4z@jEo-s`~?xt7B`6@a=RyyxS8# zqg_MdupRu|7t7T+kUAZ0e7KIhz$$sRz2FbLfxqvG+_Zd1oTR4fjP$Fz#BcV4M6UXt z;~`r9QO=?2XL#=+2b%A{FXwyWWxb=kxBbm@_#Qe|H(a{E+#ugqXlLbW zFY{oXzvmAn^vXui)(M(96MrD%g1;1B zX1>bk!QEZ!61gjXk3>%UGbaPiy+#l2?h?F#C5!J4?OXNWw4K$XTJUZ8)$XIJMPKUr zw=;IJu_{&V?Uj1t%vaA(?K_}qvj6f={JCydA&L9$Drh?$c_WoWu_NaocTp(%m8?^!sIsk6+1}8$IDcg#IJM}X0gArUZ-^RWSv1XV@l%cx`&J1 z5p>^OSkfowsIDU4A!n4M_>OkHJOe-UrK=~Q7wWSY5Bk_gmlpdZ4)IraR^PZQWVkJ( z*PMhdd48MJE#x=5C(d#nyt(J3%IUvLJstV`hkV8#`u)DJ2$c-}KYgLg@9>BF!V-%w zeXx)2nBQmmB;P@)PxOWKJ0Nj2HNIPy)Av3(PGiobM_#1--OH#n{$?F@!x(D_r+iTt3P_lbvnk$ zsEuoqdOLGm@>syr96u*{KFY=;i9Ca^&~$&<3zmIe}7otZ?b{; zxJ<{?6}0!c|Frzb7y2tZ8dc=5i`-$Jnfgl?>Vxi$aWo-c;Ceywz}!n{o!i0Vo1 zL05+l!*9^nL-2to_5Ox~dxRgCfluJ1)^d&sfd4u_@yE|i^5(lGZ{CCd<`U0wQC*9M zJ{Q}51|_dq_{jD%DESx+R`O zzEAI0c8y;@(f`zaq1FS{W#csbm6IXsF8(d&560>FpW2QIwkvQcY4qA=%$k;`a66~={i;qAN#~wJ*Nur*g%}aX7o}m50MXP-gbhI zpB8;uI(*E`qm>@H$LaOLf9-J&?i(ig^X2bZHsAd-zh~LX7v-p1WuIy8AHmN~LchPi z7XLYN{o8BtpChN$F#dD!IJg1-c^QA^2JaS;yJ}rTe0sO~r4e7pYn>P&&%MasH#W5W z!REPl`{chw;C=Mjn%oN!E+t+NzD@4|UyJ|TZR+LZxBvXfK6C!uru_ol!5 zjklkKU+jDG+kfFC<7Pxpi_z=F-;F%_3EXe|mPz)9)ZZ=oD*OIth5tPM+|l11{i5Ut zOuOIsj!E!h^Ucirf(Pd*Gjql;_s$E%@6no7avd`siGGy z#@`LxGtL40hRmn4pLWM_x9O+Nh`-wxe>e9iR*+wP;%DT%dlUC4%Dnr;&nWrs+?yC6 z7y9sbyU@iF7x)5?o#JQ24wL%3YriUTFf)Lk5x&gGy-`*9K1}YFwRV-%-yLaqWX0c2 zKkfeR(yvW2-&TLO^)t5LZyf9MYt!Ft?l)eCzuSe*dy#il^es~_*Ih678#B+PG4~sz z?`;6SCFXlW!>yKC-w6J0=uz#g;O|C#<366Go+SS6x;{5*dRM8xTkhxO!!?qhE>~ zpBKHp?S5nY-Mm*Lz|m)Y_8zg+U)%UsLN~7>Kl`^P%i!mF^0T3bc7JVU%VZ?_p}Oe4 zPJ2JE^>=rlzh51Vn1AZ;HuoDZ#@`*&zDM>?&e<(>7rLeHLVzB&-Y(yZrAap@!72He5sg zA>Ab22fpVwk$)2J<0<8ucptTIJNi&gy^p$`2kPuE^T^Kyzt&IDmY-|mO4{>t-zWP| z$G8$}w`%6+29pie(Vn0CzDYGL&v+O<37N0(Ns)D!`XpK56R5)SNh+t-jD;Uj)Q??5 zer^r({kyiQ{X%b!yL%G3*ybOdPkwH#yFHIk@Cbj}Iw#;U$s@FKW~a%|P4QUzpwM-G zBl)0p?0CW_b@l!D%lg-!)p*RUKZ~4Y{Ts+5 zEWzJ4kI>X*c%^xSHqI!sTIS7s?Y(A8i7)mfFLb-w9b(U1!u=Y+yH~!8k>kWY=zBAk z9^*UKOCH?rkl#7nBN97l{cIZ#X5-TD7#4plx1Fq&d*!sBd{pc{-S>MRmH96#-15q7mk4~~~JMaEsb>L)0$Ay1X z^v>*8sQVjmzu$0h|4H9b-!J^_!7l?JzAp;=vvLnZwOjQKdk0Sf|0cY=gTi;{b4fo^ zKbUdT=O=hBvL6>_e)P*l@;uY?b@6_&Q>nuL#Lhy|A>l(;>xD@_o2Xk1zx53(=FdD# zzs#q$oB3s{x4q+tNAuO?D1@)n1GQR}c?`<=i}_`_S3&B>WmgcF5+J?hyHLKX@E!W} z(zD!$$bP<#?@GXB0r#lT?@pyu97;SP^(o*Fvfdt;tbt$kX{~Rl%oBRPxgq=$@aLU7 zqqps*-|ubvo&8^7eZ*PobDNs=U-Ws_t-Nvn3-AZ`T0D4?@$7vT*f&2s1^v+e`%)j2 z-@lN0%Nr{n_^Xr9{r9(h;KN^mek{C{6a4998T&+PhlxHvNju9w500>})Y+%@hWh@a z>>v9zxgRLE(I-A1xwwpbhAQYsvOe|?Hb~ADj(TX0*^gS=_0Y}(4zzO{*r9DZBX_S7 zc^c5~2=yn3L&{10O6CU>*7k<5BS^{hTZjJm72r{Vz1_2maHlv_Ycba z77(X&0Drbm999E3j&7;xypSEwgQw>AdC5y+ow>E#8@5iVaHK^4jv5`k@YHm%!t&e&5~k{nfILi2g?FUbV{aO{}K^U#zKZknL65e+>^;tvoO79%fpZ#H zN4*+-U!cja%g*sGzAtd7PW@lazi$+Nv+{u7-`oX1tD~D$D0o6W ziMyG!PH>mReem4mbM?6F$6c9;;1LCUuL&aAk5xbR#xPG^be`+1t3QdS4CPtLGv~Ab zKBO*h=+h29lRWPBJn~(f1E7l`>`?*oLfv_H2sJ$>c4MEvUnY;7dko$w_YizT+n@Q~ z-uFIGp)NWd+56`QO4LQ~CXf7UUp?vZcME637q+fqdmi~8sL*5Hg*{pivvpo`Qs=eA zcqU&bFZI#SO}}S^`ie+LM0GWUOZz+Jk$c~ogpZqf|_ik-M?j z8GdqkQh~6&W{qZ~L?F z>8B-+-23)q_+#?O4SnPw6tOSnipZ}B`Y`hC|2KK$3le#qlRWYmIoon>Z_gu-k@E_E zL9V+>{nPWvFWx6CU)ue8a6ad)I_GY8m(Kgn^?TgMGffZa*{P)NHhDST0iU|?{OY>! zZdu!*RazI`*LvRIeDrYm+!g&G07)+ zA+N1pqSYUfmG6A)1CE!OV*dDPGcS1THwVrs9shSf`zHE`%~Mq)*kRZ&6!KW-kq>Zg zuI5sIOy1nDQP{A>RR^%|TK&|@*_J$VJ5Prr?G*e;{!=Xo<@Z3nS|OfjK$iDr7!OJZ+au-<#*$fbcQ+aJ<44dII@pg((j_(a$3hcv*s8 z<`Hisa3}*V$D1()f2t)B_1@7dOx+R3@voQ)faieJdq=;QJaP~IT|ypt!1F*7_d=ZM z6;r%#)_Z^E+865Vui|I$m}ih+0(Dc{2ZUiOQXYevtojr((`AKK4Xc=*B*C z4)er5V4uNLF#WmpeQWzmKihHC=pU2M*`>nyoWtRp>^kbbmpE4}A&;E>1%3U77s}XS zw)^v6D6`(f=&vsH_fSF~)97;k6nKGt=TCtxyKWn2Xz`!Y{|4$j@Vy$im_IH4^Bd43 zw+kJ?&jHM*Or7qI-=jw!nC@S-c&_4aR_doE*Q^|SOu5?c-73$PZTALWXuzj!dF1Z$ zDHpeh1AgJ_&Uxg?cmLe;$eVIAT0Rw`b|3$%Sq&GzWGeTg6;loPJeBX!rm0x&wZD1e z3juaB^BwYT;}qDp^ivqAM>V0pep4r8I6%L}AKCGo@D%3}z`d4swP|$JpiA3UpmX)S*6K&caJ@q_cL zhVg@=&saZrv);S$gQs%G*#E>29*W&aY5gP{;5QGEi$(A!{n`3E9paY|Ko{kat09+r z#OSRg_13!8#w`#3>PZxVTL zze((j_WN1VPsA~6OZH7KxXt&QwmkEu7x=!J2bA!A zw&X*n=%u~b*IkikL)eY*^A{^o4S8(yoYOq=we&CY@zyT%B))f|pBMStYPb2U#Q8MX zcbCQ4^Q^1uitpslmz(w(;uL>wiuE+>#s_boLNBp?%l@T@wO>5H6u&s~sU-HGkYNV# z%hcNam#@BVe5|UB@31e(J%&WgV&@iq1uo#v`d|9VXV~X6uRijhD@rZA0l#{cdH0e3 z9I*f2fM31J{?!-8V)ttL)uoOs`$_7jsk~LMcYyqE_>glzpyf(Ozj^|X{L*vy)zQb= z^2i%+75PQo*(5$Lx&CBcct?xqvDPmyIGHyy78=sy83^*N5&tHw0{6Ud;&jHm$)^bGV7e6?lAIo zwZtWWznzh)y7;4313#XDUrXKiJ z`kx_kQsh^ed8|c0u5_zKYwF(7h<pDx1udU>y&XSt-C4KHy)?T?IuI(-AxTbea zF?*}e;_ougRxh^C$dN$%**d&O&VsLI`%6FDaZ-D}v9sHHZTRoi5$K@lcffv6T^j30 zac5J1T=?94XX}sWr2aT|!PE{gdklXI`#@WN#%{j@dFDZ<0zX&m`Pl7`W$?Eu_D$&7 z`h7Hf9kroy)xQ`0-5I71xxzjl-Wh~7=;Q(MQ+0C>@Ycj%*K7QB%kb9~oBJ?vf7*|h z=8+qFfrZ2Se(x>n9`O0TcilxDbJlC?m>YaG>!GlZBUkW)=sLGHPQu>b;MzC|%P+Rx zazFB>F7Xtl9aF5gqkp|0c)*XS-^Pv_ynhNhaj*wG5&#$Tck)JdOvTPL|9YaY5igMV z*Vl|`|9W=KSW*{dt;8__uYF@m?ELS;kD}uwhKZB#@v91)=|3lRDe63n+>mwJb=Z2# z={hRAM6Z42>xe!y#rW;(sPCGJ@|RslPS(+2`S{xxRhXAuM_Zi4Gitvla>4(PQw`); zi+x?K!LCtF>dVxSx4(X0J^P^;|DmkE#{MC7+ymk+Y@9^5tL+|YXOPs-Nb9&)s$#Dw zs2~cVvqDh?V%Jip4sn{_l*Wlfd#7sbYi)5N_3D(z*xK&_eufotpYdDr67rIlP-Fe% z`7R3ey0o9b*5$EvmZi>%j)%7Lu*JU4`4)e%u0Q^^U38XJ=%>IdTU3c0H1oG|aCqyY z$3-8sc@An9a?j(rrJiwjJ$U>D=$m@W#Epx+tMZX4`d5bw*tz&M@!zVSwbm7W=_?O^ zrIc5%uRL(@g)+NTcR6Bt@Kq_2`>fDE4^H{i{a66}>1Xo-{BUO85r+{e`r%c*xAJ9Nq&zU9P?Dx2;`@ zzpdUINWubtiCpOo$UBFh$>S3Hr84;y&AiCkqf@-({6w6%)R!=M9ZW^O?^X|ax<1b` z=(QyL8t^^wK1+n3pm$?mh7tmiCu!cCqJ9K?lcQJ3n=1^$UxDy*{c|#&#cN*r_c;Ga z|Ijad`KaJsIbp(=(hk10><$bkmYwClk^60Qsw2UBA`S`E=J`oKM%)RS2pP z=kc!Iv%72XMLN!Q9pCw?$H5=@kr;huKI70IbfEM0rj2uMx6orc&SBQW^Q&FY(!G<;Sfqa=X;kAUZC0s};+S zCF@%*>jPhCBsJu5mnx1%dcEOYV=nn2EpR12@SD4Q=<*)$7qbudhBHsaS>iy@Z<(*P zQ&G3TI}meTO7Ghup1#Pu*VWZ$0)E$W-%^O&=qjiKA^u#k_qa?0J!@|azjnHB%f{nq z`%?f*b!C zbXAV`f|r!8dg4Ix?A2xRLe*ViX>YY#J#Ox4Cw}(trz%SB)wp&y>rg#^mYmPcex)4m zM^oVW0(2>PLzXVj&w)-|rQ+FT^&s?}=@QC;#yZ+6le%{+Q-d^ULUq`#Lv`fb8 zRo)M#6!>{P?;H4qoA3Wo=3m$d-$S>yKkBpmV5-jd9r`Qu@_kGmkm#-STUf_DpqqlU zV_l5*viQSQ34gDPwX}uqva8r9p$mKNppL+Kk(bt9lwDPNXpi9iGVPsZ*yoS~>Gy_i zb8`p3_6Fhz<|%OLWz-W({f8tU^9x1glY40F`6;>vd7#^c|3mw|=MsleV%^5h6kk)(^0rr*_X~Ir z9_{&vw6Rxd+k06ME2bu+9A{R7WeUXPNKGee?^>ex-V_pU0y2)~-4ei{0C= z@2<;uV#kC3j9&o#|C%%KHFQp$%dC2Q6?}>OrtZfzX9BVJh1Z-3!B-FR8ve8G=tt)X zWqbZ0lDMqqeTnQRA^arwF&2Th+?NQwrSI>dpDOZ5-uujB)q!Uuzqq+?dN-bdeyyBh zoZ68Z_;kYS&lH(Q%f7AVv2W2{^?n=s8gUZKWFJ-P@%7LV@KE>}j#b!4NAQcp$@8aK zZ+@_$j>&t|{)wd(CI0mM#xQyW`J`?LqiUIU;`fOd-|Q>YBY54J2zV_3UhH2M9@rhQ zE9m*@`(=>xwIhC(JPGf0XNqDMb^Ff*z;}K{@4E}7{$)+uU->)z_lO<Ld|ds)ykLgc&T3ouSoZ#bm)k(|1h`=;0zTFx8UrRWXd&shLm-K?6i zl>NI(dy~grSh6d+Wi$F4_rDyfL%*fUji;)4)`dJ|eO4ak=7oXwFPZ(#;#&=(uSAU7 zVt2^S--X_QJuUNI^akYqeDoEO*XF#L+i;?O%K`NKHDe{oPwbO%%dD^Wx4$!84(A8K ze;{;{?5nC5J34#|J$34SWeb)ETVb z{1#r;`nI*BtHsHD^Xv!uzGj&(bkdwJa+2m&?%uem{xP}7sP{*eOU^GZ^Q+TdRg}ox z6fR2>xt_wMByd5#q;R=-oEciKvR}2pVPwln?^K*i@lfcBfCv4Q0>(oh_wEmlxae6P z{khZd37s93@9=Nx^Ne|(DRAPFF zWU%WPeighUrHK1t9pw%kQ*yt~tQF!{0iUCf)dJ?}c6|;%4O*!AZk_%PE8Ggncgd=& z#r^>O&Tx)-Qr}$NNdFbi8Ov+6gT9lto3)N{Tee<$}!MQ4iUKcJu*ErZZBh3#Z!VGyI(vI)PfrK0xt)yz@q>@(6@ z=bqoV=j~6I>5ndAA;O(uQ04s>mR24LBib!Pp18=dLG&N!Px1R1@NVhc$`ku6@>cku z$(OZwFL+A%l5@1Kqi61Gw|w9FoATv~T2)k0cp1BVCX5E$_X0CufXJu8Nz^N$p zDn{RW%9H&)_?0vKw$kC!pr{wbyzr*qe7elJ zPv#3=(L=VJ5xUxmcnNqv6a?Z|f5PEhkj#Ve;O7jt}XLYqxaN;D4WU~%Rg4$ zK<}A>iLCN7u8O0Kx^`!I+lOjdb?rT6ZmyE~4(a_X*L942lV`hsU4|Y6Z+mL8ADI2B zg&vx85NrC!K0Z&!?e3NQ&b%5}PrPUyGFi7v?S?-JE4DiBt*q0)(e7UtHgNv0fFH{z z>tc1xL*8ys@09ws8T_(gg#Ei;>{7rd_%js(SCvWXXQ3@_K2ua`W*>H{Dtfutsmk=% zFY;H;J?ibEmo6O2{O&2iiaEDg{Iue6;ZU^g6!c*6TtL3;DPi9ec^=dM2IR|~;9s80 z{8~As*E>zVtvjptjRN&xM7}lg_??>EFP`F2e@D9ws%Kyx`r#(zFXN1C+4dh!Dd^jd zvjIAVFYXvp-I34wQhF}_yEAoGW8iP;*v5^sTXGQnx54^bkXw_ycj%-5WQ z<$Qav2w$7?uGnYAev#^_MlVYB)YkLva9PK<&|b&$rsqd%cQxn7Q8{0hMXv>KF7sI| z`j*JSZGU)*=Z^iS`<2>#2Dqyl_eFwVht#z*v7BcUyIX4KMUOcKb)X-;Q}n2m?^eTi z5$kT5S3&0GvHnlduP1h{q+jfeUq9o+_eP$!t_yj{d)8yt7f3(uwDrwS^vO=^68l7= z-v{2&D&xD@$(eW4jzpXs@an9`R^mO-qq?hH8U+qkKE6`@o{`kilF*Gx_Tz5!&JuLJ zlQci*FO`?vFZ^N&rFm7P&lQDLqR+xtcSY#;&`Bu{q;B-1>(TF-PsULnJ_D9jfpfm# z%k2L-+5crc?3JG>LX6z2BXTjfGf)RA*i$TBtO+l++YUxSt-)+p!_PvuhM#X-kDiY_ zV4vx3ltd3Na}LU=pk=<1e~&~(UZ60XA!;JW4$-lylPDQ8_QZR$cbK1r2SxW?}oJl zTfH$Wdh!tMeunQWoNL!d=bK*T=fj`4CsedA(1|QAgcTQr%IdX-88jyvms+Tc`hB@CF{2p`YC;a<6q< z0|)c>t8dBtw=)szyYQw@J{=*4%(}3@{ns;s_nlEt4e7tcIs(2k@N16mGkz1U)^06y z)^}TzE^VH=#gqL`t!DqMfgfX!%@(nHgy3Nrc3Adz;L+U(`Q6F6(B;?gi`9ed`mXM} z^)JrUn8$?=9(_8PzK-})XFO@2=y};8)*<+@cGJGi*u!|X`^9YH=*rOV66~lU@Eh##e-tYbvv2JJfP0Y+|&82ia4;%Hc3c2gW5ifF&CtmM3OwVba!uzf@5 zRgp95h6?&D-z^QP^bDoX065a-4(=jy@3df(5(M|}@5zVpkqfcfXTJ_dh7$HpII zzXQ+65&BiPUvXaKli{bn(mju!DtB=n9ewl^?c3=o;eXDZe|;us@^h2kIA7j|-dOs^ zG@A4e{vVr8$I#71^xHb0g(daasXFc3=-Tk7`Fp#(P{RWosXroo830GK zKDFAv;(WEqcUSMoJb4EA!e3u}I>xRGL_}`DUnkD++d99Lo^N^VOJX0-UdVsxX{e&v z&VN$usIBc%Icer$?;Wyo(&}BU`@+>-nWxXl`Y*icD^G*R<~ZS(#eUy5&RXa-L@q8r zp`;E8OVIo5!d0SYA-~K%X!S1mx2q9k)#wfEr_k3-r+QNOG=-Nw*Fguyu8*dp?bxF? zVy}k&Q$2vZfV~mtQhQDk`y%jB%ixD1{TV*OeysJq=D8#}=L7GtGalP`Uqzqy^YhVf zs_glW3O}LuiahneOG~@d?svVM`z6la z*!_m8fsVH~c?(uv*!idW=LYn=GUEhN2SN7Jdi_9*p0S4c`|zL0ix{GsE$0T#5$U-B zc@Z)nD=)0PGw>n4MQOi9sz+jH1`D}W*h8S_tk^@`CSP6s=&kXYkiTE};F+iSZPH0R zC3>aW83$Dbov%Z_ihdyQ2r`QOj;1HzJ=s(f^FS zx8;0~of7?z?}i$jQ)IlLw6_EuxPB0pk=LS!m+>b76CYrERj z06zE$#V!tqtD_!pLW|P-7Ihv)Z_e+ms9iG8w(qLy6ZC_JUob;VEl~Ze^2&pYln!}$G}(4$;7{vIM*7wFLa%a z_xj{}Sh`|vi3CNXJ~;h`fYMv9`cD#5r1WZ#~EU$VLd4xhXjv|(}KrV`bqJ) zf&Dndj=nzjjs~=YK1a(rW$mNbmj@ONalh5((iQb*7-v2EwcvSOMd^67owc|cPOV*hzSwW!He>HDiM={<+U(+2?)V1x%{9MsjU8mx2K3%zQmytix(@Se#Lne6 z?Skrf)vJck$D+6vc#gVqaaG%kUsH;)7iZOLunQewA1DpnuMSpuUV{FAJM+Gg@0r(? za$bnLmHmE}%+Jj!#>p%$WUy}+v2P2$7`G0++4H^OYd#3%+-vzfUH4+_SW(l?-&_y+ zLvNFhnTd)O`>d_+Wcpn=;AQF#>q_l7Qs0TuMQ+jW)m@L}$L<1N7xMWNE2qmXKR5O; z?RHkN3j^nYC=feXIxoqKglIf%auO%#Ufm1{T1`?$A5r-!Qk)80r}ApHF{!(s;{C^rb0Gg}#>p(EZq%C1(d_y*)-j87ZVmsX+#6qk z4mZ@j!V5 z|Bvv4(1F39^&7c@#|ZinemDs-m#ejlqZQ@;y5uwTkKFoIlfqE1@O z`fieQH2S#INdpd6&rZ+N@N7yw9oeN_Y(T26ZDNeEPf}|KSsP!-`#IqB>Q?_h2NEO z-`@xN>)(}o%Wqy(zlwR-@zQ%d$-m-!(+(fx!e)v0i58CuoxkFCoHsV>bH>+pvyKaQ z+_U7wRZV@|p7U)SRVrr;{1y(XCH2uR)i)Rq)sM@1!7uw9c~nU~Is4o?e|w%v@ymH+ zjD6lG&>|4|VeU7bPhJ{$uym1=??av?FYRujZ{$by$H1xMILN6OJiWqkuGmRE?0|LI zbxl9|CPT^+eWAZ7dolAxnZNwf-UcIi1(~M78J@x?Wwf@1D_=J>R?fI*u z@Z&V?HJoqOjNPpD&znx{h3o_#><8%c!(-c^-<=nq58+s&w?4LZY@66c@@r3Qn|?0x z*Ni<39^2ZXC%EW0T_-rFFkh>0oAX1e-<#)w!QnRxU8Lvx7CSWaNaCDSu5qN4Ltd(% z8@qX0JX2uxb>f)>-zMJ4>P2c(gkMSWqm3Uy`bX|ryJJ>yeyK4}qkqlZ8EQYkg$0Q} zC!1kh`km?2-Y9Z9J%7yi>j#h@x$8L}R3uNCb;nB0kag2vb3F90D)2RWnAI0&?yRbV zW&E;j=Z+5R=wlrodSRz^WLSp_o!WDk(IYRcm2(q#GWK!o+ixCwhI7Um&QAgtbAD3B zUj=;MJoW(ikoADa8C~aC5AxR9UpCgYUD)P7HrHd|Hka`{ffI3Nn!e0<*O%)Dko%*v zfma=WSz8=;MaOXmavm*7T$cNxvF`$Rbtif*^EZBom0Rxmh5F6N-=3=8pIL7bXVQaw zukjuBRjt2Ux+Cvj;?mWi*rUL^wQnumr4E2KyG}Kmb2a!#*Vz)clgjayb5UxaUWQ(a zJdc-#iJghOX?>1i+%O^zeHrH?AH8>JpdOTc2>ZUoBd6nEFXKbM7S2Dh@e{ag+OlnP zeH6JpB7PO(>^D`_g9^BqIHIQCLhNL~abt`EmAz!egU?-J$v6)a@t;Tvf#PwMG z_5LVK{0ZI0E@zA|1IaKnfg3M`=OfF z>qLJLKcGGL%#?G_L%{oikh*7(t2xiucp7`2A>M%cw&;1HKl{k7(Ff6|9Ocd@-Uofr zes9kgGbP@KbIsMF=Rn`;rpOE6uUXFFL|^PePY98#CT?dYdXm%wwE5pN*GjyP`12P1 z8##}u9_%HolX^y8Smk`&qtJigM+dv9$fqu)^+0PkbvfyZ{+I7YuiGqfJSy3Tt$qb6 z#ea!?<9EjPvVQO==O}e8aPW~&v*|Ac&t`rF>CYv;Qu-sVDDP|qG0bNv{#o#4^~o9X zT^Tv7zw111I-cBn88WUD{qERt*6AWIT;^ltMIZ8_ioMCmdFzi$<*}we+IOJ83*CnA zx-8!%adfG_x{6NB9l>bbEavtJ>)EIbWKO06rEdqDZPb;k7DQ=hj*0#@ye%jFb>F4ceoUQX({ABl&wr>t&@J|Ht+!e^07R7NKkeT~@2#`{$t$5t@HsQ73t^t^{5!@2 zpr11Tyzpn0_NM-{JAam@V@sbzSg35X>1(6Q7Jv36X}{~sLt8uPf2|KJJe%fgL( zhn-{C0|-CUcFUZ??&h&xvtML)n)m~2*RuNGfXENw_mFs6rS4d&W*t{uXYc=$3E!$=_&J20EI*sSr~Et{{i+TeJLp%I zZtSm(Pl#8AI{v?z58$(}g4aRly2QR|@&R(I!ur7w_!$2>asEcHRDV47HPF-ZB|ZN% zA8ECm&zZLqd}?eV^KQ#4s(w`K?IwSX`i7qNr!S!$e9rmtqoU_2{8tj6nx7ZoZ^!;= z^N8k=M^q<{$gO(Ekb5%*ut#XVE*=nmPV@_m{|B=sgE`jK{fB|n1speL$9 zsT;|;L-@YJx=p@qcIkG1cTwaYfA@6U{eK>Nn0OF#pG$rzc|<;VxlDV9`$8p;DE;2X zlP@5Ts8{?RiTBh>Td5De}c%<^q z%2_Mt%=5}EH|I_|(0OW4Ag}5;bkIRAc0QK`$@%Xxej-c1O?vhIa_j->t)%&uhAvY% z7kzQ;X3oQN%sFW`e9*|L^PE>+L_Wov$eUE0k4*f`!n+#oe%`m!apQ@xXRudnf)D&@ z`$X>@6@Qwu;HN!>T`QX_p z9~b*slb&l@erKfLD)GhA?^}THwKY%YQ>OQ>C*z^tJSOy_eunok{;KBtM}-dy>uR3l zmD&EtSA9(H6CL_1^zgkuO->??j+EgGY4;XwH-BR6yUeGfoNJ|L|HRnCN&c18H_M8g zgugD=zDdvKdrQxA2AwxdNPI2y8n)AODtB7w7C0RhJz?4EsBkvSkZ&18HT2?M6`$1W z>5p^M{gu%**y|O3Ln4-SUe^X!*S)s<5=*{jMflsiH+hcs{pEf`$=6h}@5H;ut^yt= zkI}Yk=36HHXYtePd`EMyf{pJ%|M|N3^;+Y6n)#OQ--w?+Tde8vl*x;1=3B5dD8pP=@VaL-~G{VsH1U4 zEtunb;>S$&F(ao2d%iJIN8eO^vqI6I)Elq*KoChBPO%|M8m_&->il`5Ny3fPaZ^FP=vaywG#+_f9O#s==Px z_v#Ozhjh)-^*G-$yO@=F48k9w$9}f6T-hDM_ut68{5*2f?Jm6h)Kz)4{Aa6;{lNFR zCG~>jsXO^S>S$5useW#3Zuo;YdM^Ghd?fb3I{s8u_Kpg_V#obNL>&V>hjMPQbwrx+2E;WKIcL~-p>@?j zuT==$omtSY`9Iz~=h z7?FJ&dErDC#%>1Qo5IQ{{5W${wR|Y#?*rsld9262w{l|%a)b4#CGfAu?=_4AUWu3P z`s!6V^;qt9>Ie`=eap~8VHv+`*T#nq`^QH)LLsxf&1r_H)`Qood@>&9&;CCP{w!iQ;Al1(LBQrAEe zr=`i7Iw?sV{Ql0&+*xVGT0q+O`9II!dPv5zGxyHD=brC-&beu|NzS!-+0G^v<9Lj7 zZNR_f=q-x+U6a~*b;qlh;IPEsSj=aSM~BxlzMgDK?7!y4k0f@c8F!DIvnKnBz!yEE zm`|=gIE=mV#4OR9kRygOZFr5~UA}!-{0H?mdF}M)=X2|GT4+)|480ZKBYq8eL7ude zWj^ZL(X8ZKh7J3y;eNp*_Un-|7s(lj%2g?(i<=VL#^j znZn@pqZZ$Xx+Jb*I&{4qI_H`YeU?1jRL0@EM((^}^oD@`eEOcQZi2pf?&Agc6qor; zyMp?dS$=P0+|d0bdOxb zjk@^4I5oa30AF}-`U3Dp@F&2RM0~i5tSim0OYud<=eT&XS@f(YW+j|Qb6kD?1s^&! z1YeKT-(hE9`}bwTs13h1r)kE&|EDfy!fD^-o7z>k1`I+qL|hyM0OU4Bw&*vp2yUA}qA za4lkG-_Q$Emms=tD|ywxrzzz;D(!v#snkeI#)RGtwR{*qW#tK{qaXRC^SPAa@H=+O zNvos(IPe#JkM%>}1^yyWt&ZWNzhl8H4 zo`18s+G6f$;dkW-GUhT{A`|=cg9DBi+Ly-mWaqyx%AK#Reb38RaA&wS4E_CV3 z`+360MeeG130+3rx~d`g_*VEh>paiXPgN6f*wBM&@Zf_Sr(PuZA?uQ2{bx(vGU#R& ze4Kk@HTVzxPoD)JPtec3iCh{!6`3b|oPO-DA}>V$%v>vaLAXuM;~~CQ^Vw3ltk;vc zW7j|0!g?mrD}(j4U4LkG3+o9T>N+!eypz_Gb&>Oyj49VMjXb8mvhp~8%`kRSaXtSF z{fhch<<>Kh6KU{KWO*$SU!e)>EbKFfMqCza@#dOf473#=#c zBv)`QLL7QCum4a#>pZO|k(Rn0nvQE{sc>Jc5`3f1dm1{P4Zfw^{UR4HTF$;dw=>E2 z+T?fa&{^o6tjD~wU-jN!&F=-`y`(+-M7NJhU71`};CGKoJv{hjF+T!)25#7iuKyxJ ze#BPjEpnbsJPLMJ+Wta6`lxBWyY^Xw`&X^c^*2K|-Rn&`k3jsd83(@9lTvAZmO87l zp8^FLlCQ>rux6U^;s;1cd^&NnDxDGe%zH`fa{s&t_9@qFFMtOx&4ABCA2li09~qK$ zU|x~j=Z!)Z<7GDdp+G;Y3WZ+iMp0K<_>{ykx%+S*v^fU=dte@ZLFm0g_)StqQ@C*$ zzlEBW%uBpUq#2uFM}5C7X>5yEsrY&6?9K#nrRRk0#0mGg zE#p+@(VxS38k3QeF~(8OPoaJeKSgg|X?>#qsNkOmkKlXyzDK?X9I3k~`b&k#i-KE! z`J+N#=$RiCd@kFb@rzxo;}w9PUw_HIdAL{bGWSV=L!uG5nvtM?!>~U&yixGLdw!++ z+)zomG7fE5s_vs^n)R4>PS)B1T-9Pb)}6zSI4A3#=Mi0O7q)@F+$VRWq1(mD{DVr? z--JIhuFZB<c0+UXT1ozGV0c`8~GCZTPL5j}hos%aMx-emB%-hSz{! zbG9aTCgA7T1yYYn`Om4p{psOa-ZRzWOoH(_=nITPpXY(!jB&7jJ>L26kwtz!PKRXf&$Y*&^@*ea!B1!Qt zxOid@_cGoj4)=70x}ejs`vKp|RKG3yjz0HTKUcaQ!<_rw_=jkdP5cAP)au1aN85<+ zc?3M^-jcPETjrdToNEUD`0queE>0f%Y&CH`4X*zuad#g5>Q(D)1Nc`rU7gC^S75w8 zyxul9JM9>9<&^j#`Mr70CUbk7b(pg?CUqmAqc0El0^f4qmGo4^0%;acD?enH@0 ze=dGS#)E$Q<>7Wi)r()z;#>#uE1Hw}LfzxEH;R5m+IKt5Z(RG$ea;(iul;>x`w{ zdwL`B;Z6JQ>#wE1o}`uKj3fo4Yvz_xXrDTqguHp!7JIvPOwh1 z3i$n9ewPya1Uh`9?T@4`Z{Pm zLZ06hyM}>(^wgI!Pp<)Pd@8Sm&hlRyZU=At^@ujuw&WiN{9x7w_yP5Iki+6HT*!Ib z57J+>v4kJYx+(i$7P$HEEpNDqICpsuesDh@Gl84*Yb3rtyas<;PttlI2b{G1*K@qqD5Qwh(1(#>IZ-EG<05^XXYEj z$N8L}7gX6wJ|q3SPCcRS<}mQ?U>phH;@6K6x#;p)9dAF2dNgU`bWQ3kBqM6Mo6o|& zS%=!u`Z9XtoDCVNW66F~TI8*+;~W8hc4z6=$o|c6w;^?hj^7=x3hEAVzvgb>sCE&z z2!8AHV}d+yfxiP^YfO@FAo`i``yyY}dT~VXI0kiULFO5)HX9ntQVEyx|4_%q~Wgm|WGjCnS)KM3D1&X>Fdi}!S1JLkGf z{!C;p^Sx5^){^;({D7}V<}zQK_sYyy*CUY1x&_uPLOgRe#&`XCv2me?H0z@NKxJ(F z_H70HjJ(&X*DL*~OYg>3Ck;I!pB^1UQG|TYY+^FMLH<#1 z4}ZB2Z_SsW;}P+f&S(VQW)aTNe-;&2Rp!@@^f!ZFI|@$!E`b~Kwmdz-#Y^}D>Z8~z zKELqvr&{s%*x?f>HXiT*%4lez>c&Od5Je<;we=ntD{*L|y}Kg=R7I3{rN z-dn!tG4d0+m$=}kT>ZhiJHuB4e(O9r+${Wfp;rg9(cj7UV?FWd`rK#+~>3s)Tv?Zek-h><_eIt1|W`^s0K3?m1emvmSvjgw*wm;3s+7HE!r_xu#0uXWOOT zT9K|(z=QSj;~y?&W_jSD8lY=Me$H&6Yue+-+b!=IiDA+6IA6(0(+mEH;o;-#gZSqm z73uoK;ePa>$V%*=d|1txXJfy@FI(W3T#;LDeW8G^;S1tVn=uc1W_>i@WW0Ub;AhR( zsImTbr4S!uPp?N_4ZSq#?(_KjMIVy$;P_7E3EmU`C3L;bWIeCh@>7>iPoA1lFqwx^ zson5XW!OI*M(6bC%+x~--}0U$p^>3+LuDE_;&M*+30%(;HL9o zh_jG#{dp4)tI!nL z({9ezxcI|Z$Jc%%yDIdB*MvT_T~#Y~RYv4W<~6aa%EfDFe5kvO_D73$745s7jO$0! z?Kk(?ljAkYwbyn|*?vmmD(XtcukEU`^YG&}^!N4m^nBlX+(!81Z)R78U-RuM^$h;X zxpVQS6~NOa!iQLQ9dFs=)&1yO5qmh3lXxfJPk6!(4Rpiz{dgwhwY$_l3$mJ<#t!56 zpzd?j)eBAQgSF4@0?+)oC-6ue?hO&okuBUuzh^CpO+UuE84>sm_{X{Gk;CwTVmya` zAH;L`_dz^|ReJs$J@`x23-Gn#{r;*K){mc;j_ZP7C(%PSUuntbd-LG2 z<}0P~8@|4;^U{KO>$)FvQ)+x2$L5B8XAKziC-ofjVh^U&?hO2^%l7JF8}~!!*kk7* zFVQEanI7MWh`s@zpVp}MOMW?ir5T5hhsA&F`kSLuw>!At7;#d*zZrjUYydeG7dt7A zTtU9yorGU%Io<|eu+UpurA{06MZHx!c(jtdnDuo7dx7KTEOricE;d|VHvpaMxRpTO zpNC!op5d26zrzm{=p}jdDpxPAKa5-he?0wE>j?|CBEONJx}ESz`|{|`Uh6hjEI7Q8ace))Iq*By5r6!G!^rKa{YbR$!VZ@9@d{Oc z4*bsL!$JEg{74hqYyMNVpAtV$>G(Cjqy6j8qpTmv8}D29BW;=YLO*!%7T3W*oqYj6 z$Ek(`E1ua!z3he)Z39QaSHEuD{>Au_(iLhh_LJ?ay?<9~eqr ziCj;{4#&{4qnV z%Lhar)PKT_^Z8JNs&LR3D^KuVx7&}aM+b7i-yb@^Gqqx%3;j*Y1HJ#$h@Q!OD~GHV z1JHZbdA`0si}-E?*qM@{`$wu$F#suHBUcMT=+<`BzLN8B1&%_OgJr>llZnf1TIq<}!o^BdC4)pX}-!1Dm zrJjCK!E)=ryc4}TkDj@T`ZmzVB)wYvb;y-z9q8#pqOatBTh?Kco_>+&>A>wb)YGZ+ ze)|A&5qX^7hTN?jN_P$PgQq?pTGYAYcLuun4s~LsUhL{yzxX=?YoNj&-;X2oj$Dq+ zqb_q(h2*Qum;K+sPpReXiu*nJTrYG+UPCMX7YDp-EAe~y`!^B)z2&YP#|~bN|MEa3 z-(P?AKp%M4lS-v6No$WOsTVEZ-I`@xeLn~F#j@KA?E8vck3K;AgY#?P7kWH%#BOIj z^?1ad5xFn<;>YM;`z^#z9%p{DVf6Qeg6|QR?)tSg9E{?72j|~|pY^;=@dsnzS5Vhm zbsmOK2L6n>_=y;QpvP!E$mgH!ojbY)pijfZuVK?p??3wIW0%{1)O0lIY7<|Ge80F= zT{{51A%{LJ`cR}s{FWi=a5>iwGz(w#e~h=A> zu0KwxWCL~WE%p;SXuoBA`$#-vf;WbJ;qhS1xthS~f<30G)XAYyM7dqgt+2Va(>QJ)Y`gyLS z5GrIuulC+|Itmqqtl*t@pY14w1z!3-(^05&>stHI(;bB>!B_p6}*-@A#^sC=bbQGrh`_S%qt&T#q=U4LHk9QPmBtKlgZ*~;UDr8(gmHU3Iqi}X1 z13h*WW)!mE+j5CVka{=vZHEo~NR3H%f1;?bE!`w~K%@aVWQo33aQ#bqKcn(hd(%K4 z^rZ8?wg27U=eSszDi58N>Ul})Zu^;b@>h*bwwy;()s(UxgAbZbTkU7)u4?7AmiX5y zhq8AK94BtB)W21qMDMIr=Xb8Xb6^ecop3%kK)cFgwmr~~9`ll1>Hj;M>@7BYpq_bJ z5`Smkbr|`KzW8}p|I&W43!bz9dE5l*kf;^2m zlBWS4`*|9*VqXDgeJ%ATO`n&Yehj=V?@v;_hu1Tqx2V_Gg(`Xvw-@cJq94ZW6@7ea ze-iDRi}qE~4-~YY!k;v;y|%B)_A|+!G;#dezA8J9vi>A*yl>r~wB_Cx+Tn9=aUIYP z(=T9`oNBt~kDq~m&t*StA97vfi`308`jfo6ihT?4Cppl8_3qfS&7xnc`v%r{cqRF9 z#%tfhpM+m(Nz&cdN6+#0y?nnE{-oHm-QaEDPf}k#+)KF96*m5)u=?G3>FLKT=sfTz zOHRiDR$(zd+x5QS0}A$`Dbb?C&ev2JXiQ3@hnCEdP&@p{rAGRH2%EVYu)BA z{QcpL$bao`3gVVBe=mG?YJU^$yNdqxV!Tn%ehPon#P*sWmhESfziHz5H9suIEd}H7 z;;+j3o4ohly1yy>?H7>8Zx;UgeKPX?{)KjlTUwWU2EC%KY0IMndtLqK(Zk#;vq!mc zMy*`Y+a~#y0{@PB>hMwg@eQ%3T>V4)cQz+ozs-`)rB4m)f=>K6A>tA6o6w!U*M1<4 z=ZmrLQrHO%o*$*GeK$+u^MoJx{+3Dh-LAfYqr?;V@dc7MCvh87+Ibz&O`h+|c}`zt z+pX~d?Kee4FbcI)H?=t?SKtDSV{Vc|`eEm$dWOHfY&A#O~ zGijyTH+27SfO*F|tsf4wi{82RhXb`#IkMjU!)JO$U-!PN{S2#bU6T{PVd=UyUlYy` z^iuaX6wN=gi+!A{^X@**6&<#>ull8(b{Wf+>pzr$o#sQple4vK2*tsHyJCMWJVd^*D zFWxsg&|NXLAF5gWPzani?T3ox5BIXKFy($R`_CqJW$Au#u>UOk#cW%b>=$3yX}&N} zOB`W|-mmvp^fhVy7f<&5M2X2JC}Z4)+!1A?>LTAa-WZek%K5LVGO_%l0$bKA13m zEf34iqr83K&i6O44}SIn>#phT&0e9;<1h5$2WdU<^Jls%i9b*MeBe0z-;V?9TZKJf z;bv>gx_Yt3BYh3{+k%PqC-J90Kitkavn6;|x{s~xj0-#MUk)6{{`Kp1EU4$c7di{} zu|54<+ZFoz`WlR*q+S8|P;~^lD;>w;E0S)#f}b?P7Yh^iw_lTS7wwURcOTdz{{3%k zkF5Q*z+JBw_Q|isPAi@7+}Ur|9`WK?-?%-pFFe?V{CcDINcsJ3;Oyo{z198g$lyl& zsO9XDaq?d5p+GMaeLV-*n(XM{dN-cR68|Xs->$uKVP|gIU^{$j>sB}4R_BQt26p2{ z+FetB)?hz)=KJT{OlSJwaq66#R`npyFX&ubJ=hh2F4S3r>!Gu9dYkrxw`#vUb`ACk za{KHf{S3F9J*3+O_7HZ>vEqLDlFqfU!8NjPP-5rEywm7`b0_W_ochRq?C)Y6IPv9n z+OAMPXh~5Iq9WLDhc4W_Gfh9bu1w{Q@POEP`geWBZhGk1{tE0S>dYjVkH`fpq~13> zUVY5Mk0g3+61lWsQ+`{Dea{73bK6qlhqUG$+56V{Q{CpZ^Nz5tf!!o^WK!ZcPR=`0 zOT5&S`5v^#ef8?MM1t zdUyBT)SP54oikK^4*Im>yG`V2V869oV=W#;AHJ}2?czc7_R8G;iw3*EZ`t3@a_+D` z*ErZuN3qSg^w+g5&bUkV(}AGMs|s~tgLA=PGtYf{Z!%vXE+)skwB1yUi`m_FCrR|u z58pwbcli}|N!ND#;mu9v?SmWX*XMW0VK-kS*yozdIcGL*9Vc%F@%8Kv z!cT&@=sA+#fIoUA_FS@x{Dzmk_-x5;Suac(!U zOLk=5G1$%c=0Wc{CZWgItRxNBU&GDeD3FOkju3^YN`Q=5e`&SLF0gk64tDa?E#d+pe3Y`Ai@>u44mAoC+54+0EV`&w+gkPYIb(Zrw zC0_!&tCIb+*Q*y-8B06a3~+BV-8_zR`7Eg_(~a-IzRpN}|K{_^i-A5=uYc~fyFSEs zvo726*v(5ekT1flgY}brGW-kvx=a~oF8Gy4Uzs8=MC%dYvl~Yq#EH*if31K#VZC=_ zXDK87o+G=s_xI)SL%!$8M)qkMlhieVPX%#9OV8&>FG!(mi@Hpih{d3DJcVyQO9tA%%-Ea2vyh-#Y`bnB` z`gx`7ufFfddi?nKJ6!tn_uZkt4-EECpuhJWInMf=`oY!DHUpL-{ax+)+k3otQ|dq1 zvhP`3Km1nX)~{?{$ehQZKj5k9uUuY8u-|Y0?h%Qb(0-uO{eBGxjYpdQ`FRKB`Jc}R z{P&i-d=mVdOJ2wb^xl(7rWN?B`<;W|+5s-Q-#||*$qVuILY+4q=^zh6u?~K_=;B8> zFNW_dAur@4>wMm(l$=*p{EiPt{T-KHksmt0M*G=1SYP;XkO!jUC<3^pyFC6?lK)W( zKkY~J`B@Mr5zL$MT6{;-Wf0$yBwq`9(fKg`Iz*)(p-b{7cH4~GZx@wyk$KN0KkXRn z8Sn+6dsWOMTSDBxe$EqA@DtmupYGD_B>HWGev_<6YmmRS;RyITj(#^BISyZ)OuzqY zu$OT+5I5i$Y6kT_+~{U%-tQ%GD^ZaXd?$*2 zmQ5i~{rnsBBJcc9Ur&idsncaK=oNV{zJ>Y#ZhT7z^%io>(RTd0KhfgujW10N9Qr>b*s$P|IuI{aHl?r)blCT!ieJ`f~R-r`IU!*mq)1I)BCXGY3~p?1pHg?lW2bV2KPxWmONYA zwFCToJ^vgypRXECF^^BQnG^XytL6jnGg)sP2UX6#v#Fbu3#*Gw^~WB)$G&`I?>IhS zI)5y9iGJe!UX~AJ;RE2GpJzCk58Ua+xt*8t_yBcjTs|<@uS4V4i9wHX>(T^tukTCg zUhu<#|7g0OGS1B;KLS285B{D6E=^v%+mo+-P3?<;FPd&k{FUI@#=BGG+g}rYXmGuf z{ga2D?UKAZ{Dg&w`eb9e`d9+FQjG8Y+L2!BL=@wDz5C++iFaSzKY8fLaquOGTTrLH z*YDw<{nHnZGwvX6A^f!$k#EKQlY+GmzAER}emf4G=y;+@`zP4h1(6%zjRUf3nvR7&d-?1e&$rw67lUiw!2ZcQvsPNZdy(YZ!M_)Cj-#S3Q|W$(=s)nA zX>FO-mOI`2nTL+7C(ojE|Ky$Q$AH&wd_N|;wB?@3`zNLT;h>(fuh(T~XRD8;$TJmu z%u621CbgaMk&jXkeQGP?Z~`ymtCYd=@nekSdQOdP-VbCsP(+5Hf2yl;I!B>c#W zz0gBBJ~+ki>`y-W;(FqxR@OZBY%~5W>Lf_rKH*mrmYF`WcBkr} zJxctb@d$B+GRzOujX#dglG^5Y(jl6c3t)SZ|R7uAx@ z_a@O-G(RalSKZe?^f}!+UNJbAc~YL;iKhm;;Y)fSqJ=y=^n$W^b}Q55sUg3LaZ%v~ zkIDXD;+Z2GPqPm(xn2E&lWtz#W~u8I@XsI~(|-0yzxXp_14sH@|Ja(l;R7w#qz|}$ zx*p;^T>6haC+mP5ar4~BeS2>3xafDzKMr=mmwh=F=nFa?O8W-`yVCRjqYvg>e&X9_ zLAw_0sT_P-w+sAy>^E!>f9L+(Amgpfhx-S2xqi%MiQ{KnZoIYVQ&R7(I8X1qLha`- zyDxCRQ+IF;a?yIa=RjWt=YFmIr$PL5MLCqPy!zU<`gV-|$Hf;){`V!FxgpVap(8i1 zdtYP9ko@ab?6hoxe5%HmGO?xNKU1z>w0ZgCLqaD=OqEBz5pVFk&_(3|r{~~t&WF2o z&CnqJ0j*c+{DoP~@Vx}`V6OP*XuVDRlI;Hp{B6eT*UNZEXZBwQp>sJ8Rn{-i7kj+= z7g0Iy+hHDoJ^U(h^5DC_UlW(O8|@iTUcd=>qi zbGT(ZDe}^&hg_7|26B;1TSflity6Hmncq+IBIm_H z;WPR>9o~80TF;od`HCMOSp&RIed$NCuTWg?{J$JIE_RL=mr8z_o1a!(58pqb*F)y( z(9e?CZlULBQtVdX8u))jk7phH_oKo$O!!A*V&8%qrB3jw_~W6wUmWRY6v4Xt{vZ7v zp;P&eoGVV<(l_xPcm0=jTE7^CkEj~*Nz%xB@iSQPJ?e<;L7xCl?s?x)Iqy3s_2ctD zJJQQ@IqzFB$VSJlKTrLqsMPsZ=b%RkUe00t3*_wz9Jodsh&Q*XA8Zs}IdYUdsrtNn zGy}dYPpeE0KIfkcIc-UjI4<_b@N3F>gt{H)$s9kjk$ZKM<;KIEV`pvfG%_d2IqUph zkNs$dRncyupZghKn_W?`mHPey#-9l@e&D&cxUV}2|Fm6(0ww3oSN0_152Vqn=5%K7 z&H*R?d9Yv9k{}*;oE}@BaQ!*D{;OrT+Fa-QmPI0$ zVrm3?p}vLuZ%6$G{pD_LIe`DLWou?9Rz=r@2i}9S*BkapSm^E&k zrsji-z)Mr&36ML@_|ZDkGY)p~9_JzUg5OHc2eRpJ9_O@Vr2V?8lQG7j<2<##sqTl~ z;XkdMcbRhS?(9Q~c1C)}hylE2(;j+`$a$C0t$~Yjp6`BvF@QuG8jF2SlB!53J1)*i%P5a1Q_jg2*`?U+vQ!e%ZGRzX;x&haQvT-an63%K_rM&p%k}`n4_?=?A~! zzff*G!cwj#c0Bu*4o~T49gmU_@>Kv9ON_n!K>?+SUdvVN7S`9W#E^QbYk(r zKK#1ui;Mk_UhVodiDz4U;3#^`YoAj4^5okzCarz+gB&Y~BTg<7y4L(!^V!H2d(A=N z2R=OT?=ye+#@5u^4|c;ZG~dzi(ev)`^*Bw(I$v1Z34tDppLdh!N9VbDxq*H(OZJP4 zc12X)M-Gxl;nqvjdU>G71>ZprLw^tK5num^Ea5xkbxhKarua^vcWHmBKmNG*Qwyxq zZ1k>8@K;~2Hv~?|8y$BJzO6inA4JC)a7{`*x+i5lVyuJq|LJ@=_JRHSl^r8}JlFBr zT3(dw`v>yDUmx-u*N%W+2r%2N#~t)%zfkDPJ(|Q_)-1)z>jLZe?;V+&zI2GPkXQ5Bfj#=@q7*c$Ge43UEJxs=inOgFDBQG z9Eb1tcH=CG?{WaDZ?6{ZT=YVd-xo=|I`ml9-)c|vx0(gpJx9u~qoDZ{N!Qt4llZxL z=&3pUh%M-;7W$W8KVh<-dg5x~n|3)pRq#giAkHC`ybm?svk#&R&{N~s0lxnusB1P! zPffgkWc};ZOE|YoU9-~ug#OV7rY#XYb+Vro`uyF4_$5mHty+K3*Q9m+O$YW)j@#XI z1@)#=z`eucch}8L)f|IA26ci~Y$C6N{k|1jsUsqKh4rzKcIe)!`{UH?At>pDFnd*L$ypF91%eh>XC^-?c%@3&+BDD~1_@V`tP{l%T( zKRC#;lg}!p&{c`jl^c!WUKM=hrl8*7d8&5Pr@6qtpSI*zO?Lag8WmD9j1Matu zpeO3_X+3DtI?RUeCdbLE#f5yY#dx&;xilYWTRz)sqld1>j?E)iH5|*R%=8BZ64VueA3}6c4p-l5B8Dgo8sDEg`Ii%7Y;I?cWf}ecO)Jn9;oGehsFUrlM zti15P^Va3X+ONFSUgTqM6#n`>^t=0CLZ3Uea^0UC=t54cO#R8hqtuaGjoxcPS0(mX z^BSw_Iqd(~JG}jaQl`3|rz@Pos4uQYO9cRx-7xP<@vb)pp zvlYN!)~^VE?|B*c7tb|Se<^S+dp>Ft=c8tUM+4`h+FWCtkBZ$GoVOR8kIFcq@8&g6 zWn~>tO}o|Jf?rG3a9*h3LzH+s+E-Ag<`M8GvQo~WV!rp1@5}l|SFf|xQ=HG*@|c~4 z|9n4m59gxty+y>sWmt|$?Y#3;qpNSqsD*r|o^y5b=oOsGG}6oW*sl`%p?TG-$(H_&WP;Z`+1G+}yPGNs$l5^X@!9u-J|#z)!<^tFKy8_Xm1%=V2HQ@AJH#`b$adj753u;R-cf z&dbZP-@$(S9*1+^mZz;pus^F?88>uj7>R!nJ7C%)yeIfad&%z)OTXY5`yGzx`Ru=0 zy$<_p(yk!wyzhaZ-xc{ty*R7a;=9G?kI6d7_axp(_cz_(ecA6U)8F((yq~A;Q9;Jd zwUu$Czz=CxEAZvJ#~IH`vnd0AC_R5NvWj!qOxD+ztJout3;hK3%~!=AmiUyZ+CNcs zl6WoTm44qpzcRXHVb^ahBv0ld=rISL%Xwb5;89}#$Wi8{KIh_lWO2&g&U$f8%DI(A zKU(gIk)uVvr~M$*<*GcHVO#-!j137r)ylqzZK#vzYp-OCpw4a)e+uS(vTl>{XF%{Z zvWoLf!K*ibKfDiI-#q?2H?k4?xjg<1xOh2vz1{Dzp1F~|jDIrz{NqTihp#30lN;&c z{WATX27ht_Prf%WvJpC2X*Fds>{AJz6u9@#yAANix1+(IgMx2H2j}{+K9L0pAAe5Y z{)wuu688)J>h}ZuxpL!gQNQ?g@aK7#pR3P-6N|w*HP#o;7@nBe&mRH%*y7W z2ceIQ)c10DpLJ_UbAGhcrxgCgxxbn}H3@&>yy4~E`ODRd;4?Yu6dH;ANG*Iv<5xr0 z<4@AB@G)Czd!iX zizD5-$lPpF>n7I&d4nI6LhTo z^WfKyM{4nl6IbNs)y#rF!7rz3Z#2sL@D=^Of9_*+uJ9*W@7R9|Uk>=kT>pH|^St<> zBL7I4_pyGj!#_?4+#~17xt(%Ov6er2eNS5Md4;Q)KYSW^^FI9q>zhW7L%*f#+sgWy z5jAdoe*Oqez=~v|Wxdl(YtYm$WxB7d)`<0P)@N7QM!Ix>p=LM0U z(_0JHKImKb$NIkF^64`DO>g3T;2*5-O2$#ZPbKZvyYlD6NIT=HhyIzz8(808jBG@Y z4c1rq2jiWheQ|yL_k;Bn{y`lGpT3`*gFP2>?Yv(J-u)nS=fita(D$4K=iL*p-?a9Z z2W#=i`Emq%(0y-#oI6?o&-`;_b$@Mgt|#+S{<)rZUDnCwT)*Df&~bkEA6NKa8Bp*TJU>{O+Gm zRQlYmOA=qrcy-=G(B2vwqJ5?=MOY`}(|H)db7!mY`{H>i>`P}^zj=%U_^_?ue!qCW zOxkhJY4Y>?@RQDzJj;kW*Oe2ISeEuhIG`sEO8hN&=zd3!yYxEM^Srcx z)8C9e0uQTtbT9s?P_)qNNPFV!)8KEY!Re0CZ%I2t+6_gNRj0i1Rh)i&=*5?BgT7S{ zDNh-t%!-BoV>ziWmRdc`dEkS+vN&bd`DKycc3AzpTnl0KQ@H}y;p#+%JOAp; zw37~YzKlcYrrMk?{T1(%(}hmgy(`3X#u?!Xo&<2ykP*2P={%TvQ8w4W2p1y z5wGJuMSZCmYE_hZnsu?o(Oih-jj239UOml>?_iun{v!98S9q3p&f)ZUrq@PJ*ObQqTs>$91{_^|mlL;<$%tPQ3lYL1SpUu&Jx1AT4;y<5RXL)>}F3*t) z6)IHyazxt82=bQjZ*|@%a#}?#CFem`)LGO&;3?Nf`M$bW<-v<9S;h z@&$NR8>t&d+pE-<ug7)uYG-_}rS&*L(dt zH;k@l9(5*hW&&UE+2&dXpPTR8vEfiZ>lJ4H`G`7DX97QIAG1z?mm06|M5%lpD&JBt zq^@{dxM#^B_}bi_!f`nlE5KhLA0n))_*Fs;^b7n$G4zKt<1p;@(W7A%>QS6eE#vae zs}6-1*$?quDjlaX1^9|##|7S{{jRRHT)fun&igK%RmAR$6u^xNv&K3h^gq+e_a;NC zB4(lQ(oGD!v&q9@T;uUBRHw|^ts$l7r}2<_5gSI20~f#l=2$~bpTHv)t#If^#bON^ z=BM#%@;J@-mel0&2XL8rrM;8h*>B)G(0?q^AoQibXV<}xr1=!p&`wOVC? zr#)BNK_7Ev-C4&n>#j2Yrd&V9?*CHRl+d^LeWfhUS!Z3Arwf#sj}sJo4nZ*2@*|&E%=+D>B z>AxR-S*y^`!E5PQ%2E_~-I@$;%?c@Dn~o;&XyJx)J-Z(pt|=#P5WSB>HyDc4^( zChx<~#r{%^muWp@Vb{s`x*}?;%V+Tmx$|XQ6>9AXOX~hD$1fy!=)ngUtZQ#ogwlJq zr00x> zN%&Wy$Eb#7_y?HRg}G3{G*tVA!XM!$3&k#CU5)I9(KTVUn$q?=vWzZvg(q+8uW;>V z6;j#WgupHPO*g)$F7wkpro9AfzRL05`LbkR5L{1NXWe452=XyV*jV^l{)G@ zH^YB{i~jzXBA0aPgpa_dD`E+2kI8ird?Y9JR6M-ooPtH`|9n2ezlir0KH_~B`)T7S`hmL0<1@wQDmB{A_xavF=t1{~oxgDu zA}`lph42yHuMj>`Kwb+U$&x1`-!np|;Uf!^+ibul4IK1PpO1uukK~Jd#1K9botlpr z=$UJ-9LGm|yuyET-e|AjjXC>d9(q^nW}i&)QC}|?`$5jz#J-q)(&DQ5g#SEJtvFW~ z`YHKc>*KX5f4lHs@Bw{QzQcTSjNiZ3TGnSryA5TF9%ceY+xe`g&!P9jKWZK8j?p!o zPg9qIF7ZdGzB^p~p)TV+x7OS_dX#vqI>*&_Ywf-}1%Fgs!TnCHv*vT7&6Pa&^oy9& zdzU;nVv^U*{(@R(yZUt<_7nXZ>b>Ysz@aXWXCC~ki@Ea15WOlT{hHYGSgX#i1ETMm zLU-5_mlKf6G!gx{tVt1{;2CU`0J_dv{?-e#{L-aFk)Lr4bIJ;lU($0lDtHL zkE}z?a$LP-rl;Sjx@5$yKUGf8Q9s8oT7+*!V*tWbE z!pDu(+&hd@o_C?Iik~FSy6X74vzHT(%JXxQW;!SIlG`%6k#=GSfgj;&+Q$U1^LLMS zG2IH8H+GirTW+K7bM~DK=Wek>Tjlq-p?)dwNb-VGj_76aP%gVODSBrKXg7md zekJx+h5BZ!W^Q{^<tO91Z3YfLyniO}R?wpl?|0tXw@d7|z<)CXx)ymd zL;N(XllI?eKZyT(ssF|h{|)OD@GbxS$c(t<`)|HB+Q<5z9e4dVnx3&!KQ*?I@y&qV z(_FRxMt@J!Wf%H6c&6zC|IMey`mw8LN*wU`_I@3gw?hXGcp&^H16`Hy8~DexlR4(+ z^UYN;wfm4^kbkLEp84x`lkqE+6}$7z)6?b1XY`}kmZuIOFWY1GQ-_WsujKpim&xDB zWThUASqt6n25x@5k9x!S;eD??7JlYXS4ge8-uT|3Zq}#Aa`T~}Tem)f+MG=G^8F>9 zsc#-SNHz9ZwZxccx|cmes(R;2j8pZ{DzeW|w2arv}H z#!548bv|&+<5#$Vy4ujE{!T?J@I-EyeqF|jdBEkK5O7h{UE+Hs>m!tw2inc``@ef= zjhM6L;FZYkgiea^%DH|Kb&0@>zfw5qd-b<2zPB&(@DDnFMD)FD>s5sVoHc!!b$N3h zc(GgPASrZLh|fzE{mN?j0_-5z;>s(O=zYp?p7i)x6n~6T|Di;#AZHYE#e_eI{L6|T z1UV#f&Tzgz%KB)$s@NRQZcovlyqDtl3e`G)q4;m;0el0X>Xp1PzWo!zx3qpA>M^t1 z!8_faYwP^ZaqVMy*RNBYPu^^fq2KEMeSV|rr0yl-qrdI~dXxN~;X8HerTNzu*9|}5 zGou?>rM2^@f?uv*JAnW4D_s9E<4C!Bwt@U*FwgpRV#%=-1FsLrhgT1!kr9EmW16k;M$pGBfZ${21P~W|{YU>5f9L()@TB*C)AR28VVPGR{*C{0Uc6{8X}h#J zX8-dc*2%6UuR9IDbmswo;=g9#q}Stxszu+-ID|{b)X_t|_#dTyZNgA7dB0#xdjDuu z(>F8Be_qUf;ZSpxvd`ju;G_LU<@#@pIX^jsziX@Xn}N?Tj(H~jD-b@PXZnrk3E;C* z@SCL6Z{T@m9Nnth;AgCFlWpxw0WZV)(dZiJ*HjHcFQwmov1%pft%g+E{hl}P@!wn6 z#`gsN`g?7q-}{EUKM`7(C9aCs4Cg1K-Qd9@`5yeZ1djT9^JCU)hmOi~`p>dH!9116 z-~8|Gk>|PBd;UCf4tiW#Q2UrK^ZfTw^alh9{#Wn)@$2VEKS^&q-u(1;KH!bNK8gJx z@Unk4x{L8Q(2oPyO2$9r_T#~EVm|@g)js$Oa5ye@emS@oq#xVse_}shVLb53){H+M zbz<}=^IIt6K^~QiXQA8Acs$Yb^6(=o&vPDrPW;Z_fhX|sMLD%k=+LnKbF`UpFXcNq zK2q|Xr=*{R_x*|e{K)nDg{UjIkN$w)FGr68zcTBPmwvKuWF4IU8a3cQW!B+Ux1Tq% z4(c^`JZ11-)9HekbLvnt{)S&er|^l^3Nu)r@pz=)Z;M&A&*R@J#oygRN9YCo9-rq# zf_-OAKcR4ykFQQ-tPlHK^cnCav=BX@Ab4B|k2P~$PTsnTsQ+k1Z^*DNt$8z(fsOil zXP_S}p?wlPbqVr)`8YmsA^JfQJ`nWlRE?oGYxx%F2cmc8p`%M^|CI01gsO7JEdeTZ?Pg==*P2wtM({C0zT(9Ay z&Km2JeE?7YuGog2lm~vIhZp$0k$LPH$LD7k)GH43RBS(E4CO0;!)ElP49`V>7kJH8 z3+G=s4vt`;@RQ>FCc;6>fv>v!)1&9{_@d>;!k9Dj`F8Nj_(Sk2qSjSd0UeBgU*p&L zF{}RhE`#@LD#SimSI7Gq_!@Go8G6<0r28-WBXVyW+XeAX_P@MM>{so7nJf=I{#8+} z&|jh7kjnMKzxi|hGl5;F{V1~*k_rsmnnXVlzGh&k3xzcHo0onajMyZCSJPxCu^v+zUqWzHLeZ&iz**OGWW?vc-m zb?r_vELj)9A5v@PcNW(bebpR0UMcdMeI|E*#~g!iHOf8nq91c?J#d)Z&w}~)7vZnR zYZ$6=485pmFHAK*b^h2I`k!KchT+T~Yv(;(4`1fvRH}K(YJL8AmHH5nafFU47FC}F z?yoc633@;J+rqJ0v$Z{kJ_|iyb8WgRiRMrs|UkuG$VL!gWIbM1uu{vQ{#oTbj%nBw*U{UcENkr+07rR zHn^{4kbYeP0EQftuq?Is`Myf^p%#g!5x6}({bY)HZ4>(j|4SS1#TjQCd}L@G9}zy8 zfNx1X%mVf1mJQ{1ntX4m>`y?~zCLI!37fUZTcMo1<;Fq0eQYE70Rf&_Zy)OlsXDp0 zaIYKO_k+(G&%_=w;nN-Pq4E5<6#t8HDi;dg;h#qCV4ns4Bm7i$Sk3#)8g-w;V2s4d zu`b{t>jwU)nP$G5`G5G{T;<*QN);D5!o06|*UYE0JP#RHnrVJFRdjP-mi8CYf1Ybk zJlmZ>ZLeehBnKTBsY}Od;kyenj`*Jqxhmj#A>WS?|Lp4DKqH`QXNAZi@sB-fv;M(6 za+eAm3KD0OWsr;5C$ixaGp|hVG$BTbBS^9?Gp|g!en|Mo{Oq`WjUD)J8P{DgQ*BAP z{N3HprVi8EG4#y4YU9Z*HvXep%h|#6`SKm+8IyVE>DM@Brjzh372WE@ndfTen`C_V zTG{SA`@J2^H-Y_3^w(G~c(Jm8y~;e~D)aU8%XJ-cMgFGfzsj&K8(RZhsLSlm*Z-Y; z?=@rC%fgu23}d0CZ-ChQ7Yt#GfSdj4^&6f7svn9nlMj2Y2^> z%Ha=w=hRN0kD(X{6(Q0=`pM%6Iracv@~hTKB7#Z)H88cayx;40_Z|=w0wm`z=Dt zbKN$}9?i2q#`>?w;>X~9vbDs|XkMrmN1-2~oP53*U)Fqf1N$W+x4z=qGpkyi-+Nx< za*gOyV3WV^PTs(c!aqvm*+N#W(dW=yvHpCAj0EaJF#j0vP0-#*eR%9B`xs|0k3EA7xlcSKpL;-ndK8-o4a|f=}^&Qt*3R|85<^3c-Vn;DNe+Y<-2AmpXdG^Ihbx z&Aifm&eeGkJPf;jWA|#E;Qe{$q30$f{*dwZ8|pt|HF%n2{CURj#w}#;k3k2rPI2H8 zmG7`FhJAz3d4<5o6!`e}4KhyY&zqP2e(5@>_(Il&`7K>e+yin@z0-@EoRKFUnEtfC zwB%Zhw~Z5Ti~OD;@wVjqv2uCV`G0=Druk;UgUS0f>QiIu`TfoA*OctjoGpAS58s_( zPA??DABj&j!TTBDeX^(@*zGm(-B1B=|E%b(^VH^B1mErBtJN!6%=a0w>H_OqY#&}6 z1*GD(mX!A#^o%m! z)$^|rI49wMW#D|{3`IZe`=14!ID+Ta7shr~sx}q+{PW%T+1fq19-k4PE_PF~pNVkO z-&>#_zw7ya+SSqyyj>}A$joc1ak9SM&+k=%U+;~!Yf+bd;dujn(f0?w(Qz8GPiuiJ z)7h7Fn6{2H@Z}1Ob;7Unc@^#Q}Mc^|$Y za4HC2v+~`HD;%}hmq&loaZpj>pfc`$9C1)#h53<#)y4ytK!s~0+A{KP?tJ?4EEyZFJejjTs4&n@I+@Z7q8Y_GdtXzhdEb${rW4+{O2 z>#ripzIq1nq2r(+vxl}ciYbAZ5k^b<=IKv0d@PRXY z;0zx)!w1grfiryI3?Ded2hQ+;Gko9-A2`DY&hUXVeBcZpIKv0d@PRXY;0zx)!w1gr zfiryI3?Ded2hQ+;Gko9-A2`DY&hUXVeBcZpIKv0d@PRXY;0zx)!w1grf&Xvwfh(?h z|NEMLBCAAh2T|qk``~*&a@A$;c^@In?w{m$$lv?k{Vw5l6(1|_k@4^5By#CqyK&VbsfWp{`5Z!S5(@$Q$6RV{_W;l zuDR#CUw_Xv|NPBAUu(CfGkgB>?oHE{9jL5$fB%C~ zg!*P-dkxMy<_g@escC7-ud1e7T?wkuaZT?Bel@hO{{s!HBCq`3yI%dp zn)AMJ_rViqeewBc-Z!%5pC7+!`JeCk#i0-E{^EgF^PB(Yfph-)&wlyvp|0$rP3t~5 zPE%`%%xQ(x1FOd)A6W zpILa_mchZd{r!#q^_}Pbdck))4uAN$AAT~v`b*!+)cohNPh9x<*`HCrwd0<3pTBMT ze?DNJyYt2GeCVft{=JRQ6wZ3QecI<{_WdOG@yl<#V?+JC^&i>Q-152iy!7g>5B$yF z%zyVg-m~zJ*FOCHiueA_*Ax5x_RCM+duMp(cmL_LS6}t;fgL9vdwk0kA2|BJr+=Bb zXVWLT2e)tk=U>eE@F%17ul~ihpTG982Z?4^U#WZV`FDLUf9s-$cd7LGDQD*UwuQc~ zo)7)i_jbS3_C)l>pX{rhx$JK{uDbfF%YL-r@n01d|MQ7uvzLA18)M)4R?Ws!i!Z(O z>382WJN}V>c>15GH-G0_*M8(5=Iwg?olpGk_4j`MV=LEXk|&<~@b5j`{LDA*IP2Fp z|Kjn_est`K-gCPaKfWOP^yd!!)5sNn-uu@NU;d+A7gfLO(D(lQkIs)S`N6-;kNnG( zYu@qQ?avwaCZE~z->K+Xv+&%X?*6a+*$=NC z9DZlR+dg~Gd+xkCRsG{{&)j&gQ*m(lHGj3L=bP{`Hls20uM`b^rB=-%Z^d`mI_2*1M+b^5f0dU3ujf zZn@(R|N8ddJMmcE%el(S-u2|g+fv8>*9Dif-ulvANB-jX>ONC9d)4RrzqRnjt7hNw zvs?QU9UuJi7c<}cR`0LF74N+Du6Mon-B6ZGWzkdoOa5n_BAZ73_?dh@_wVD^r9SfM z8?O7zEg!%2_K&1Kal=PGa$Vee^8b(jzwG0m?7HQ4v+L8JzTxJN-gLwD=BGYs{?4av zGtIY~7soHY*zCI1j4xZkfA6@|{LH66{>hJ-w|?qVH+|Z??S@-E{c+xX$Kv?n_De5) zySdoB;^Vj6{0CjP+@PPWFxz<;H!u6l$8Wm+Bfs;Rn?HKPEi24r=5KxK=9@o$tNHG= zE6k;rF1`4p?U#JieBZ4%q*C<0!o2+dJ>2V~-ZvZMYAV&lRrq~!n)kcuKeznrm0SgV z68`mc&qX?%?l~Der+qHtI9+=(Sx)<0WYOu`lhJhA=R6Lc>wf+xuQ8|Zl_Ms3Pkc8| zUQ?~g7(+?9^YQ_89+AG*S}HU|-i}Rvo7DTMwe8T5!|F)>pT)UWj;a_Uugad0A`df6 zf87;B<~_HJ@DTY{_6*0ZOIK@mhlet(%8Y`WpGe+Yymy3}2eRr1DvQL-|YOH&MbM$(ab*s77d+!&8b@^STj zXODfu_BdG3lT^LrYnhET3&PZ65WL9qU3q>t`K5LvbyU;*+L)}8{66vl>6dx>u@)%rZ|ab5T$@4Dz0 z-tYCxS~3pvjbGvR173XQ?gHS_0Rm3qc~UZtLOpI4~^a?M55b8^i@)IZ9VdHhhWG3K>! z7UvysT~xI=1Ds!w-?L%$qFkM@dP%O}>%Yj=s#HIct6iynEZ0J%I_5sFQa^E@SE+w> zpI51WbDvi!&Ns3PDxy||FP@*X+;fg>ssF$^M|>xuekNDP&FiEtM5X$JMq9m-0Zt7` zFJH7V>E(-Zm3+}qW6X@X^|xXwpJ!S8b7z~wkN?Xc{g=`;b%EC%X1yK|x)Hw8P+)yS zvW}c@;pSsrHpKW;>q7UuMrCer>tlr)RNOlcG=14nu9EYX;%YZ=nXcANE7+VHc<|15 zoe(^PoAtuqrY{A*fn%kiE)agkc>tFU**xz_n!V5o=jpllqUl>TJ*A{>{yW}IyO`u@ zIu{NV+;b0Jx1DORsLwziYMtfgT~e2Vyi3M?ubE4y$uF&=u*D;WiZ+sWn*mPhF?&y* z@0Xm9q}x^3EsCoK&SRDO^I3|6$@{`r%f7$PQbU}d_lDoUDWR4_7dZ`z0NzH z8cz3^>VD=e?O30w+J)*Kj;l?~`whQ;z*3LAUjGyExuQYv8N6?NIiaL3Y7hK7L)~=h zEX%olTs=5Z*Vu&<_~_R2FbeN<`Tf0)o0qHZ&AIEZ?nP{+fs2MyBXG(IfAr+vq&lRN zcrRAKTH}>O6i&p9$Wl9!*^Io4NEOdD*%jEk{e_$W(Pz z8*cREPSb5(-PF>$s#D07MFJl?qArqaA)*$`HU3ukr3t^JE|atuxZIl}@1D9YKHp3< za1OHN*7bAcc(svSKV(xRs6I-@EL-PPmVbQzZzg_srX6l=#`95`?yB+jR={L(t`;%g5EV(&xhj-&OWQChz(6 zfaoXRn=aM^qW}5ZQz7-s+4Pr3j?^c-x=n4AUvoILXI7dz{^*0Tu$iDAH7kvN#yHxj zw=DQvMxRvbKI-^Tm!oy<2Xlpxa^|KYg&h26F7;3{v};PJtce_$ov_?G3S31lhG%Cv zQcUWc#C*AExpftuHO{7&fovJ@wmmxe`wInta^;unKNQFP3+;crj z>Kg0w)Rg|+*Eb_`ZsHy z@bu3oEIyB|in1R2l|s+vydm|08*{9yGA^^z3B>Xi>QQIhdOkM_|5{QTH@CoNR@Wtw zTt)uQ4e4)oIY!^8czFOJ*UZ^Wj zcZIqT;ORbx=Uc3sa*V6eyeA^(AXS>2AD3gfLmQml0{6mKQ+&T6g**qJ=BB(l3S6a* zLU^vz9NQ!ItzJiuzaN-S{fInr#jw6VoJ8JlwLhJUgAYD@ZP=A}zO8x75O{v74ZKWp zzT-UfDd<6&IrI(i`I4OXJ+5Be22XB;FUieIyejx3@C>QvYWW`jR)%xSPy)CvkHj;4 z55LUDA&2uP#e`tLu!jqmZDC`&q=L>~CRSE{qZuke0VsMC>i2`cLoF8s{dx*nD{Ui(S)_!i(v z-LTt+lJG~NOXeNg@_*TT_xL!j^8WwqMZ2qurEzRxVJq?IVo6wvwXvMYu^el1#eg9j zE~)tf%_bz^lq4*-U`l|IV>^y*(sgmD8;5oR0Yq@a23+czP{tvoE`+j4Y-;OLHZ)D$ zFDYG81Ev`Kdp~Dpc4f&*$hTj9zdwHC7i6?EXU^rhe4gh#=Q(HId|OfO%kMO!ldBb1 zN3>p{7Ztto`XB4cdwRvyJDG8nujtvFgP*6LEgx=iiqOZr)4O*3S&LKR zclQ^)YsN`7`x5&^r<-f8vNxzl(EW3QGO`=3K*; z9Q^GvU8ls=Jaf5&o9%`zlsBcE$U_I^khiJz{ln&b!H#eo-?{0P) zw$Q(7+`W zn`$^<3ZE788phRPFwfy?nBKMb50*H{gG#^KqsOujd-Pd;r%x|ui9`QXb76^7(strj z`Hp5x-JBIaD4}l8Iu7G#c1>NsSu~}7&I9x2YYkg8j;fTL?@K>6WXt(7_N?+9i*@e3 zdF~G&4;k#rg-49t)W@}auLEvhc%*o5iTW3sd`B=!dtx`^YD3LC;>X?ijNb#bdkn4j zsUGkeIv5Kr*KyJ7WtNh2&9F~v_4g>HUG$jtFF%g_m=~_jZiSwufi(7>bHV)c{o)rI zBin=yzMq%qNtth4j$TT=W$quy7Hp$NrRN{f-_eLKJEG53Neq~2b1mgI6!rP6v*r9_ ze$MJqs~0ubsK-oGmADb<Zl-ps@pK0}u@coLIN-WE&2TgFPdqjVyDA8l2Iwjt= z*)r@j@B8*FeutInq}^cL)&9*?PbSsb4PT^PlXGPiA4*Egxv=Xv@RMU;=JQ^e?-)*N zpObz@;E#+8zEs*-EbUawo6=u!Inh&0b#d5xgVgVGmHLsRhWQ5z_u7-jmp!nq4@2hXA9+=N6vH7zeTU`r)~TjU0&o4f1KY9 z>TdM{-pYJMB+4h{`1`pZ;yyy`i%T8 zCsj$Vc2d1@qgk`n1m^rW##h@-`U(9^;uz3bo^NsB(*xZ~zB|sz&&V5c+km~z!libP z4>dCXv4~H|edI>9d-P5y=0DIeA@_@N-{VtXYT3>w4f)>sLdI94lJeY<=N{iUcn|r2 z&l_xgZtLtWenZDpmz@91{Jw3A{_f*;`3@ue2-;CQEcBZ5_vY_HPr2W*OWm6X7rR{W z3ct6F@7dF?J-kL8r}f~Ruc_x&zPl^iD0Hapa;`j%KC2F^S?uJdu2P+R2k_lph1Y4h z=-@lF=*|1P(oG`2w|2Rgi9e<~viiL76MzZ$!Y zD~?m2>ag|w?Ok?4;$byih1VRnrQN))=gVFBSwg2fy2_UvFQD%oj&AprUBw1DZ@JV} zxZ*hDQXK`|&xBfod`|pYc}vvqj?*=cj2qH%q=dBy!kpL_52|JL$i}`<3yYUcOPvL#KFqHrkmY?kfFH zQ@+xF%7ZUK{Y_H7B;~DWC;oEHcI4Z^zmBXf*bZ_&pYo+@`Rk;75jksD(N63@Fb;Kp z-osbub}!?vP36^t@FDnaQf<2%?IQkDDL;IF!A7sLQog`3fhh;OSW_uqM}N9@B~-!{ z`as7S`k#>gpilIttz-2hT;B0;!7kRQ8>Ih6wIB8E%4|CrQ%R|}G`-$rWqkQ`+d<;S zkgjA=F1o?qVIlV)ctx^q zH~rYQ$XvUJ@u^0shtC~LdE{$*YkKXT5_oVH<)QZ`DUY0fa*;)O>OY(EdA{c=RjF{^WF~OFKXxGO`X;+;;mi0w~qJy_ZpPYRphzg>m&4s ze)XvW-_c5PY%uMa_$MOI$D)(uxrkk_sp#*lRFU$?yR_pZ)T}P#`3QU#d3Hq~e^#(* zCo6u3im0Yk8F|Bg`+i4LKk~d9J0|0%-)hIz1v{5S9*%IHE#F~HWhjq)iJ#{%?;Jp$ z*HiBh`0GmlUHx6P?p4Qh{~J;+@`)UYUsA@N4kFL{C4Uo!lp_JgwGml#nM^gr-R zX2pQ9yE5BjWz-v^#3`oCW5W0FbwgJqs;KLdNQ5P9BGmFM8xHP5UvBD3bFLGVq1wYm!o(}YNpx6BYKcH{6I-W+MukfQEeyoMw)A#{>k+TXv zd_THr0Dchnp3V>Gn}_}ZKZL%*k3r!FQibSw{3v;TU-X2=_r{d(_xay}YaD_fyI_Vt zkFUtbxy^-p)5%JI%y_frUsDo(;9sY*Z_sheDnFp7$vS7i4_`l$f-jT#QQM&TKAj&z zU)2A1){ha&Gan4}L;P&vN00C$aRz?ugWt)Dd^Ps&(|A9t4}J*#P2&gr`@QIgZ*P75 z2<%CqA92x-E%2kFALw5ey!Z7Z;CG-OHKHH%KUIPs8T9Jq>W6D|W zy;1ll`q2YFj$k*Z@dN&u*yRAPgn!_3Qy=<)zc*b!;9nZM9O#GWpXf_J`f&_B9rW=h zn6Iojg8gNlmdfgRrYZUdUL^+L$8zjapicpPMW1Zgr-+Fe&7eCvRCQ&eU%^0 z=r?|BpdX@7*pDQkjE+>*4g}{DPv^&aj~{|_*t^<=FNIgazu%|*fPZ$P$`5~D-qZs> z4q_js*^hnTJAP}x58vjBDI0$-qDjlVHX zKXz$(tmuc}sEK}b2|sYRrtxD0{}Dem;D_)}^rHuU;FqX=u^$Ni{f_oux|hN~mv!gB z-xU6#A9a23V;}qr^lA8E&)*k)Dlk4Dp96iW$DVaje=7TSZ{4NBj}xc!L-a}bF*JoA zuM~c;4nB<^@H3laeH=dK;Rh~K@zmO`ykNjl&aqM)w(s3pDF*pUU&Ve6A z;HU6I_$d6~yCyHrj}+zkzEfb&g^vdOh{2B&$usc7<6lKTgnz=1B>Xr?9AcV&z`x1- z(EcL)SU3|uw&?j!g&)E{2Y$%>?F4!e#QpB~{j0_BuY}wL{*~|#el&K$55cQIp923| z^vS8I$Vc$qVXk*W3w?fcA@AU{3g-8n@DIGL_*cR|@G36z=@i5Y^l8}luSB2FkAUAn+_naL)`dK! zO7NpRnI8wWe^}v%=#$`8zu*<)4fwHWCcJ{5;A@?pN5PK);Ro|H|Gnw+k7K9fmGDvU zYH%jJ^7vQbhwxAMF*FlDQW}3NcqRNZ;RpU@=W5ndrtt&*iMk~;UD}+EQBBXC>Q8cU=Ku}>_o*ro4!6Jv1eWD#eV!D`hlGa%Fo}Y z{lf}Bw0{Uc`ryap`T47aA7`@X@G~3XvkUN1@Ty<%syeTq&W|JTgY|&`uY`}nj{)IF z>nstoU08^Vc6n(6ZN!bLjQW=x8QxC2dWZ}>elgxY^q!1aD6@TsMYYXL|mXGaRJ{C zoKHCiz0&32V?FUG)S8;)bQ8d9tW zJF<@HuPY>YPt~Xm@*d^3))p;^SIw<=bzJ>qz5SZw#Fb{rd&qs|y*ha>rtXmUO2o5j zOV*x3QXQ2Vhvl=t${YI|q%9{9dl-gAhT z)n=?c#TvfSp01Ve;_R-st~}2D8hNiY>Am^#o>F(pdqv`DwOLKi1@-w_`L4x1^%ik$ z;z{xz@>3b#xzbPKZF+pf+iG*xo_tKbrQWVD=VI!O_4cc!pGkQyQ++QXaw52rB3{h> zx=860M~h5l5A{sBVB1NxT=csPeu#b>;Gy8adi=3xh?^px z!Sh4~Cvxew?wR>{r|{FL%9onL&n?2wqRunfc@+6oQ}}t8@Du(8<55%ixmEaCp7eYQ zKeq`#jY;+U{M>#{&UyqqX+1~n{D(Yw#piN0Bdk+%&4JT4j4mR_!{-yv<-1Eh?xi$c z$j{Pr;d3oTaM=d;c2KUYh~s)VS}WzUQqE4dHBZ9Fwl+JjlIkw>@?P}Zt|eX%zNk*_ z=LPq;4;_YUiPsmP$6~o31J}5ZK5nQbUZ101OQfC%xW#=(@Q--C10R=iAHIE3?icwk zOfB(x_^dAEzMTMfxNoqIR!h9zq1_d7Ux6#!&&I$f;`JgAtE3**Z{>bD0{#%MFOYvx z;`{FOFD7|UnMh?FHzw=2F7>sTULMl4@5gv~PG-GYa&@!diJ|k@$g{g(TUzoy@EcT3 zCB8q?y5LOT;k#4T1=5UDH8eZ6TwBN&<)V7IG9I;(`_N6TX?D;LrQY7`(5g~5b7fwp zZs$JsRc-gL_xbnt_~jn)-+RKZXWXl|-q!geRlLl7<|y)})boD(Cv|^=GU*cC7&dyzdoXNAGM&@#>Wmk=}LLVU1nT- zztYJvu9me*mrt)#1>{C8+d>{6d3Qmc<7TrY_A8b$HNT+$E4Cp|zv=?_OXx{Q-qqiy zTu7c{hVPbj<_f#BO!iacIXd*CGgsPyoU{I)JZaP3jCqb9((SjlY;>Mx4K1P>I&`rK?C3$&)zZTG?_1yJ(Jv)%&U|qvpDBpvWewvyuD*kEcsU-jO6?c`R ztRs_8Rv;h0gM6|yeD6@rj!QEsgYW;rr!4xy_-ED^@&s-8GbK+j$OnqE-f}hmxJa(# z?Wqp8P46>F6}^2aVy`oiN}g8R+}u~5M88|bpD*D5OTJbPzg6I)4=Zzmhj+(LcUd3dCk^tX>$)}npQw4_K8*{_S*5>c zw&3*rDV4n01@8H?Pml*u*IktL%(l7iH=Zm+)IZidf1kr|W%9lW|7U!^c-htK^!u)? ztA4NMk^3CvYiyiiuU@C$cXXbB?L_oxd`$D z>d6mq;g>IO%_{%cQCsh`A&x2e;SRsy<7RI^OwEG)^$(WyKAF?*$K-ylW?K1(+;`D) z|NW@^R{Zw&Aq3xxCcm#FdE&wTg^HdoKu;}+D;t4bS#%feOP<4U7JpLeNu#F=$O}Ir zxa#XukQeT^L&t0fd8&NZ)e!m-e4f89ufHcee;s~WRCUf@m)mW?<^_EJ)~wbSbZ^26tG--@f`Jo3Y#znUpOe31Nbs1;jY zawTuv987!mVUYZAOR4lAcB4G`x#j1F3%|h|pMPx)pZ=#Op}U$Dwevaj8@nht*sRPW zM@wm^`VGY_^?-to%Fh$`e-9Z~9(~fTQG;@&o&kN8yynd4QA#sjohQ!zo$~H1 z<1wyRw#E0ShsYDRh~umzPuxhT4-1_g`ZYJ>;mDMH{^-24Y3MkX^q*?)F0XULU zKvuaT_kSW+=v0Zjw2QnVCZeCnLsaaD$PId-zt<>v=r-kN;NPrys?}I8dT)AuL4Y6r zzMDnr+yDE?lD4;6PTC`H%Iu_-$ULF5%vao>9JP_p4$n@vb!5sMsodmV6%ENSoBaG`hK zxYGMQ_Zsha^Siuj%Wd9$bF+8t__TMw`wQN+_v_yM^yA*|Wk2+OFMrOvuKa~}Kc&39 zeKpV9-w-FyAAC@$cYD_c^85u?jXRgtD2` zXzP9X%iy1lKR1Pc>TbuAs{wBw8I}2{@ZFcssLVr=@0aF_z3S={;Als#?x`|*@2^+1 zF4?F3K(1Qe97S?nYIndZ;s^gcUtfqBVn;;Ou-<*Kd2p7oT$W|tcFAwv)H zuIoolu|viBQCr)c7RTb+FrfF5JXW)YcKB`1Lv5y7zegLB+pYJW|sBUAJu) zOwNs{0iS8tY!BvPp1l-*N&GwP*K0>D^xyYC0)LC~dGYmTThI4e+qUTbn{CADvfxSE zmfY^5)QkUC5S;XI{8a5Sdn5QK{+sMCWIv*z_l=yjx~%t&oVB_raT_%U{Df|~dhoy! z|4HV3#Oe97oP|C~+4n)MiG$g#$fMnE5Pyek?KvHfYdDvEA5!jarR{#So_!z9tj|im zb~&l)#s4dimsFnt?^s8W`K(F1hQ+)JdmwbnLEj$Qaq&+VAIWZ25oPz&P8$EQzu@lA z6SZB!cYf2*{qDS;uSM%ur{B1KQoOFCTkng9ZaIk?L?Zdcqb2xac6)IHwcRQRzuU1x zj`$(=;!*h8DE$QA8&i2(_5&p3d&&igL+7j!_>t(at$p-2X<~Pfw@AcTG|Kqn`zQx~ z1@wqZy}Zx&loyY>v|DRif(LPV&ye@L_K=_MQHOpk;(3|4NA-Efv zpQHQ?^(9i2cj>2mPZ@qEO!`s6F3EeY=qc@H=}*AdDfEslJ^^2;zaKuhG0JCq<-LIJ zO-tafgMI|>i#->9&GYybJ6rYxB^3LKXrFj)PoD2G<6n339`ywK<3%nmBOjFr%Jsr` z7J``f6PFkI+pPbmOzb#$zS_@$UZQV~_%i`N(SyhXKFZ)f%c3j>jL0QnfzACcPdlQ5d5+5TUM>jZgt=TwaDPAG=AUJO!rrvJp zeWT)!K{x!vE5zSVYCK^6l)|p%ChM8)=~+w0+Gl_dbD^Ux>jD`|^tYmC^E~-tpKtj= z1-?kVa0S1~5@*>wW z)R*A<>U%{V0zH%YOpf(_dC!%&0qr{UC%`q4U;2;TxcoSLiY-~5)%IyX_#pEY=hdSn z`uALvo#{V9Bmt>XPOarPv1aH@Qa2_M0o$+&pgmMHt?lqw;ws@Y}U4E`@Y znAh|ErL3=Hup3Jc=5{0JChOOW9OO{*U7rJ_=lkma(m4mJi}70Elgv-xQ(cs6k@au6 z!k-4YBA2t|YLdSrSK*74FVm}qdG!K#Xvluug4|zYk7ViW4E}0-p_w+;frDZPGthG` zapMA4Uw&1at#*{q+tzZ?&T(x|s~0kC2lMs5K2m&k%oSX@jrc$7M&@>Ve=aWPo;*8N zlzEcZuUd7VdckJjmWh35e3gEgw&(A)gC}YF&((`F7WMfT@piKwcv^sZZiFVSi*_16eG2=;{-|;u zxx}x~egRkaUgW~;+-2;}BR`$H%98I;nb-OBXFlWkBj>WNNPj!eU7yqE9gufA*&pHk z;=PPB@JE96{MjPsHtYFA#Ix2_<6de*uCRl4Aw=H07(h;t!%qhU2th-nYa(@-|Fr5NP+iO8rpA*B@Is=xK5S_@?hFBDR5k^K_8H_ z3e$B(GHFpa&f;(5(bclwRHz$=DS zDwB^P{X&j{e$^76+Mh#y{r2i3nRki3tW`>vQ!}=^1$z&!4;JzN@ZY4}-1K%^SAXZ8 zr^>Y6jh}5HHxV^F%I{OPZ?-oIKYbjG^pxHG{N6v5iyjkxuPPVi{D}jr%66g1x?S`> z^j+ds>h@9U;rhp;miCv|kJ=&+>NfG4xqfie7XL-PU;F1POCH?^OPcOA%S+aN>5uQH z)($pYqKE^wUU)>$b5cX}13IGD_7_~?+e^Lwe1@^f`+ENJ^51_VtH5dA*YlZ||Nfqw zTFt!g!XsM$UjF;rZPiU&=fWf8jfmgya>svjULDlq*Yl6*@0Y!K=1|$%r;xjqWoy5S zybY~!)Cv7wR&YfvLq0NGBgK!8=8%iKOQ)X?G~mbU19^Fr@uClEfOs**RX6kNJbvLo z(SF2|`Kj}XQN{CA&3Sqszkjuwif+N8`~dq#P2`|5|8wsdMSd%C>EleK&(U^yR!<>% zO!~Lj(Q^6BV#k*ADN4~N?2)v#`+4!_u>W_DS|We$_EB4(N3eC&7CY~58KqsWo5c?b z`WwUpYL*p!x%9@}Fo0Z+K%Z47T*=Ex_0SLGvOzsze}Q&i`hDaQ`^WnQ$rE|`?<1E7 znJ2G0;p|55UjF;YWeT}kbt1nzi+}X8`;T0%L#|hyAkPH3eA(|Kms|AswSP1HeaDl_ z9>yLl1JfT+=-(^JK#5{&`>dPmMzVivJMA(}s{s{JOv{MW)JSFLH_8SLHIh zQ~RT-nm%tG+`HOZE=8VMKNPuLS}gA`-8jy0J$E$)B?SLepU-^Y$!qCG)B*?GNJL7Ksn65W4_=#vAPY8RSO%(k%7+ z`kd@rZNBkD)z6&TpW1HZ+@gKIJ^ZkHM_ufZZ?{y??i9b+T(yh&4SsZw(sra)&ueCG zH>vhyot)@@!IZdTvnm}Hde*D)tdn7XUIM!1*e@1;#D1`V{a*>)9Q1Yxbb>ExlWogB z%GS1hww%jREBRugCz3CQ9ak$0;E^mZ?y(BsL0tSY=!rdlQ08fsdUlYe1YJAY4w9FG z{g=El=vLovl7}e%tI~Em^4VkH9lr(z@3{KwWAQ%v#rtus9~4ggocME8U3xJ%n|}FM z;uBx>?5OpC%(IEd==mFbe`u8cRr_7k@1P$M8COB(;nrX2^SOd?OZ^sfRA1M)&-}_i*TMLj%%_Ma<9`!ZPtt$v0#Nid zu?NY#`|wlv70Hb`wb9_pkw|_%S80s#g3pJ?O6ZM$AAhAZmZ{zk`Y)O%?I-i`($ny{ z@?QSg$DT?H-yg9GCH$e4;0%1Ii8iYpluN|%8%^==i~Dt4AfDBFTk%6b30+O53nCB5 zOO2KX;$Hn(v+YQAzHlm8Mm~|N2Jj%yyh_er$q|mM$u^IewA*r<&R0<8X7AeZX)j-4 z6M1hUx95%&?&VtFQcw?~2SNUNeNT3Yxr=>AJ@zVdy&_KEar9qGaI^atjDu14f-}%B z9^F?Q&hz^&*R~C%+N8fZW<@RR-*q?-MfzO=efm{n=LqA7o=`iWgSt3Zw4qzA_Lq@| ze({%~W7@@E2A}Y|MnvD13SWdy*~doF>v&3V4}bhO-%^{>=v#M+JWq)$SdVEsEw?N-?4sI+egK9zx2z;@kaJ7WXM}5<5$NaYI$it}_@3xHex>gJ%CpC@Ch7%PZMK|vnH}n4Li)IlQ~<~ z|LWrfHGi`MbeCL_@A_WkpvXR}i1F+w`ckiC-G+aQ?c%U4alet`umvu5 z7-~P|R6MKWs41#GJ>Kux^SXG8Epn4et!4U{;%Tv$*lOf z)^YK3WqkW-UoA#1O!zqbtZiFxvF!=ySM5*Ww>9>s8av2)1@C`0nx)@9UGNir`BaX2 zmSAs?hkDTm(M!<>=p5)nQsk=$E+$h;I(C7#{G z8zJu*``D64zp$&akEVp2_Rz1@LI?l78ON zOf`0{72M4HYSgL9Lo)x^Z=cG;pUSvL=qLV>_>C_1qQ${q0B6O2WZZRqsU>p$$nSO8+bM9fXd^NgMtL{R`wdsr?V|5`J{1s^hl$OybL;?|IkBiQfv|WF-!onHYn< zr)oFa8}sz%w;w5Payfsnsla*^aaEZ&6&OdK$+=)~VK5qdL=-Jhf1uX5ac20JKv3;m@V~E zgIe`G>!b+XN>|A`F!Lnxu%(`oyFbHf+;Yb*n9y%|3A;eKZtQ(l=IPjv9OeCW+?1>< zTC_Jv`wnuxIL$hR*j-2GJ=FDgo0sB81b9=aFYq6x)rUQzoyZ{lu!$?IpuQY@@%@Kn zAN9fSpuRv}s(g3gJ9bg{?o8(UPJ3gT{ewQ=ZOwO)%Yg6Bi|6(Fd@l)q%ai!-!gn{W zrr(F}1=hDU--TY(Z_vK1a~8z!Et_K{kpt`=d=BR2DdA5}+a2t^-~;^AD+*4NN4y^al@i5$4tr#SLqpf73q3IADNTQCNHQhFVmesp7iv43%?7r9`4z|?-^ zd5ZNl_$&5ViM)92LEm%76z|pWocbm`Z;}4g#TDyo)%sXpJ9n%|`5EeiFRZW8Z@s<- z9w1KMIkCK#)$=fO%!S{9eDLRFf2B6DOUD5wtz);1*~ni%@&|uwMg9!HPE|;p_@6U9)B`cW**1*iBpO{O#IAyFQ)W7f^`M`UPEj<=hP#2#G6kq zf97@?hW1#B{L;rv6Mu&81n0*G*XCbnxm7LCF<$mD3>nBB`jaTw!+GMSvR?##sSR#P z_LnuJm0d{JRP&T2?*aLWh`%H9%#nVwUn;%SDD2MGC{xAFOd9-LYOr5Ko}V6vnPv~v zEA%|GH6CC3bcxFP$*V!Hz-Pvb^njTT=HjF_lc2b$2xow|J1KAQdH zCuhOoe)5xL-2LQNK#%Hk@{<+ku=@To{nhJmtu%X`X>pEa?T^)UnuX+ z!aw6FWoKzSNq%xsPFptA-f~iXSm;YTD(!mmHf7#g*H3)tyC-&-SPV^NM%Bx7>4P!ahm4_%tzs0ZK zDfj*Lw;*ny?kd@G4rW{1F&*!lC4OH9dJU>{8|zPKgg4I!zK;_>+lxD;I=ySdg*yJ! zsLqS%b3^Vb+cLim>T8txkjEz3H+Dk&0aMSDg1Apy;#>58dA~}Pi`adUPsSbSQ=`Rs zrX|+NJ4(z)=*)6|K07V`u#fmO@-Fvn@k=~EqJBjVzXbX5;~RDT>o~U;8fl zN9+B%9#^AFy9&8HP%7jt_LsG#*l)_bKpjzAa}oCIZ%La)<`HrZGjvvQ*+-SEB z4)kdf`(H-i2Nm-hnO_s%LCzwiE=ych;yqu056HE(&jp{<6PEZZP0Nec2=tkyT+eSj z-}U^){-Wy`a!wub1nMI`ckLMX(PUvK4EzguzrZ+_7x5SICqW5`dp8Nr6r|iqSMtAA z8veN03&F86^42eTO8ApYk7TzRl#f4Q?=J!?OW}*n^B_JHTW4O1-Jk_+e}nq!)Xi%h z7r78z0uO)~o%d85#h#~`pTt>@C_=|5_2rOO-=5bGQXlcjpuRV5Hh;cN-uKTLm@W0u zKJ@Qi#ClW(KQ@#KI~aeR@Qv}S1H@IKXTbmZ<-}DPS8%=%=hCD)PvB38oFWfu_+jtd zBaxRpe3@3BbE((k2Z?;8DgTo7b1wA}=(MtozYcLC_PVh&bnGn?*O2+9S||Qwlfu8W zC6493XUwKtuR}hwTsgNK+)1rmvt%SI*PE9-kmmbIOBj|*`@MO}@!h0V+aGewh&r}v z`;rk;{4wX&G3b75N$w%pxBTpq49}6fUhHd$dKLBS_h+a-O8vV1GuJ=0-W2m1nTNm# zjjQ0-`^NJ0FaHhvX~9q8?gr0Cwkg>s+aP#j3V%w(y%D=MImZH=SWX-v4PMH8n{th~ zB67}ZT}Mgn&!P{$-lV#bTl$|Af5t}M{rOg~4vSoCy=m$$StEJsO?!H!BFfVj^i0pQ z#Gj_qkx2R@V~TmXYOxAg@t3(e^w-xH)m_r@+Mu3U3O^qH;aA%d-&a9ANbG!;art?< zwF&gbhV3!K-ER=*l{lTjbKgH|0M9NxSY3BZQSbgDxE0{!G(9}1XY_@ zb-W(x+gH^amE=2q3RZjih8vYf2%-l$M+yIKxvTWKwleQ`iJJGz`RVBS z$LJ4oyHMJ(z+=BZ=SVxq`K%=UIS3sb{Np_1j2tW2S^UE`>NUVeUrvJc4C3D19mFRB zya@USzdt#aBR(o|bjBa7uclVge~bBhB(q8K$x;$GQ}oMUS58ITO=GsiC-UzTe0$BwjEde7jyfV1CDnTtOwD%x(>&BkajEO{QFIEAN^|T zux@=2ItKj;`Wf`EPV5!*s^S~xZxo?dfQJ$nLLTb-IG_F?<@(@1{8aas;eSMRvu{Ak z2l7)r7mM@hj|kpUKe(WRbI{tPJo4eoOPybTLmzxyU)2vq-q|pE7ZE?f#Gf7n|F=}_ zj`|<&vk&~0%6V&TC$t@Cl(?p&&s|EJ&De42zYU*)@oN79yb^t)ToYR;+lbe01{z8Z z|A__X~N(SfT`@bcg0=-i*AIkBvzds?>4bGS-Sq*kPL%v^<{R!w@VDBdFPx#DO z7Cy{aPu`aNf=WGIXWqx!tcZ)DG_13*?v6j=e?P)XnyiE1hpa5QPKNQv#lM~$r-|D; z(WB&V?>a3$Cwf)LiF^=0bFhQ|*Wz>D{MwoppCcX&y@R++HNUNVcnm$Q6*~fdRBulE zK|wrE4Zpy72?^y0zT|;#KTg@ypEcXGe*qt%hiZP8=V#8ejyuHhc`P<8% z^R6p@;oWD&(J4`He4h8+msFQ`_4n&2OsnlPIzk2yF!T6XRYyQrXb+bH1*kAX>KO&Bc z{0)BwVPotM72jzSYrCMd5b%FW@<$R7RzCX;l zgfEo1&-#&$d-&tEXFZ|z=HM&yXvY=5!*#_jKY~e;^FvyhhaESWof4ms_gqYqd>^zm z>9>=V^EqTaROkl|spph@2UEop#Q&4(WIyGhZmY~+86SD~)Fb+2>iI6YXU#f~OEFI@ z)u@L#ukVE%Pp9v`L;BkE)y{V>Q2l>IwtUhZT3xvIQ9+%J4lljtJ-W52_!J;ZtFYz1fR?@K)R5#h5- zeN*`C3LRv;6*|1scu%Lh+GI0c-fv6k_4mNuGQTvji!tfzWDaB zR`ft2H)3Z^$y>qBW?t0J>T_6HmvtO{8u<*?^#l7@8?j4cY2>()C*ZI?%J}^}v>L&i ztgNHq7woIzbg&*eHD9pr;(~h*bh@dp96JWS_FbH_H{oBMx_GbJ1i8s;w2s^2_t}3o z3Z02>eRfpIywCpHs3lkRGs$Zq4t_$`zp?9v@SEpq%#rfmdL#GS#~2OwKQm^@eS44O z&2#<6m=eDVzoU%b5aEijPK^tgOV@gF{ULC}Wga#}+{Pe}VTHjuG7bBA9Pv9+=B2wP z;p}E*?qGaO5Zh!P%07pGeHwp4ZO9jfE#h`Ek1|=`W<7t=^2z?4zu%yX=g41mzXN%F zHt{3bKWawgYD-+QbICq$zXNn%RFL_B;l&v{lrtH0L|mwDdYsn^|93-LAZAlU!4N$7|iZk8+k zubd~v_iTryJH9b|~;^u%WHSs_w6(|N1-7c8v3G zpudV`6DP&L4!#fW^G6MN_iGdPIgVl;65Z~M6xbIp-v_5(m3Z|6kG{zTSt*m=r{I0Ycg0<0He*DgTbgydM8iW)QRLZ39-&p9UvO7}{t++} zIaN(EPWai*Z*WmHWi-7b1O~jk#o~WSULo+_>cC%DzWWY6RTJYtZ-aKHUZ0otZTZ|_F*7mz6?-qMR=%X6uoy>6Y@v(~M)~PYtt?cvh*Eh~0-dGU+4U}q5mNh{)1y=oA>DazH?!x{Xs*4uYx4l$mnWU{C55c2_o6A4jRzSak^)4T$lANEgx6Hm%$)hY&|1xj=MMab` zj{HYAmy5egaCyivPL`3Ec&{z%DMSYKesJ=Do5m?zska_^&P3Eut(o3w$5n71i$oG7 zjvg~^Ps;b$Vy1UKpsLH2c9b|bp(SJM@0HE9Jh_{4o>+8_ZO$GMd*Y2Rh?~WXhWHB^ z^r4P*M8+G#2ROY2BpXa>O*{dzd2j@zjx&G}+o(#X$?!3NQyNmih z`*(<&(T}|@{-u$9}!htObYCQFVE}#ksIbu9qLDp5S?hVYM78CDfLBoAntb*VU7svoCwC z#EsS5yw-uqu1uhp&|)+y=FeQ#qv z34X|VaJ2f8=fOFzQ<7h#9{9B?jeeDzLLN=(ie?{wnp8Rh+wF(cg$QI4*SF%>ENcsefBl z(($B!OWghtKh?tJs=lswOOJQmKIq-gzs~!;^KIVmksc-ISkgcL zJcsm(aek{^@A01B_ffC>9-$-hRg;)*XWTC$E7m--F8uCij>1gB=3miTkA6nK=C z^^M>i?Bo_4fDA73$otJE> z#=%wdbDV<{QU4)+N*Ouu?OjFA{Po{J&Zg>hU?<#mzH zs~8o({roQM333$cfbPg+O{9<+FYw)LAJ3KX1urf$_eyUbUS||`(9gPf;UT9?{d1h3 zj+l%iVrRtODPP{yp50oHRY%Ul_i5)@`sa@L@L@+K#Ax?52f$ z&V(a=T)XXdb>$(-aXw8+_UWr$gZ$>ijDCI4Uf|yc^bP!$02imu2TqAS%@L;=!0#)H zo%iu?8F)i`K|NFbG5FDN(tu7qyl=thG&q$Nzr}yQ8~lPl!TS|H8^}c#zgzqVoA;*p z57qL1JvKOCq6Ztn0fYYde#HMN-Ua(%SmDVD?nENFw~K$qL0eibqKQ;Wum3lTJx9I* zd8^=vMS13xvL3)V)CzDU&AuDM0q4+by>fX=M{=ysV$zjYkBRj&d^(EsXlwRW6#V%^{Z{jTB&<Wc}E z*yxSm2>b?@z)Lwtg!T%+viQGc)*WR&SB4&yb40rJIwPWu-$edIlJ!m*R}>s6K^Omg zg^1uts;U>$al|_ZAtE?J9=&j)WTB4>sTcc$+?*V9sc)e6OpPx3*Nj53#fdbD1{P5ok<}tHX>AGz(6e@;U}`#19f`wmR_6ar9OVIJ*C{txIkghd%az>-k;w zfg7~E4V1n5-Ais1f7KoU?`*!i?LK+qc$RhfWri*DWmEKo>#v0VS%x9|TXM9&ig;)S zI>o$rXr`q(a-hgM_d@vRu#OT*-z4Lz=$E>Yyg2&h%QN%XKN>e>9!0&-H74(6iNkWv zz_=4vD=v@x(INO9k>@5jw0iiKLuvHcFR!``-ovhT*sM#_k9Ow2(8ucIz6m`!RZaUr zJ*9-?v)Y5)=Xqqg%YC@qZ_bkWvK!-mmiZ#HTs>bL68}o*uw#qNGlF?|yovtV>?e)n z-#cE0F7wh#zAKy3_c1dO`@J&n>5%(QO#NyO{T4Zq^QNo>=Zva?_{Zj4qx8-jpyOHi z-6fvCWd7bF@+F~9Q%>!rJ#}uTxWncBxi0%#pr^D0-N{?Y$vS82S*!K=@=-&jAIMTZ zlK*I+`Vp_$XIKTOE&a+juUbR?f>sWZzG9JnFV_#$m~Q=r?^l)=F_qxPohH~Sg19CZ}QA3|P&@eD8?+7HG9o6-*$ zG9LGjMP6I~LdLrcvOgv$(`swP8fW}!x=r8(4 ze|df8{(AI>zFwS#%r-e^hx2O4BS3zM&p$xBDq?>^{H4~vqhCf+EzU+ZA9D5iB3_)T z3;oI?cU{OS`V1auxva!%eEFRkr;19Ps?4|p{P6pcB(KnmQ$2Hs_*-hJH~y%|J#@m~ zTQOmBKUk*<;#2{B1O8k`e64`KlYdqKN3fqOCT!;KD~MOwH0z%u{i}A7XZmg0TAo!~ zzNF=_3;G$fV%If2W1G`zYYBdm zaI%g1HV$QUJTxtF-z?P+H8iHkkD`3-N%S|c)8M`>=afX_@S#LMBJTF_4D)Z_PY-@e zz`jKLQFSx%8^%FViQgLTCA(x^7qc2-FXW(eBl1|}c}L#ExlWt&SLm;qvkQ#FTqyCV zNp>QBjVd_MZ=T9&oR2_vJywoKLY;u&<~4o^@vkLyhr6VSYyk&SF@pFZkc}g_{RjQ&B-P!-+NchUubirpDkba&PO*NZ0 zhjF$W?q?3= z5gPydp(&xil<$%9;G$nXsGoSkUmdd8zb*b(9`TB}pO<-5UgX;Z&*w1?ry5`B7Q00M zm3kbUQ@S7RPPJa8zJMKMm7yE`rCk+KpB)!@OSA4{F;426+>d`rtb9oF95?FrAi|XP zS_Wv(m0tf)@3$}qgsxRO1?zsZmY4C%w0?YPTm*5YwCB|~xjphAz9jx%&@T4Z&wFye zG;XtcG1Iukj@pbvJ*V^gfgZEnoD@e=XKR19F`l#bmSx=3C;e!l9~qC1llu|0zf;T{uSjXc{@Qq6=zV5B9+G-W$ltX7czlS)@h1ve9{yV9sWb86d8sE2eW%s) zY2=uASO<1oefp?f8_D6dCai7rg*&+1v=rDO)%)6e@^MtBC zJ~5t6s_p^Ci+oMj`)hQ3VrIH}I8@VJPhY&W_`|s1&v(buTxa0V%yjkmgJt*o}qH3R>ikb022>3ZbxAt`()Nj=t69DPQyNO2>#o@VIG^V zQ5~|*Vk5_#6VU2Z^*t!xW!V3Dh;ei$8IOU!@;~th_Q<@O_VBaP?ZxW8cqMkgg1$c9 zM#*E@i~c~blFVO{@?7w7E`Ffcm57=c&sEnOVq4RNZ6*9fDOcwHJk_yiMB@DZxaXDC z3odw8eg8*-iz)F_%j5@dunXH9aNK`CHIF>ZwUcoky|LKs3dyCPCMrL9+<`ybB6bV9 z2KLGk|3mPyhw)mce@{p3N_Cv}zX@KQqMzj~?p~#z*0aLrz@Lt7V4OLgSI615XrxMi zdH;Bpd35#t=f-o@b5Lr@N7$Cb{`v3M&QqO>j#b}>TzZ@iqAKfeCPSI*mj&bO6?;jc zz+M*mDW9W!KjU;czgyECxthjL_j!%W^VnBqLhnjH?VpH#1^WQ&*cY))`0e+zZXS6P z$0qxeUOx+6vp8aX1ouGW+ zmrk0$A0JPPeMwJ=N2ufDrAQ?nF&!^F`yFet&?9O;%hKn#1o4W>ymSupaQZ`DkJ_4L zouPwyPFd`|{mXHK->PA)oKr9LPwuDT4l|ErzE$<3ViTp9YFUU}Sd5GFINRxWFfQf$ z|G>6?4!U(}yofRxuKF8?_9n$QcIElayj$!f=b)Y(&yZBsFMMOZB)@4#ZEy?2=yPPi z<5L6Q8(y@&>W^Yie=BiR_8$`0VLxt!-_*N6e&-}#jNc~hY$z6nvkX__tqA+9LG*z< z{Yt<6{xUE4mH3I>?TimxGTUbwUq=0niFMNF{E~+iv7-}ZK0+eDU0KKBw}oD8DA6wR z==Z;I0K7RkiC)Y%W&f?jM}!{C_mdNfxL3tL;5Ys4mUFP_t;p%=_UlSL-;(b{oZcUm zoJce6uk?rCwxT~}=*hg^rvv-rx)<%6RpSKos`5Sf?e&ZJL0WK? zb){Jo=)*keS6TSXZ|FS@_hfzx&cw(!T+}@YpEqR4vw}W?ThOIO=UGj{iz)pN`osBO zwG$=qv?9mu0^v_@_Q|%x@F%mZdDzVmw;Z5f8RR5lzjDGs{#1khJ{^v=|Gm0iZyn+H z^}W0)-^B~!BWjNAaIcKlu zySdm2`QC9JdZMr9r2THz97}&kFQ5~7)32DYu^Z|<+bJ`C!F!i$x}^PXa|P$dnc(G$ zV_u%xiep}$8h+Oz_!`Iu`6AF4|EKbPS^L|joZkrEaP`iMUUfpBrzrl#BB#n{=C5rN zrjEPl{I=fgfwsdUw-vo`&510n%{2c1A>X31udXC?JRdspeM?P8{Eivus6-y|n^fG^ z`4OCR?9r*s`<_Cld{4nt8c!A#s88v ze*Q8DrAj4P4^Ks0L&tqh`R;3wC%S5-I`1&;w0Abg4_Msy{cP~hD`$A?lfilY!G6}M z>yG3z_6bf-%SQ~(7x(K?Bi-mL{oS*ibw93!)!y&DYrWt5)_K=Q*6aI|@S#s|iF~E> zgd(0Nbau2n`uU6nwZq0<_mR)YcphVfB}?aHzIZ;P*a2{ccy&U~i=7%L<_jdm3F&7L zC+woU6;(;Tqv*wnTQ8W%!=KJvX$Sb3Y%y$E$DyAC6V#`IJc}Ss81S3DO33w=R9e$7!tVlya}2PaWFr?rT+**USZ4|H9Rt7U z)SzL@cWBHNs_g;fuhn1Ij}Mu35^t&IReO2xA~6#0v(IlxKOm^W`TG-PzSE^1z4$PE zuzxnBUMQdjNuyrk)2&J3%LV+GWPC}A@dfV(`Ia5((JKxYm9o!kWPSmvF3iFQg3Svh zPf`3B-b0Rq_w0#JU~gIPX{Q|gNVogt+NYHJ4JRiW#M`?l=TfeVa?s1{qMTjzx9!Li ze{$Sm`qe`@lXiQgocNtmuIkU&kyDhbQLFnXmyTBRDEp+{s-3eBP%Z;54Nwl;Nz1x> z8SNXOT)ygG1mkZII%iows^qVqPUm-tKR`Y9Ma;Y4hl<#55PEm1OJ8$1&AR0If-^NL z9ZT0qytK8lZ-Mqy6}N--I#u*_hn9`-W}pUTL=419vWYYs!lAg;S| z9`Dg!dY(_Od8gAWXs=Cec*9}zDtIrwkbcrwtMHe0r}8&=FD2_v3OR0M-7ClXv}xHA z*HtnvutXaCb)v;1dYy>n2QMx*=X8FUb57rPfc~cRILS*gCkp6WkEujX z0QiWkBL(Na)pqCAejECCIT2a!4Ax_&URm$hIbk!O=wn^41mFAOrhz7P-xC%8DLLs{i^4;pde4O{qWgJJr8PzR$EId&#E`qYrq(_*W6m zryo~)k=qBFz_l{{CGSA*>uC~w64eFi@bW44ib4QV*0qEzTH=9*-qYGUk z)~96M@WW01IVKHh{4ZIrP3v=WI6y(44`VhNrg21Y${TmEzdK?yB*fm~o?FMkQ#BvH z75VMc+pP3QX`JqgEY0sU5I2@hAJpx;eKu=Z7uyAhOiWL;pI&Ma~5X>e<_bd{IvLXINGFZB>uo zd^kgUJIVWT$QSVQGTbEJ*Fzstf?2}V0=|~K^Y0z@Y3C(B!}{<9a;-*W-?pabhbNFfrEXv1 zIM8i``iopKReB#{B=^S?X~uWod%STvexBqm@~IT#ouMDW`H_(Y*|w{@t9D84p?-_z zD)rm9Po!Ur9~pDDj?)F>QJiZ;e{AwJ>1UN6g^$bpMV)2Xa!!*y5BV=3xAGlVC4N%z zDJ51LPD*S=#StNV9{`2B&utBlu8sn6Xwk=6a%B=rS(cxo&G7HEa@L&_@SFbKDdP?D{#46Hb-QXV`T_l>>PLVR?C)4|b*kz& zRrR>ARp>MIyN(h0E@T>gHlcf&X@!pq$m6yN2mW_JXU0{%ueMA72!Kqv?{c5=z0mQr zdhAb5ILL7tx}8>!b@zm&h@(TN)5@zqnXr)m<0pkh;HSFV z_P#p^9$vj=l6?OU?E-$Z-WS9?=SwmVF`MwW(OdTEc>8>UeJ@?&r$C<$iLdZHqP{ed zi>Yb*a^~Pi6(P$UaF6GeeK~W2eL3irq8{H54A#A;@5gb-pOCKrc;~U1J=pP*?6#Br zIMAzmCV9!|^BEf!#5=7mRezEFylNP_+r(cEl!*T=Q(IhSdOk8` z{|o+vwr^^IPY>|}(vt6JQy#k@^Pm!VL4MP)!8$+#c_1eFSV5j;L*5#pe5?gKkRd;; znS4p?awJl`eLp6=O@aPpP{~31LX_!Gsv^_zdua;@rx5h@@)jy=y$4J z#tT2&Ik$-MaM-&(rz-GkBmVdFW_4!R?}#4x=NCnqR-2bLSLY+s<=w;GNK?G?k`n^o zO51-G>z#KpE%HgbjI-i5H!fF_U!|H#wI_2D25vnuX49{x68p;-XOp+D3_m~}ak!5F zmyGZl(jkJ2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09 zfe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3Nj zAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW z2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8 zLL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj z5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB z#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYO zaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m z90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X z2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09 zfe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3Nj zAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW z2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8 zLL3NjAjE+X2SOYOaUjHj5C=jW2yr09fe;5m90+kB#DNe8LL3NjAjE+X2SOYOaUjHj z5C=jW2yr09fe;6(IB?}vZ+X*y&KIpvs#QhRo8J2RYp=TWwQsWcE&t-)MZM`Yuh#c3 z_kUlWlsdeBEB(I{{Cq~=lG>EoY%3L;-Ddt^Sg9<(<^SFj{`fuod*^e1cEdXdj~@Pu zUzV?|v5TiRpOgE^E$@Hx_P_i5Yv25%hdy_?-H~@jzI@mE+6$kkiM{2?JB>Z(to{D+ z^WJBDv+mLlfBwtw|L#rS-ZXT>iPYUS)-xYn__2R){N(Mgzj4V2uY1cCZykK|j%)t= z$lfQfX_8h*h(7S%kTbq9U{Kr--eg3hNk6!hj;+o&S`Xjq%zx&$RSFQi=LG#8tE_%n& zH9!CSyFc=0m!9*NC;#_5F5Ix}zpwwu_nHm$xb`rZu5Kdx86{gwc_xHy5G6+>8D@ux9|G(H^2Xt&c7QReaH8IaBF(#&aXKQzr650 zJs+RQPK{;QwQ?EULcJ%0D?@!fy-Pk;QztM)vx z^Y|l=ZoKlEXa4*HzjC&(f6wsK+qV7a7xUk7YijO^FK+$$Z{K}y1n$PafAV{Mcm3f% zUKV-v!G~`6gCEy@Zy@)pkBZ`?>~nYi^R*xQ+iU*guZCXH`@knxeej=F z-@Eyr-!|*R^Pk@MHw|kqGmPusKK}>f#rp0KzOm`4YtFu)`HIiRf4pqyi?=k7zTvfz zt>1j;l5_9+;s07OXa1ZAKKHgQjU96m&wXX<54%rZctPax3u5C>?EkNyykU9akAJZC zF||fLfAm!+Z~5f?U*GZL;v)lJ{?5kU+OwZI>(-0E^8Ax&$ z-~BrKx}l{vz3rh}zV-C^fAF@)kKBFBso(zRAJ!f^d{J`ao{!8;|NF_m{`J59!~3p$ z{;BtF8~dAczxmv{i=Vz@?a{MOuDkZ%UjL7)|6uV|>*6Q2jkJCx{^nDkzWJ)`{6Bp3 z!@qv`i+$g^@1d{#{riqRx2yB2^7{r4{rTW%-#h+t;jb_L?9uHvd@SAY&nNcY{kDem zJ*#%4UiihETN3-uyYVAW|HWt4o%_|-{L!cW`EAd>?(L6!>yn?}bMPHEr4Ie%waI^f z>KmWE`l;f(&b$5NEf;L-`}^-4N-enl3*X3OZnwIeum17{H}m&PU;X;mpjleWQl~`6 z_|rP246Sbb9np`Jy6K&{Yp=WEmYZ(9_Ldv3z4o1H{rvw&|GD(0Td#ZnN6hO!@PQj| zx&G!GZ!q6`tNDia-e#Kp=F;@irRH@XG}9NZNDr(?Uub^l12^6JZu5iheeca5FmJo@ z{U5lAvKK5#FFF6Bi~7wa=9M?S|CSG5_x>A$XDiI}d6+gY{m@M}-*D|4K6K0VH@<&` zd7*jPdvCerrVpC0xqOAW?4o5$uRnj_dh<;myfK%f_7&z8|L+rjZpv%3S+1s1EnEff zv$fuR(|>OH*VSA_d@}y^#h#0Gc(LcPmv}Dyc(M0uUgEjfq8EEV`4Z20TzNmJ=l6SE z6F*s`S4~Ql-Iz)?rLDa#qqrlg%2rYt?lVI_%0jiv_+5YtF|JNh32RDLM9o&#=a0BT zwMKnkDZjos)K^TX4a)lDsjZ-{UK-8$-Gc! z`>c~`+H05h`59|aWoawXVi)YBYL2Uzn}ZAOl(*sbmW;|%-aLD4VRt63y3bx~@6PhV zT-EF_j_nI_s!TiS1JN-A?fd{v5Z3ZF`4-e2gqvoM=D>F~Ut-v;gXml7xQ5mii@ z(1rdb4eNLzuFRydDg~+~fVNUbsZ$FLRnGiB?7a`19anYdd8+$$zbAD|U9yDKSVAhP z+iqdW>W5oc!rfS9Abd2$laxb-Xa|x!v)-&e1Dk!xW*P6mBqd`MAq(Y+2u4_9Qh-Sm z<76vDNqPjslnoBvu|pojCjGx({(OiNNcP2HW;7p^^?uK-fA5c)wr3dZnNRfL z-Fxmi=bm$Z=iFPbUyi-XmP^oe>{aLFBVx(Iq`t>|CX@Fj)Atla8Je6-pl-YD}C z#w%==(C?F;Vj!35J+iY;%J*}9;t>5XjvS&Nr(c{y^uxUQf2JR~>0;x&Z^>P|z8Er(-wU!w;Kwn{F>v3mJccA?bb2K_`cF z-28q@@6N)oJP6%8`-hw7yⅇ&_0B(1^5p=&H?ycSAL1%_pOKT*isMs<@sWBs99s4 z{iXT`U6H$nhmSoP!Y^Ww%MF+_*<{}EJ?&>w$~&u8*3YJ*KJSsCA^K@p(i~!*73MR@ zxUR)~+uINBfFE`AUXtI$@Mce z_$u`qXBg+Mrk}D!Kan%Te0P?18+qOjKb}QDSf4k{ zIJ!>#8)nc`{=RW0vbaWEnZLMWy8gMpdxlZ?d-sf^@42sz+q*Z-AWxp}nQ55!xSIF& z#@A@?Ow_~k=;pXxv@d?oACLR<$}{Lrs%OJc-u`!mznOoohlWpCyF@RCC25!ZjlPEG zZKcaC&jjdU@_tJ@^PG03ztx^+@8xtvYL8aF-#L%u@07lk=`sCYgC3(-(7V#O>3ocy zr1MYP?<~lHdIfqnsy<_UHmW{jS2wCY=j+ZJRsG}NsFZb`i4{&5yI#1_*qzjmsGdHG z9YRlOS0w(6CT83ovDDA74!AZSM()IgubtufumrDF`H!vL9evDWT(0Mx{BON~9=e7n zp%1>6Csn_(GbOb%w4YQvlk53i_e{CnswU5#N@D^>4J=+4G8t6qz{}%d>a=S+VZ?0pH9OaMFzdC=K=irZ{ z{><;m{8`q2k^WocE`@Cpy7aH>$ieUJ;XU-XyeFp1*`sn-|L++8NxR@YowE+A=j#@J zH?_O&Kbpb6a6jhXxY7&%#_5N3A&A)cxAAuVb;Us%PejC36OoClHbkGeQ|-CjQfr(+ zZ)Mc?!eTEyNMnF)0!c!Ot{adXtcE ztMbkZ509a6gbqsnowl`6h$?rLYQL{Og8; z3hSd7(#U$gt^b924xxcuz>tS8H={i2ZzLn;4O zjt?DUe^UJ5zpCWl?-S=`@uK^`&EW5_KgO>+FF0P5|9z%x?3syMy5mKu&cH{tYZ|Y= z;COK_e!m9a_7X2T1>!~bXJ^XLm&7T(Mf`aKJ+tIr%rxNFDse=LAI6ARjK9~og?_vI zU}|SO{Gg-$2K`C>jk6`S>w;5zpfHzTqAz^moqi=G{Jw8;E-#? zPycczB5u;>4jIn@#zo&5_ZMcASP6b%CMZfLKhwYUs__%^jpL1u_#~U}4z*9Y9bMf| z$GBDX$@uAxxFqX$q`hDK-7hNLCFP%8U>^MK;BQuqJF@ZOxTB+eHr^QH!7sIJst;*A zi2U9>gI($Ep9%t==IhpFon!aMedDKO9Q)*>?|b04%Qf}C$#v(^+Q~ZhYQ(zEK1)BA zwktD_J(a}|FKe75R`bIK zA13;f=_xq!QJ4;nEC-R%$6ucDiBG!p)IFXPnjbbXfUBe<`0BB96BvkgJ== z!`S6d89Ut(e|Fjf?;!TeW*!Hj(}P~C`mve1+KE`7viVHue7HZQ^D%P!nEHk8`9u%S zAV2Wk`I$!Vs_Sl8aXI!j#pT(2I`Si(cF5VPb}f&eQ+jp1nYdo@5d6;YP|BwcJe2aO zNA(eVV_C7juZ~k<{-kzC{f9>#op$XAr>m^zZ)ucL&Pn^zUN-`+~q$J&D(8 zU;jH;ry?6%_g5Fn$R&v@oFeNKxk!<`okIh;v;V5iDoX7ATJAKO#Pf$A{q)nw-QUXk z)Ho@PmpaySy1xHrz2{$M-SlmNPgSl~^y@RB(&vRUKI>|FQo;&)RJ(IX*D-^E=j}(8%NTcs6?X2RridW$`lu9Gw{mwWm$-foH!z9l7#BZoym3=|uOhohD#93t> zUw&<-!g!r=Rx;m|{zm^^@}%HNKu%O;yt^IZKh%A|iQ_+Ue$B);Yc&p3eqra(uP(Z} z$j$aj33;{76n$NB(v|$A!L>{(@ujpD7*&{}V@pzt-+H zxXc+4`v&-u^RXmB#gg4yq}W6*c^n|yON|2U9BEYOys%qB2fIu_-VFO>bgQ;+%?I1t zORi-79vtDhLw{L5&)Ht8Yk#r)_K)#AWW8m$xSWg^KVR6xa}Qjb;Z1jYDJo{~&y4fD z$@mG5bjH2yrDp#8*$JKtaY*JLDsBD$`6SP4*u^CO;Y?PcuP5vO+lP3LzU2PFZ=2s0 zCN6BRQ{l6X&mHi4ceZQcYr^wd53U&$;;p0Yxu+a8GoWS`r!-w zZ{<(DjlOp%e`-9>_|s_Pe^vRzcsYNXZS)b?nfxzyayY1>f2AAa>)@^Nef!Pv-<#eR z|2_Ou@papK;``3=_&WO0`2Ox+jjwxuH@;u~{rKK5F^&jK=#~ur7wO$GO zr^5%aSJvL6a?;0NO_Xe>p6l=ZDo306HIt(xdSIisTG4)$AC&u~VdUB7@7wh0TSlI( zaj?_I&!*9L^d*UF!O3m%xT^A0Ix>BC+vNE|`S}5h^~F5pPa0v(>z9N5GvR{`=CS{T zoD8TJO)~#xuBWoc7R?hmdziObugSbC+kAun_9`r|gUbR!(QK4&JYH!nd_ny)je84kzn``|!ca`8m#b*h6Pr$II{n1l%LGwfbui9ul^+OIm&0(9$}nN`Rmc{kO!T`oX%u@ zen9DTNd8&&H)g8%t7QFh-}v*CFGsd0KUFTu&wQOlTj{@oH`rRWNBmqLPOnqB_se0e z-@!*`M+ZMgcn`igBNclj?2%CUTSs1FRcFL6;}@0An$qcg)5txsE8M3a$@z)i z8#?!N&^b>3P53fiW4zqooZbMPm0mfldJUI5_1agxMlYQMe#}qhZ$!Jvnxp!Y>1Sfk zK4|z^Q2lJq*vLJxcie{$y*ZtS7_Y2yflkTkiB$h9tcNXOJ3n6molQ#)t6uw>C)4@m z`s=G+D?fJxW`3da*VR0B=%CU$or#@&!0=P)tp5ka9g^5#?$b{Tony*Rl}r10k?OzF zxu?#0$o1j$E0xZo99F%y74PvL*HnLf)obMKOhoZ|SN=xybD$2L{W+b9{r`~CY4@pq zerJZOB>sW>$fYOer`nf5<-&Ni+`dGr{|)$XsANWCJ zKI{BK<*!S-9nE&RCw}Rdl+Hlu^kmjU?up;xKI@cGkNrqkvkUmGE8&Bz2W#q_oUBn7 z+=AWan$nf}C134k)GNpI8lq>$j`qs?^(wt`zh0X?lKO??`nTI7zo^%8kDM^iE%~5c zeM?T7=axKRo?CKCuaU@y^cu2$^ja19CB0%*<=BUzmwI#m3H&U6z`1Dr?AkHt^>TT> zeCYjbSPQAW_FoJitPzjze!}o~a2$H!ODBKbkurQCuS8hJk$k`&q zpT%BEQeWWOe3<9yeRd$Yn#W#OB9C9~mBQ8M;T!wpN)`8_hr8O}aHoZO*}*uUN8a+f zS%-Bpt?e{ke#%uGQ_y}{#{)VZ&t*;Io3HCi&9lryUfcD2(`>-F`>B6; zT$BEo&*s@0?N{ipP)gc2m7by$V8_GIQ_}Ywlcyb?^}%&UPkN=K`IjpGVUN7&+Y#&f zrYn_`0rVzy3i4yG)c33dVRV^!5ACxrJ`nWg?-A z?X*YF<#erH^C0C?V?ozUzHhO2K|dw#!2wT3-?cUyPZn(-824)`E>ZF6KX$e ztp5=qPUw^AG5&6_{>N-?wb=cxxWsd2ZKZRCxMWrTR5s(K`kKzqGyWd`)#2|?s6WE4 zIFpWvyQbCNRj`AH%Fc+>V)wh^lG+WkP3-Qa~hX)`_0rJynNyk#sB2@O^p4IL{;t~ty}>o?%~rKH;GJQ$H-f!!-#=UHYq2-3KkM^nQb*z1d^k|M z5#P7uTi_r|)|aSP*x*^UH`N|lQwrRR4t3FT-JNb&vUWfHBVWs^|BKnE)!uyG4J+U$ z?HB$IXurBj)`aev26~_6A>;9=v;F)m>p@<_^D6w$pTBR`)$yqFK!4gF^Z9ubm#xs> zniB1?Cu`K+(BGQ9(BTjk>wA&bg%~&J$?e5RVsAzgd$S81RL1|S9YG(czaEkRzkM2y)w=2J&bvgvI%}ivd3^IJ_3!C=;@W(;tofM8 z+_&W2ZU8^nU1;`C4ywH=_sU?Y?p}1s3ha&CX`8%^=1*MB*LeS0?Ztq`H!k*D9&tnT zx1jw>0eeHgKK5=+;GSXq4p7c~m}@#7@x~`+;k(?-^N2i6{`^;z&cVHbdr9qA(jW2K zCuYkU-(0)DLG0os`WsYx!@LG}Lr0VRioWNY_~ug@-=%uU{;>{waiVe(eU@Z@De=vx zH9oTrHB5X{Qu|SF$-8XieWib++AH!G*Ad?kA6bQW_<9ax?Pl6nVgHNFyWFqwO-1BL zR%dbB>p^NKnxb|Bdz0IfG;dh}v^JTyGs1hd+%KIQ<30G|jK%im3C$0?v^#*kv0L&P z-SN#Mvmx{7us0c%D(2vyq<@w=qTRtOwud-3;gr1E`XC8;0(>y(T=WKCIUg-(7 zlOFTkT5hkibmd*V-tV(Gvc+?FYecRP^DigW4u?v zE|2jZvrFf9PWk6(-pDm_ctYb&m;NU`^4Kl)wXVEN?RRI>6`!p)_!B!O7qHLdqqSeO82en;yt1sRVV`?s{U~%vJ|6b@E3-m8 z_-dYe^pih-YPP}nyKC+x^egR;`Fv%zPWu)58!FMB@rKksH+g?l=hvt9`75)mGsdgW z`YrL`7K!uXTd=Rt?QABm9k6ba{Yu#9ugymAb4Mk*?DR_S^%@Vh;Lomj@-Xor?F$RU zgZQai)t*D|X_t5qJN{zx;+frRwAruR?qQ#qPiimHI?z38_aboXM&4_&mlxFT(OTm~ zt$RPEcCUin*@}HeF884t*NzA6*;%E(!#=0^?cDC^czJx9+db?vdX~|d>_@mq@wKb| zLFbpRzYA*T=qJsKKc#uH26k}Oytuby;?@rPoYAR#L5`h#$#FRL*~#fh^WyiYpEUjf zI{kbe7t~MUSJd8X{qref=jne0`;1-5_}Oir|7^AlH&)7vuez_G+MXXx^5U!Wvw|G@ zB?`QXi`#!Qi=Fsh=f&@t zK<;()uw%c>1@+6UE9XEE^HcS;s&$+YT-CBqqbo1&{oQO8I#<+9@|4+cL;riKj{58SbX}m2(q(VG zO>=ebuVimm^$WH3b@v+RtoF<5e!-9Bp845Y`@Xm?ds}}{8QitTD{mzwQy%j;F{96nSzMCjp)abUy55qTWgY?phNz4cyg z2JtK7cDujp&Oemrs*00V*!Q7vKpDH)5eMY^X3OYhOC0_K(1|?n zQMtK_*O_N6&p%wybw(WN9CBiRr*;jz;%qFp?@#Npx87^#7^lPDycdcO6s?&w1dgEKevuFz2%_6+hLvJpxV(%zi0zKd*zUFg0s*0H7c zp@aQdZ=4ID;|89CcUzv{q5T#{ee#6hxuie#XT5Q*O8XW1D^Rb!MEp`v94g>f2Rb~( zp}ZGi-i2|4mOS4|bxgTep+Bt4Y^GrvLAIJV2;yowFg@NjO6}y|%LF?+r zJ;9H&PNgNU)wNE&e>{iWJJ)GOXXeKlFUZ$h?#EfDEjToN@2IgStVjHH>wAIPlLqT@ z59<`V>S|9S?92FS>$LVGCf?|Xi_-D(efY|kJRZ&MiDz&HE8WD;$2Ct_@0X*-o?wS2 zc`wZEYM}O{qI`CZJ+CW&mClV`J8tTVi|Ri%hhBE#(2P#S6(RU%i}EFx?@If|k1Gy! z3vyKLd&GM?c(0t>hhR+k8S{C>5vePGRqm_CMfQ)+dC0vZF8Z(EC;M2(c@*-O4)q2- zkJ6$)sa;F+`Y&C5=}$ZIt<wbM!7z`eCT~~`Brzlj9vihl*4)TdVH|Mi{cJdtlw>poZFIr?5ZzO&kivfd41{LIp$#Qn@}FJxrQOu0;Ac`_>e2d=3!hI%b;Z*!G=KThuanPq>?c2H@PDuF+d{umyZZ83 zCoe2t@$#Yfg~j_Rz3vWO*TiXjekk6r@1L7%qNiCr=znAmy%f#Iqu*Rj-rlm}`*}QA zG4bG9txFIGuPuetjj3}C`|h+u-AUuRI{VTsyEf;tpGDVoQ|rd>i0j7MuV~$veglSI zom!(4@b$7VYaAUtYVvwI$kD zIA5aiV2EAY1RcbOC4GE5r;zBuJivTtx$oPRavGS3P2>(|)#qWj!< zp1t2?7p9YS9q&^aP&;5l?~AM(r~SWl{_m6E5Iu*J?vKL%)u`j^{wU=B@>|!P@x_d( zkFL}B!pY;lbe+8P{Oyb{s5d&Ce%JV-+8tlGUG+vOFKC>Wt!wwv9P2anM_f6Fqw6fb zZ^>0RC~JMu^TS5JtkZSv_Hr(#!G2fvfo!U{)765kSHCOj=j*>Y=l9E?uIo_ijaNB` z!+9O;7eCDYvrCGT*bs(4W`Bm(LCg7g><9a-i636UbM&Ri%^VHu$Z-(<4zcUx%uao^}J&N;M z{e2hCqh6`;NFl~~y^8Zf`dg*mi2W!pU@zc6+j*QU?$7eJiZfm~-aD-MCR=fx#{GFd z#NR=jA1EFmFH`0`-WY$^HSeZ)@4M9<>z|*)ey@o4viEGUX=G* z)}gQYI!DM?gZFHaQ0qF#!%FEb~%~iYe;OTfrlrIjrr-Ls!E(Pzg zE>ioI;=TMlv+^^4?koRTHxBznv|f^*X9n*%EqTT+yyty+E@ToNb;pd(EWg|?zx+IN z)!@sF&IIq}=b3r0MPGd7AM_|*HF8&6%KCOTR@?KoU3u`@Q*$-+_Nt~$oc#7vB#Zp>=8L>llRg%7BQxBc=^!V8OI(q{<&Z4Zuo&tdb{I7_b=yaOf2^w z-QU;IZ zoUpIg=ypx8*|*1H4XPng?4(*qLPn?|M>H{3go?>=S}=p{`^lh z-t3ubus;z0k@ScE_;a(~R_Lz>KX(cL)}#KzCtr0CIy~J^$a@UKc;A={tURt86O-3x z-wyj#u%|bksGQ`zbbmMdm;XZjjx`>b{VJuK@CQ*#T&eM6lle{XzANI~LR=Yudz1Yt zYjnQ~f6Hl~{VGW71@2eL?8mBh&Th-+r+Gq8>uBmXM%wM%3AJ;=Jd^!|({n(>^ zpqD*Vuy}{qB?kBAO z-JA!VE9D6_kLJ)%M;)212`VCL>nLh7zy$_znDBMrW7b@czF!5B&H&_Z#&LKD;Q8FJ3-&_>&ztiM%H9%Yfn} zCzr#^hu+6J@~}^s`r)ABBrn&el-_Q?AzSB(k8-~u!}H8DJ>SmNoC~oi_2~UPPuTE7 z;*~!28?1vqJ^#pFZr|Uh^ulRxZU(f%~2h@SoZ?LZP`vD>GLP7hr$vbGjJeM_f z{04QR{k&H*^(8Z&p8ubZpzn41oO)3HeA7JZrSD+fy+prhe|r9Z-lqKu{q?b5{t|w% zPyI%f_w_r50pE+%_qa0N@VvCtm%9AMm3W_HaxO^E|Lc3H-xyNAQEu@&U48@H;n2S8 z{Qryd8`-&+%x_3r{Fv{{!*4JjERd;(q&PX>zs`HeHGN)FyrK3u!5g`~#BX4SGW*e8 zUvlP6{Lx+q{{EkPd1n%*{`Y#0gt*Tqul_&Ex!$od@d@i*>jsgYv-5AC4^hS}PW30} z1LCkGKgTus-bj33_eJZumo<8>*Cjuo_6BV&v0wS3#(U;_Bbxs!lLt=rWlx(p*DAc5b1(QY-G|NlaysB#FV`0sr)Kshi>uV$Ac=y;wtbg@o1~PIjeD1S%|B|>^u3AsS^{wth(={{>FK)J5K%n+nXOr;)N^8 z_l@ws#5aR_E~1{t!KwdE`SH@_f$!^x$H=Re$-iklMjVj(ftL@xpX`Xoo{i6OXgpTw z=I=G~s_&eyq34;sk)ND*nPuX?^K-rN{XDN4`C*0lMD0z)x?vq(w7KfJUi5BI;}Glt z@y-7$BK9;weK1@y3tO2k3EfuD9j++vk0j zr=IJ@PUPo$XQ=YIZIx%Tv1T z`?@thM*PJEHqw z0?o@pr=8=2d_SaT>~LxavU7GCXEuoo*+1O6A1^--7~|snJRrE(X*svpg^TMyW7cIS zF4pf;x!=q0xjol$4*D>4Byfc8uVkI3^L+W(yUzGE$y@4v!P?8wK7Tlgw@)PB%lq!} z>oxG_FU?~&TkP|xd53kD)OENf^=vWs^Souq_viX$y~$g0u8(@sMXyiR>p5uTJEU=4 zQ*mVNxZ2lsYM&#`TRzfp4w`;>o}OD?r~4u!;zkPRN(c7&;HPb2|qjRb4KT?{H&VsQu}(+)VnxG zqke?A#M!tKKPyHKIX|nbRdCQE>46m%j&sy95^OO4hIgfJ+tMb!le!VUH zthJvj?>hJ4{koBR$9HivI#=aqXvWL<`3mTaTF)u0#LvLw<8yw7@Ux{J+jZ`v_M66j zb)2(X{ky4u+Ofa#qRB(9)%_1{zP?{Re)fNC+}7=9-G4tHs(noNgSuayuaLJ;|AKtD z;%9ZANPIuXL4mLPOM2DMV$XX^u6q$X(z`okKPY}r@r^7%#j+56Pb!q2E`=Y78Rg3iyreC%K+ zJ|cd{uXUb3d->4YiI496?&qmCmZ>{Yx3aM}#NFhdpPR2Lj!*5N|NqW6(8du#uehC^ zr;6|A`)6xr|7^e7!I0lED^=M)OPy;paHp|bMe}_mlYjoJdB*Egya8VBK2Jrz6~;-< zKlSOpTkI(LXF~`1;J?=RUvBWtK8cp+-<)@dZbyCaM(cSh+Vl5H{q^g4st7vO4r2fN z9qfF=ff z@Jg2*y$Id_pCtS2^7B;Qi_1Ujx{Y4S@4`RUQD>zt?EJ%dm+#$o_dDY;^3vhT-xqQZ zu&xEY(mZUPb9$D$VZmkDl?-Jk7>)5_I@6z@CMm_Jsx>~DoN?G#=_0j_C zsHE$f_1=Bng?{m$>3Nru?z{E%y?8w0jlqS0cq=*Y((-)mf=_)*&$~F(m9q0Lw6AJi ziSsco<9777M$fwheAiXib(Qz^ybJ!0_-1f{^~rea7b-35Qu76h=ML%qCH(2_SK@W8 z`Cr$3zj0`R^<`~w&G*!Xu47%(f3~hy*nevk9x>lj>(RP5bjs;Qy;g(n7g|?($=CH~ zsfV=we!@Za-4QqbcXQr_d}f(AT=xkg-!!gz`S|P3_~Hoh1^Y%c|7v&VU%St-b?%Fo zn-?PVIMLhuyi0sP&kt7ou-+qG-$~T-E`i88^}8N%fmIt@aM}N^>zZ}ZecpwBd7hq6 z8Q8>otQX6Q$0I+uXQ6?eNzS{pJb(29^BmOkF32T6??U^s;ug=GpJ{1t<~6A2T^cRl zN!0T$O_Lwovw+=UyqgyqEx1GRNyXrl7e4X0K6nC~fKd|8IciR+qBEJlG(q3^zf@53Sn|ZDL z9W~w)v%hFk{Xk|{yW*vp4%`_`EufVt?p&w1Q(Dficj3-wEAe-AuX28!{IcTC2=>3g zdcV6-idrAae^;E}QLAcR)HdIl;ESX9-9DMxK|7mvJM?$!b-$3^_IrU>yMB+a{rBU3 zW}M$;uabux=Xc}KkJN72`a68A+x&O<_}w@+e_lDj^CrLJcOd!w_5)tq`#-0@f7j%9 z3bWt8tF%2op}%|Q^SO}hcW$d~{hu@Q=XVk}>b$_K8~wK5t-Gl6b{RjLcdhMr7N>RI z4W94e{RTL7Pr&a8=D7O_{T;%H`Hd(2kB9W1KQBz!#U}jB=O4AbpXRsI--S!@tt96v z(tNEX^|mxGntm@Q{qBq;`N{PAE|TN{v+t;=Uk=fmKl!S`v-kGv_n>;Dx4awsO&(nR zjnNA+`^pZhTx|H3$wlCc?kh|5rZMFQ#1n5)`KaDx^0C|IUpl1oZ`tliyv;=3i^0|MRM!=tHibLEG;tKBfAJKgj1Fw(-y20e|oZ z`TQgL&*}UA>(AeUoBHJ5e*F$LdANhbE9gmQ{ngXoi>x8H?Drxi;Y+*z+^O;_Grw$o z*=_vaG_Ly5DnF;qE)DDYa$D_7xlR83o-*?1c_u%%?fs)FKlCw|pV!9U_Zj)M{!VHI z`Q`cVn#vD;1VK&=W0K;s{LSp!&blTcR532e_G{RYqLL*@=@(y#>b#df9uLW zkLMZx!nXdmDgP=w&+Jpwrk~$`{khxJm+$S@{hZ{THxWMuoa5=JhaWNP61hykH|TdJ z1i5N_*{(l7s`7QvvrN8r8~@o)Qp9|;$k%CmKC1Q!JC?7{axzYAAGWD{BlIt`Pj1`# zeD@ZfH+X+b;vcqnZTjb{{L8KKue3cstMU)f&&=KDZ1bPH55S)?^UwGjwY~rS*PkLJy1jJ$CGC*FJdNhv4-|zwSc_wa!sLx!`lY z$>a~vW8H^PC6AYVkDY!a;`rqI>H7215Br)=jK>4dpI*Qor27zBo}bdVvQPIRIC+2M zLryQ0X}>~$eYy`p*OAE|un%7MA%vFZ4^C@*&Um$jN(=t)${&C;guG*(KRBcLk2HT! z()^Da{-9TX2hTV89kWkN3M6!W;weh~ed1FKP55-Z*EoZJmrXYL zc(Xqz{{8yw_vTNnyNcWe)o*(Ir&sX`|L%TazgIsj^-s@;CFB3@5&LS?BO84Ak;RsE z@crJm>yI%H8~VqBbyvBjzf z8=||z1?BU;pm8Rm`R*F|RFkLptjUKabc-Z%N#0M#Y4lI|p@SR;r8jD@zQ&oSUyzM& zh+esfUhZ^{z4Dm=yP~)=ME-hZy}Ft^~>{v56M{_b+?oQ#u3ZDya2zRUw76&tL=6-<@ewNd6??Myub6Ni*hbN|8H-i zzg(01))#C!S%IFUAM#4m>VITZ`-LBt|M>-#IQR?QbO3Q!-)Xtqa00Gs|-k&#w!7{a#e>?keY&`FmRgJd5wLA-=P%;x$?WL+K*Co+N=BM&kgea$DeK#Wao|Nn&&G$GGhIC3?~+$sf8+anudxpj4;`=6k9oZk?O?ss z$v0|z8K4i_>Jkd?AE?K6oU7V#ewXBKdpGOuco>|Aj`fva05@?}IvnWOQ1NO=O|GvP zeP2J^@Pj(}C_VqjJiGgopRu3lm7V*hYv11B*H~`@qYutjx&5xgeD|m^SZbmN zYjhs)Va-(251Fq_q4(u$@iDCHcXgaS{!ia^ROJ%RJLu` z+k4^R3g$=8CDE=^Z{>8|_)%xV`I!y(;`TQmshvk(c8x}hPnY}TOMQdvn;}naMe~{$ z92R?FA%H%a+7>7u1`lit8a=eTlX3XlsYvJ$uXfElK5xi=#rNO!EOvAA5&t}Ld$e!c z1wEheUF3dd!%^1F<|Ebf4NG=xB#u48`z1OqgC`kpy_|B5ofz?Bz2R!~W<)-* z0Y3A7<-F3pN$s5Kz4K?WoGa9YnNIqBrBu#akaL}NmBgFYMmGwZtdGQgt+C__3w6cW zgXM=K`d@!AwvUP3!`}Q`wS)g$?HP2+egbplONT#j)E^wwb#5pAAeKkL=o51F-yoq) zd}h1uw2NL2p?9oPnkV||ZeC>lifzlURjHE?+P?l?c8@*SuirbTqRBete$UbR8UE^F zU4+<0aOdrWpo}ZVZjSZcQoFwieFfwlz#hx{rs4GRaz`#aRmn>xZP79;q+UF|66O4MJ&7a2vrL;CBo6RCc8+VwYVdPZoU z(N}$L!S9pj*St+W;uNv>_u(Jm&)`RmT_5TDq<5D2^r*e0oml?gF2KiKLO&k%R{a9> z6ZdXfR z(}K(TnYt4Di7D^u(;oW*3|)+y#IfLd{Dk5w@2bwL*!Ykf(D;_vx4O z9uwZP*k=)=7hk*~^%L$@<0rOOVn31ezsBM*L|O_xMt^+JpXhPy;47cUF}!9 zX4BcOC^{qbTdvSf;~)0xVJ~~fsC#o?xTf||>!Z$?`U%cI9S`fr{DMT2_z4fYs(DO@ z_6L3Vi=XaV@A!#}{C?PRKRC}i=~F+!`+df~LdUCW_zCpk#+vaHeWRWq&`+Pz)9j|l z8N*M^i0s^TBK8xGt~=JB<@7D$_td|ebB(8Cz16_V_=yqq6O1>aeu8VFAL~?Ks`|TW zb=E1X(|pI0d34#!Ci1|}Cw^kR`U&`%`n&aI@L)i{4t(&5PdfOJ+B0VaKXC;=H`N?? zrV8HbJ~Am$FZl+ zn!0Z0KfnPlIce};5Wc&=wVWKmHN+vPq=^km4=CDrm*w$AL)Kh zRg=lWYc`bgIC=Pg25(}=Hr^Ng--ac=5NGLBG*e} z;H*g3f5EXM@>kH_KrTDw`1WUXB5LO%S7V z@LD&{dds3szDVP08*8^m@h;b7p74QvYv4Gm@s>p&DlRF`rJw0d?y{2={HE?VE{}K@l|C{2mBAwUF8;9Omc<&EEne)mlK#Or{J}scid|ntFb->|D${| z{h=pn??dcY*F4hr*f#j6?|2jY%svtIUuDkM>iZS?TUYmLW&GEz^~M?aIIus&f2sc` z{)JDSe!v-#yX_M8{`M;d56BeuOzE-ew=L>=>Wq62Vb47JPwmQj;PL{^;?5Ff9UF*e>y*B#JPLJZv6PZY2)wKorvD_OjU79hHu`W z>k54H#zpwKlHZat{s(>N_CJA%1Hjq;$6XiAIL1F_eoOIJ8TvZ>7XIqkhH3oGzG>s< zb__-29~pi^^Eg)KuLjg#xs2PvFZ|V+4KoJE8h@ALSm!N^p~|;euc1FFU$gk61IJoJ z<#>H0c5jX4k1VSGC-G$FU#>qI`?1&GLQGZ!A=JZ7j}HX8FFvpQLtN?rzALkT`y_Vf-@Z zDt<%rgu}PV;G;h8Cw^{2C>%{$P^bG!m`=aZ9Lsb8# zi#B#eei*+;JN19A#uv^{Xx`bHQh%ZI=e}pj2C?3ublcjlsyHxd_deROo_%d6y|Tf* zvQlAHqca{6-v#<99=-X{eSmS;H`mvh>R^!Wy4eTCv zH}qi0L>!O!kbDJsaP_aSe%HyJ{*^|Lc!%&4(6MgHi9-6@edir_KU2Z~@M?qm#2=MK zN9}Qv7oLO;kIqWXIG$Y5o)2#K$P*D6vLC4^Z#`aq=79>`@L?6j^B|q^fN`Y2wW;^@ ziu?!lK6cLLTBY44t_{W?)+_wjruP}&Nv@;G{oQ({-Cn)Vdu6WL&wj4j??JsX&XnHo z5tq71SZDl_6XO8%(8NJpO&pZq5@&#U9Leh>FE!oSIw1e6zi00h<+`DZJsF>er-qM3 z0d8!iz&}~OtoBnO@XM7&J==YT-{T@(#BhRV*dGxoVg8md_Xd8AA zy^t-9eiJ{QqAr4dO6^_LN09T`2SmSjDd*0|_aE6Kx=&HU&q)yQ-lZocuo-B2Ui7;| ze*v&lJl8W==6CIpJN!{O_gDv;A1@Q94VGoP$$NvU!GV=v^S1-&(8f zg?fXwj~H*zH|_s&7wsM58Er4x&r1E*48Q;CMxXV^I&=K!GX07}d-Sr$I@kPo$aT~g ztvibo>ng0rakf~Skujq*Ek=zH<0Ou-uF!Z`TV~b={g(dNfp^25W> z=NrA)6*T%~ zr5~9VGyh$xUpH*7BM;&}>M@ri+V8I*H}W^uv(1kO;K_q^^bxu4_jUyya%IDr;#Ilu zUi6uDJNz!QzKet4LFg)2)sGmQQ~E*h662h^7yaP<6GwPY^9KIu#R~6-^wXrj;DNZG zZJ)#5f(x`CVgGsB&)@?F7s!Ptc%SyC&cScB_fc&zg0It`;XZU+JTHMyzv!9znKirV zr-}U1em3cS=y~X%d0)=z)#L?#dC}qhOW%pt^FyDB`*CQm>&tw1khmx9$JYCd6Y##$ zQ`4*2-|&#>=Y-NzkZnU%;#uMitrNmeXCr?*z2fRVo=E+4^sg3KcitD|YadX4Snm17 z8hJoimH4xdE;jJHVts6p@m6i;-tpif>vOest37vy#O&X3Hro42bv@T@OQYN?-f?;6 z0hQ+nezlIAli%MeS-~fjPLcN*zDwa7_$l(XdunpYW8P!+`H{+T&A-}l&UU|^g_(DFp2>%c1`~1zmhEFV3G;a__mqYY4ZGYRdZS9{o z?X}(}zrEmL%&C0KTkZL(@=fBV zLh{{YY2HGe{0dlUL^D)-$xAY zP32ogzUbdd@>RVya!upLG4$KT9z+Af#%@Pjhl7w+8SUYUo<@`0NB^Q991hCpL-h7x z&GSn1F0RlWy_fs&A$lOWo=@(7CTaJ%`8e&&;YznP5lop{_*VEDFjo_28iT~BDd zXPbTWQu2+xda3DNJzgz?SJ4~d)w^ai9u%|RJHtVaW9DHMZDSu&ys*~tep2zS#^)bw zD2@mJAlD6%_n6kl1{Lpcp8(F}!Gx07~? zcew9sI~)7#KKi9arBmD4>Ti;Eig)VTAGn6`%K9ApGN5=T()YO^q6hd{%_9tOJN`Y4 zI`p#Z#OyN??f1H&gZVz{6rZQnf z?}v&x^yqsD-cIawam06ZpBDRTo>bf@yQ(I?)W5A_;>BTc-?|t<_f~M2&FkrUV5;r= zo~m*NzNC5ZVX3@T^S;B|!P~53De}Jbsa^X=OFsT@!iOK+p8#*emrlIRNgMiap*P7( z6~SvsexSbCgs$5gjWgJp;z8mSBRBtRCeApZiT`5cspr7KKiCNe%(i& zdac5r8$$95^t%`NRp`%h-?#+-ce-~?eLdjsy<*}d^zZA7_@i~If5<^HydBIgHh4|- zPyJPr_dx&tcrgGrW_UYkr}`Iwm$h9DT<$(Pvsl%A0Q$H3!=#<+pQrtyXBDd(Z|nUs zba&zH${Uw_(u-QZtUzb7FUm(Wt#~`dQyzJe=W0l2KlroR2i|^k;h9D&-j)Tet7d%v zFUQ-DKj1)bDKdWLrN*r<#(upl@pc*eX=ArK@wVdKa?dsJc6rk1`$~BGzzX=-*rPtS z1A;fOgEHi-)AgOMyUwevcpp9P#P2-@zmo^uTETu|$29NAI>f#uc>7(%>8M!^91x+b zXU|KFvxu{?1M&#*y4tIi@b>lSJIq+SSK~(d-P7&-H72p+2tdnHM-O#@jbl z>AxWBwpC-iz0DQP`xLBm2LBgEtI$LL+kymd&&T+>_RS8w2@X-d41=#5tix@4j6I?d zXyzxQKWOrSkKO9|x*ua`lLx+ozD=INk9`Jjvi)P&0}t|V6L>r5moMHHVPBC$If7r% zQLY<2yiW0;M;(Rwte$t3a)P&SZ~9@4y!BL*{%Jo|HS`p%!?b5#g59J&^i3(=7I}Pq z2>xZhDi7iQUg+_9sBN@*h!omhm%2{pZP1^LH;58xK;jDZSTf-`<4d$Vt>iOMs&GK z`-2rjPrr2qec^i=Qw^C$-{hd;ZPjn&+~DsFZ@<^X6*=B++Di`ol-?fW(sL&gynU4S z%=cN>E(!Asd9LGSc$1B;N-wSIQcAF0SC-e*5ag10MMbsng&1%9uUc{`3+nr};_YbL65~5tYmwR4khah8dEEYa z(_Znmzi-Jmb&%~#K6;SQ%l?R0E`{i^t#})`Zv&4tMfWRgFJU*FA@FvPe>ZSgYWq}Q z!DWwsumZg&-fQ$^+aBcUAx}MLMnBF(B40f>R976}?pm_j9G;jn-;lTyPf;!=u)_PyVd$}_8?#MojTyP<*WE10>5fq&R4tP+`JUPFZp`B4%2+f z4p%P04@dVWR<(X!xoIh2cCV_&@=fY{>>nZj0^c?!`IK8{61?s2U8*2oec$I8h868u z!VWv^d%TQ(WbMhP#P~dJ|Ga4rA9MNMyaXL7y~ua>QVn|dBVXhoX+C8S@?{)nqFuh3 zT+?_xng2_TTcuP1Zxg<(ShrWaO+Fk&ZdBk`QdfGZ>0UivP4*ph;_Y<*&lCpNX8jgB z@pgS~5ql2ae*A-h=Fz}EY9FKe>zDA48QxCt8g;`vmLl@;ig(KB1OHRp2o5}?_ib&5 z-_F`8-a+qtZ6~Zt_YQZ7_3dao)^XNO@lIX)vw5F-s6OYt3~%dwT`yCLx54`~kMXt# zz3gKt;BQ6k%xZXB-mv7UwRmxOdyV2(>|}f39TUZ;iLWc?M{ zw;a=Z_nk}H?|R43yS4=XkUROeA6imjb=J4H#`8_%{YL*g{4k*XCh3hT_^E7uUG?oR ze7(-O&b?py!q*+_#?4*z?Jq3Z_$T+PzxQ>AanZS?zI~Va@eFT|Fn(S6>HL!=7dxf; zhaQOHW>@{W_12|`!+YAUq4P-Ess07vZEa_3zUPh~T|&yS9`R+ zedOyt`>LQN_Fw*wnK&-Bo5}nn!OflfPV)VdTHn@v;tqDp)VF`+;!t!cpvJ4e@v4?NRWqL-5mG-~QdNH&KF>@b(z`(I|>LMm@Sh{W+~iYhKb- z9Or@`(Ta`W?SSh__3c5$+w31;eP3Z6a;09bdZ6`k@b*}RxC5LwiaqjCgRP3Uxt{t? zjH_$l0M?6~CEizh&k<*P=*=Uai|cjali(}j?@ROX-|U;zdf;E&j=l@=<0x@sll9Lz zGp$z-Sr@@i%x8B^^gOGKg0~wY&$2&J>!E|%UKj^{Dt_wG_96LgZI9oOQQx%xORa~8 ze1Bc>R+aWy{S>?vvHxE2mayJZyv^~3kp9*iJWKnsYw&47@fQ6TrogjR=3NfJ=e&37 zbKp(pU3)T_|2Jd&c@Dl*nNL>VPH?sZ&h}_O`gW5icuech%p6lK@-@Y}^LDZiVv2hYd#`kCg(>%^V=;kTo50B<*_Z|`a_AI?=N9_A_s zU6a4aKbX&nCwNc&hW~*j7kWeZ zruvfM?Y9$`vtM#|ino(~GQ165=Y2VK*z`wy@QX{@|1{&c+|Tg#d4mhq=y#GUnm0=N z+4No$|HwnoQAHkSParR?lgLTU`|3UvbDsC16mNgi(7%?j>EV!gKJCY5JnTbW-=1N7 z6aSr8ydY_P`w;kxd6Ulre}NOb@OE^1sVOZu%l+^Y{Gew^{Mm<;F8p&%ab}LAR%qwm zab~I6R^Ps6f2;LFd`W#<>FvT}zhi!Hbq979JuB$`{u*Mof{z(_XL$QIVT_o8!;YB&L`w}TRyhr6y%V;4_Byvo>RQ;P&YMrEN#E} zh&)I;sjr1jQaM& zOBL$M$K`iEsO$PftS8&HB41C$)$hh&kDML+?evMO=Qtze7DXRl!tS;3`xA--TjYB| z<%)cNWl3H@JNJ%2>np4Ex7u^^ALjcw&JNwbpubOQr~4OT`F7#$;G)tga@_cFDSQ<9 za_)Kpyj_(RyxsWpQUm>eO%&I+SHj!7P9*qQF6z8*SBA=~_x9NX}?qA4%*CLZ^8aFC_ikR<9Uf=f2I_li- zobF?Gwsh9FbsfF*bgz!Llk-iTc>56XKkGvdI%Xc-c-woM;!p7Q?|cw_#QuXH*dG#k z|3=qYhPSE5=)T=y2|lm-mErBA{ziW5vf5Q!+f}rmq@Cg&^vKtCqImn!&n?x!#oEqq z=}++vc3mdHJNR$DyJPTFhPU;8!1MjCm^hC=)lLprb?C(%AH@Ij+}Q};-c20G%4$U5 zvgCa2K+>N2_PZoFCj$iOjkEtyC%&xtMM?IN51R8_DhK!fSyHZb;_QS!X}sU)-{$MN zmSMIoMpgK+g?ZU6{2XUI?CJR4!fk)d{jy#gSm&ENakkvHT<83iyJ6yw(YqXH+kdcx z{4<=L;C%Q|S3j5G>=7MD{rDYUQGK+vT~*_bq@C(r08Z9+@O^`O><^d9YOnaargep+ zo$8&+l^^9X-%Hcy0rA3B?z{Rv_Z`?d49>1`Kify{+_vmfPg6N-+>^)`)?6EBkAbsk zzkU>)-RK8rANwQCKjeNZ`r4Aq%c*{3I6E}@-g#~+J6}Hz&YsclN;)ad{`3P%Z@Mq% zA8_`69nN;Jo66lzoV`hLc2Qnw5BEmC$}QPPZZ}`^#|_reN;sSKbcOZcZc&^~T`$Gi zM&966&D)^=GUo98AFeCmY=`l&pDy}+rAmA>LVr!&kCWi+LlyLwyzx|>IyCm+061IF z(^JIXs_(V$fHR@DE4mg|kf4Q0URAZfJom%<2t#0~X zXPq!Oj`ar4hMsNIwORjvb-n9rU330}>-GEQq5`67|ui>kVzRT33AG@gueb=fxA7o!Rc6P8S(;?$b zDb6m+7x&Sg`vt92m*M-AZ*XZ|r=`8xk14Hd^IeQvE7(`9k5|B%v>$Gm_UpAyANF8B zD&!@UPH;B;KYknhV*dTguaNdxT_1j7=L*U%_$=GBt_^=qA#dztevaTga_9az_+qZ$ zYwVI-Nbq$4zP9Q2p=s{({g}-m@}yp5>ge>dH!}Pl)H=G)zHo5&bOpZd1!p&VW#oE~ zyb0gsQXH&wY^tn)gTdKRuZ-SE{|)wQW&0+Y^v`!;w11bY_8(~f;Or{jRk*2!KH_h) zypL!4&-JT@zVL5UaW?w?2y%A$`vUT`xgu}PqxByIkFrh*R#;rdK1&I`a2SVu^@?ZZ z#1(L?+5acYKKuy36Z0#0Z=8qP_JC<$)P38L;^U;BLB>dj#2j^rW9nPa6Lt4?zd-%h{O(ue*KAF8y43JL{S2L+9iCm=n{p zW)t+FFBv^qT|1$t#{44nhyG?W?{uH>s&!9UyIgLqQ~K+c>%=dxG{)IcY1!3tUUv*E z2d#K~g?8?-!R1;j9=~?G9EV-|xwCTJ^V1KgT)S{sW7Bd3(VcZq@(c!tDIRD2QXjo; zIfC!$KI0UJZL1PjvtD*+-II8Rx@TP1b~i1fKZo>vAN{oajmrUk1bl9AcG^DMM;^C- z-n7?q&F&4$jyZ3zX<3k8hQp$vW%T}q);;-N!VSxTscV1wgB7Nk*0q`6ch959E0=4n zxY*ye98%Axi$uO#mK%Cb;EwB+FD-Jpc02dj)@AI$YVB5g{$27Yv3w5@zj@eO-LGw{ z+!`ZFXTEPC$)j-2s(s&rvvb*juW4QTKs_?&1cv>%9^UbN1>)tAW#OEF*@vL}7W`ey zzQ&jM#gOsR_T*8{A@{WX^QJv-z#j^&!hjgvZm?D?9cL%!u{&{giktQmFZ#swU3Kl+SC^Uw zXMg&Gfys}8A2gp;+pp_6J2#u)C+e*SRKM0vDGouN$$p#-^5|>T-rL%)p?a0HQ@n%T z``WIi`*^JbYEK<)CpuozPVr8l{ef#L?Bmtv*zvWBcS3!i`(^ZZuWRP9mOmeED9*N_ zmpXen@HH>H63$NBueG9s%QeP%kvMyu;#d4ghF`~^&uy)1f8zDa@KJ75oUOlqQ6;`o zJLx}e@aw8LyUzY<_D3h{n>@win(rRI6`bwk9}=9MeiuN-EB&V4#Z=6?n)D35)pOta z-B8P_yUQ;8;s{Ytwj`L~_W&GW|B2>%^E?C}Rv-@Om9g~)>mTE}Ue7abu#xs<{K_DC zHJ3c;M)GNPem=ohdnWXs?l)fn4}7Z!`S}IDf5Np{kiXGuS@}3PZ0cpQZ8%_D1k06q z$`tnvpNE4$t@=BL106@+%@um&ecV_1ol343lKYP&?LMEp_f*o)LelTI;&BFCG2ag? z#oxJEYn6XU;{%ba2|duK{Vwk+Yey=si@vf`utWI3V z2FQz5uE}`hwSREAqF({oCeN`>l)+}aGxg9{{r+$xd=NbvJjgjN__1zM>|N$Ow#4T` zWV6|yB9{NoW#UT~s99gKr6$k0oXgWZ0eWX?o|>x^BJ3S?Gr5y|0pDXNvu=oc*ILcD zEqlzf=kjrXUOOn*Gjey@!0)cOR81%7|tBaYYKpO5J0bbOAb zhup8|tz9-xxd;Nwe~5t%pS;}%p7q2crQSKkVr(HrnI=Oy%a z@tT!o58@=z!&Kjfo2LFK$8HJz29NLi+3-ODZ*Sw=Ujshv5Zk#|_gTgKbNQ~z<(h_z zfA0ql`^S_1@$)~i3_s-+PUy2cp5eKSBL7Hq{|CS4oZE|YW`76N>|qN@kkhr^|AqCgY#& z+B?L3n%56_Z3=$@zo;{r?~^#!9rn*xnDupstLLj~*Z7V#`jPVCuz8R1rjTEmxOsE@ zeMm>Y-{{e5Gv8%|zs?rkb9BAO{Hy4DE_|D+7@V~8`lB`duB3DQ(fGaVkH+t9)%SEg z#P3-nF`cE6W)w7vQr)3?&wAPxn03o{ZR37!x@h_ty6Ix|ydU$wb^JoHVm$|4zNPl$ z8o%TB==0_~KlfXwOnm(T{hRq5*Y<9Ytol21`rVVDSFZ6p#U(x8m3}ulvuDTtG}P~~ z1pT}DT?(}`{H}?|y!H2rt2y6Rq*v*ZUb$bd&6R&oboZoN(C-qAOuBFQw(9Lc-|VOT z=Z69^9{uvoL*C(UH(>Tu&FFLb`{+MBR4M82q&zWXzxmr-IY)2mM%S31`3^`vzjg3| zagL?GyOqw5-!;Qu^y=?!)pC9Etw-eBF7w~W`l|BY#>ySvE~8&Ib*d)kYOTS(Z=;{@ zPv*;xC;Dd{{Tmm5&?o!)r^5&9tlPcQ_yg?wIQHN~u0Q{5qT17R=yxJmbJgn=&Vw=z z;~j9;YF>@{sPP9fSy`*!cPSO6PwN|q#{D*Yb&A9T&3qk_FO2p6`lC&=PKFaZP{jY& zayw8fR-qyOqCLSK04% z49GsE*V$^D-$i(n+QqOahx+XDSsy-((GU7e=EUS%$ak7Nko4DrGs1a_(Xt;ld7SuV zXUK`nxqaE@U#y$^TdsOsbzSP;4sinfXTR@<6c6!UT%Sqs+;s;UQAocsNqwe)f9j~u zP`5g{j2vXpHh#71d+phG-3K?j2Xndh%5}#cMPBRVy57iRKK5plZIzm++sO{af-X1Ag(@P~%$nKP|fkFZ}&- zg|I<*ewWGDbYAz)+@*DSxq0R;t=qdlKXXR&O#UCvyvDq@dG@QO&NDvy8WS&fzQ5An z2WEfkn&Yk1zU_YRkLq__^Lh0?@}c6hw$kGU%N3nh*L)sH?*Hm?#nAcLW#rAg9$t3M zJbrH(okQOLewp=tjrZHeXPhTv{-xu%Ut4aN{RQiv75YD;90{5_YMN55}wg?JJO{Yp|{?_n~FI|6!+o zzRWTh2<)rKy|3^NU%!t_L|YAn{GjE2YPn&_p8uQgEjBP)zs;5VH~*ggt^8>`HTzZU z+UNM&p`WClv~y~BQ1#uJD)Idr?oVMK(Yw_CXLe$vu1n6zZ>W^-$KGZ4x9GYrNRjV_ zY2CAb^9lcap!|x*6UU49<0s=`9}KQ}vA^^6|F?I>!Es#oeSgba;1)QLSXdMZlc=yD zC;}EC;V9A~8p7pLqe;|8sgLnYD>wG^lxb~sD%;1IMA1y7dcjaML^905V>>kHX;&H6 z#>lF2Y&ou~W+n?cmg!oq4r-zv=%zeoG&X14k&i0v%42%e&+qN-EqE|O$8x1<`~JC| z-Fy4q+u!^BzQ5o5z2E!2c6~z+lP3YXx9dyRUkmi3#PdmCuNSYExFm4RvgpSZu2LTQ z@9*(6UO6IP=pW097+Ol)xd?l!x4Lj#`j_B~ou75@UNMj090taL_oYA3%@^EsxliJ9*IM6Er?$rH5ntG; z#9!@;eo^SN)X%U6|EciHu|N3?{^AP5W-}h(ymEZ4uhn=(=Ot*OysxN9mr=SqeBadt) z9|7fOcgEKf`{L)-UybiS^BeK~0_!nwJu3?pSs$^i;`xgq?VbAP@&8}=-MIYI&# zu@58a-7oVm{Lz-8JYA)kIouUT6 zqxYqz{)bf0wfQVP*Dv9-PPs_sRY)aqA}Yn%6gLZTzt!Luu{ZdRt#$0NyxO(Z^{>K5 zy@QNbQTB7TUR-`4lY8ZUC0@VVh+eP4H@#kMu1wwpr}m8Yf8@j07DCCh11=0ehlOuy zep65XnDrFyr^TLN9TB{K7Ccd0!^JCg79^8xcE zzeKZJHsW-{LukITP%k)=$Xef-N^@q4JI0dz?C3>Un*J=H?i*+gYDCLPW6+a7d zRpNVJukxN9#PJKGK}Gd?s#P^B>RC@fno& z!IA9!rmXAu!;Ghl&#<)5OUXFrHFiW+W#xv{-{qs74~oPzQ`*$8tC@vPx}4@ z?<2P>+J9BnGZqRyJK!_EJ7WFXpx}`_-=#A7{Vn1NTV>)fcf0OrfIosJwcmz#L$SBt z6nfP;FT!?}Ubnz>l=XA`m+lC3PRpm~(AZ<-N3O8`JI1`B2;H|MAIQ~e`KasjAA0lt zEv3tu|3kT7)bs7f-xK=JMW_q8(x*?!wSqd3Yb~dq(EsPuG5vo|9oPTEXXEIxW|!J7 z{RnT|588JJ2Iafm*w2xiyWvj;j2~Z@`ge&h)AO`_%y&xJz7^J2esCOmi(EbdJ(T^h zjz@TNcdK9Yaemh$>Ty@8yZ2XZ@u#QjzszI)r|>D`hF=!_mG=(F{!wL%9@h5R)r+1| zj}M$sSNYzfBghT7Z^`^sF~7a3>|d?Zo=f>kUOnD(pSlXZ9UYKzV#i-^SG4C#DNd^U z`mVXqOZbU;Q~jUfQ`;^`__kdx#od(uI<_c zIR>Ak^5)C;a=YFE|Djix^lK5iC;dw0y1gH)<9ph2795ZL)<;g02Z;)2zZ_pr{Kxot z^_%hiXTBZZpZZ>W{lbsq=V4(;@;IwATVp(3*co4+-WNZw{c3!F?l<&35i}KvU$Cs& zh6eU3``hDkfAYI=Ios28O23n9>nU&wzM?$(UGS~~PHn2!XQ1|ar%D_MTp;#L5o^h5 z`U^g%N^B3YXNad6hdxS=o6|UyQ?G4pp^tb^^72R6JyJjAn161N|5Z;>+cUXAX;=J2jF{jRpJrlydg3lGkrR^W*d}(3YQJypI)+p=h0sC`qsXl$Y#W;{ZT+Xwx3$p`v zU(W25+^RgLX}J@(9C$Fpsc=}8^TuL7OIE&`%$wr++5SU$>X&`!P5hlZQ#o^E zT-z8@9(pAGVcoA+=9Ss`m(rgVIJmKI>vAm%UZLLtu4C|-4_`^1KlsA3BFba$q&Q7Z z$RqIxw>2KfXL2Zh^Col{mU#u^dkgd7D)kE9`t)ySz`9YF+OQ2dEUu8lyvSiDU#snf zL=N*Jhnc)~>bdkj#dp6D=yeEHz>e}(*lh{Ur{z}L)xl)r5_TaX+(b^8r`^$*>sH(R z#mDOSyVk4q_+w<8qQ@GC(BFgj^B6zND%~mLu0-#khmSggGmB-uQ~V2_%)=;ufO2wQ z;<4Oc5xI={sPntmPd^mZL~gf~YgO=Kbc^%yF%Ld~4*um8dUWHUwFsXdofynT9_=OP zO<9(IpmC6oNX{AK{L2H42iV6qN&97;-|#?ViSnb}#iyH$lk~@;|2w&d-`e-Pxo=TV z`u{%e%g9r5->p8O@j>0;hI8;$x9tYC0P5wN#Fb|D>IyndH|Ex<58Cv%gkQz}v&ZUJ zi5*t z=XBni+oT+|?v!h#TTRF{$gA7s+LHd2A8lOWm`J%VaxR8kea2hX2Yu16{s%O@v+?*q z;}Y_y$3yfh|I2v2#qnU}Qn#OuM_$IGCF4Qdut)s~|7YV-zgP70fQ(0xV1;Bn()k+a zr&>_}UEAaF2;(7kX83m-1Moq@?|Fyu@Hkf>9S?OpUROoo%ldau`Gh0uYCRF>G8E~@ z9d2_DzGHp?F3LIxdZ?7)z|8s~buMJQQHA2CO67-n6#j6Dn~UYASMwj`x|`vg$2za% z;|u85Ug1BUuVL4HaGLLB+()M2ANU@>@!xNpMPIW;YY~3o`yUDgryO@8gioHA@oW;e zE7xjX{h3_LGbp1B84_6z?6EAZqk^3SJ=;1=~+yQw!~ z|FX18JGR^hk|#f=yW?}dmj`|CaI15SlOujJ_($y#Jd=6N^?ngAKjgZzGB0&wyCkv0 zGCp)a+?a#-!iQDXeG@)R&p+c_U@P+Z{x%;zo8Ut8!!mAaUyK)pDd9tv@u3~h%arpS zt*`t14Y}9r(r@xfG_flk_)jsf?c~4r3IEYv2mkeJ{uBRROo#O%Z}5le_TWFUx4l5G z^A|+k=+_$jH{gi8@tuy~m&jA}n~f{vy%zop#g2rp!1e8U=Cjy0tQ-1Q;qSHd1#uRx zoI{q*ui)Q6uXm>FTlNn9{p_dVJQmztR;$u@l7092qPBljzMQ|N_LIi+;{S@Q`;7ED z>yA_?FVB%b@|NUeAB^huV_a}sL9L8EyB}PLGJnR*Pnb7`)F*!Zy39X#AGzr5qrVm2 zZ|lG4V~t%hpLQ#aCE-t63TUp4$I-_ddflu2zn1$I^7aYQ&nw=y7n|UMt!5*xDIJM_ z`)Ff;_U)$wiKEY}6Zh+R;uLmziF{`2>HB4#eq8LvqV?0(@Adp8UEhwR9Q?m1*XFvm zovOUMhBS_h|MF0bFaIF6Yo|WKbNI}Act~($E%{5=Aw2AdgKtROh|>NZU*g^B+>IZN z?f&BDV!M9oi@N`Bwcl&m*Q@O5HE<)1FY4?!cJ8CHo|)lC{V{D9C-&xu{}{7$sg50x*@=pER^qYRc4GDS z#7;={=4v~U^2kGGCvtyQaHy*1lbz=muAhD}lKh40&M3AM@5D|FiMtxZPJAQ7gQFgH zVi~{MAh-jbsBB&R(|X=M2rl^e`>f#8(tb|jkz0cI+)Mjyy+VJ`2brCi+_&w-_q#e`dQXD5?r*kT|&R;N^B=yCjLX@y?nB9 z1%8(KNS$boA@o60>_qiaeTnkHy_|!=|3@BSU5I&**onMH9=L6?FXjKdofv*g<_W9p z#8K#7$8Tl@XBxZcUp5|R8ZcSE9uLv;{C^hyd5h!GD|*WZFO%`;)pk4Wu4yN_^NmB~ zFKXL~@p!DQpC44m(6jjG<{Jm0Z(=8Qi=Bv_*B%eWJ{g}uOzcEQ>_nIOv)G9~-?;@l z5&U$df5o50PK4ite?{*w?ySRU`DycCcUi}w^hjP;(HEQHKbQWkp{GCSAusUJhzI|m z@9=Ylq8Cy<&G$bPwiuwd%}$K@XO*2O{NoD$@V&@Q#y>GVvi%4#J>Dw+d};_gv5Gw~ z9P=&fjGFFg-iMU#1+f#GTJEcJMdqD$HK^6;-*)W8>gx4@8{3JcXT-m8c#qhL%vbgZ z9|{+uFC%7sf(te3Pu4R>_ADfP=>2iyEc~6Ur}R$1hsfK=9>E3rXTP9CUn}P;tV`i% z5j)Z2eoIsHUttQnuElyuLFOr$A7W;eofy+0vlC-_|J-)sl-P-U=TwYe@ZY({A;}|$ zomj$L()Jp5;(qKzkN6MCe^P^=*I1wI7CUh;(+i|+3&gLJ+6M#_$9CdS9Ct;0_VbNP zxm9*z8;9-xf!&3iPQ~*y{Qv)lX(z_?%Iw6LUf_C%oyc?a zPQTcRivD)j$rq1bO3v#lV<+wwJ2A6sW`6P^bq0OYCF{S)XDSzVAN_42_wD|wKW!Wm zJFm1T_U#6-6BXt6!Ph0RFICi761~#?|L|fHzoq;S|MLH$=<^l-+lyuNj@XG@iM!MB zYXz(HY~!HB6WkYsdepEJoAB2Jb|N^rLF~j<_uFSDt|5OZA9UJV{#Z5A>)!9Qow#Pd zqxc+$4*M4W-M3_2B(uNVmn8lz^-GTUH)F3cjyJLs^TE%#POn~WpzP2PDz+2VQezH( zly7ZX8o*CzW0y6-t7M*uAMuLtBX(gfVxGhVLZ1hW|L%&cm*m8M*c3Z}`-6ge*i}A! z)QepOUiFEd6g#WKuIkipnO&teE$sr=#NQBrGjavzZe$1TkaEc3m|QF5i6;_2TcLed2BZ`uVo_ zdqV8MX}?vUVIAyR|HbnIETiJ5ji?^IQF|%`w=BQaxB}kEca-vKQLY~6Rm+ueTFd|Q zg8X0aR$rGZ?VOiuxl4URuGlmGN&nxazN!E3QZLH2WUFt<)w5Nq&r>_emvOEz-?S>N z2Kp`a2MM2Qde}|9Z=_%JYhA{_h93t1`~-N|1pkZ8INr76Ja}*43DCj4t?M&?9=ZH-=*@k){ z+O$l${ld?T5BC42<%8sNlySpOg`cb9pYIZW_R)s6^pAO3$GW>$5PZtkx4IuVWm1p}*j{Eq-*y zufvb7PF9Ftka@6-UuN%OXWzVh7Q3~rADm){UC(;*rsYG}yNO>+{9v;+!KXM*DD{JR zdF-2zcGEaSbzeL$PUJMjn~{6};*O;QtZPYqMe21od9qG&ZLFfl$sc3<%j_GrT0_et zjPphu@0+E6s*bpvUw81<;4{^3uD`7wFXK-?_}jr}tLs#EuX2vlS{`Z~uT%X_#lA7r zSUcWB^6c?>lRS^~zHMLjoqCh^+~I(ChlPGs$xloFYV0fD&A0`en{uZ!d)!y5vX}lL zhu&VOA zGOuJ`X{(C6W-D=Xlq25_erxeFg?B7tC#{N`8>T$H$_;72BJrxUq$#xVQ-HLA7u8)+Vp-!^DX&9N}qS3 zZgQ@H*Fo=ajCw-jKFVb;aiw^8N!j6UBYs6Eu z@xi}&ap`PEhxI=Dd>B9cLyODc##*?qPFA%(vkOBv`S)gWALR4w-ymPGe&1%BfZQ{$ zZ{HVlvQBcTK)g5o`)ZHa6B6$&?b+&l!GrF6ul!#_Y70RU?-bK#m)E_xGy;E!A4MUj zlkDTJvpILN#=d2-$ELv>-k$(J7`IM+l`90U+Yo;W<6VY6NgS#2pgZM%xVw^TmGf%L zZvWS=7Ww`=*#}X9@84UmS3UeStiL=5U!?7KsV^vz$F4m!>^}Ts?e^<&`+MtgyfoL9 z@#eX`!MjQenLe29N#zDQFrQ+EU{H4=5?7U# z4^NlCm*MDNIF;+_kM{`=*H4OktSBFxcBS3szjR91wOct|hwq15&p55?>M5VDN_q9C z4tTMmp3>ha&~XxBYm zmv&42&OO)FuTEE`pXFi4zOMe(yl%HL>72Q){^q>yXRzOS^}7D+eMr#%{f>QI{n5Op zL->!Kd#QeZqP5y86R;-OuQg&XMcrR-M=1RsRF$)$8d$uj$bISI(L1=|5i* zI<)@SvGMD_>)!Ib3!l5<_kxeC>OT@aoYIYjM%Ao_o?{0{SV{k<6nxe6aPMbUi^A|zvsWk_fy}Euhai4e(sd9XGAZRV|#|H zwr5_e)~e{;(H(L8ma}!1`DLaj%jETZ4!Oim`;}$p)n&2YMT`EzTec|sfX+O*_o~{Ph{{O_a z(#-N=hQEJfe*eD)*J`odFi6=_qyW9H?7RtxGb*ed8#E!)8H%Yv6KpfRJ_EAQBZ{lZl_RYG#{w(t-sW*aH zi!}^AOXZ&r$QMJroj0rI%lJ=5L(e;p-FY~u%|MTI-sbF9Q@5UC{d3SG?*ewK&3B=f z)!|lcuEw=pZ&VAgAM(%F$!CLgIUOmJcQKWx7T%coTB{Esv8K7C+pumzvnwU z#&hlOR*RAsiTzB^lNSm5c>Q!;Jyj-NOY$R8&-#5OXUBQyF73LC{d0W3PrdJM>O464 zEv4Fg3IFkQQ#}RUdN$s#^!;Xbf}Z00Clmd@*rLDdRPl~koo}Omx~0Vb?Fl^?@BY!c zGj@NMiuO`J_J(?IsTS~m{;sGtA2A;wUyRuOJ;n37{;u6>aQk?s2YQ12_~ZG1JN=aY zEtYG6OT1n3UER}V=p*M~#^2Swuj=fW!Y)bTjn$J?%9EdY%l)i7&SUF|Xs1FwX**8_j4$KxM>4+n^QQyoK>1{x*&i~v{obsf>dBCHaKrpjjq;3_ zH0nM;(xt7C*v-0+p^9X->35lbvUfe`1pZj-e1e9Ug&b`gDdKJ zoA--G`x!ss`)TY$<~O}o?cXdviF~B{NR$1d-dI`llbZQ-Ehsb2hd&J;5g)$&loGqR zXDo2Ndu9IYfe!>niXDZW->p6tgvd);KkH;)US5K~6aNqU`KzZr_^NkY=OMyR6VD%* zPv2V1{2$P-@^nS}$>aIcn*O*@`%~?_%v)vt+x~-RMWiQ*Y7ZN?dfY$aBHH`+^@e*(Gd!DEJZl@cS0)|HYg- zDc4p`@(2auZ;j;YTIzQunHLu6{{)U0zHYTYvZnV__2dJ0pg}&LN#YQi#Oq9gpYZwG z@5O$;o_ux3!Smg!A@9|})f>LYytxXF(1&e>70DmL{>g^q`;h#uvM<=a4L@oqex)H_uNxJh zXNi8wdW9$RuYmH*Q)S-OKof8NLc zE$V6i@Bg{Qk zwH=ekLMcqfsO6<1o-m*Fl-yZ2Nfv@<{<(y{v)!pB>!MQB$_w`(>qsIl;kwc5~ z>ssI}=Ux}jKxbvCzgm7$;=kd)8giQ4r}KimdZ_CfDRrs{?roSVN?soCreq)J zO}r;`@53L=_jl$QP5aZP_fGonLx<#ill)Hj3jVSz`hV6^_w<8%q2gR8+jafSk2|i` zeQ)Uj^ugx%{WPzi+DNONWGi`rh9Bl4%~v zo%FK|ACJqr6Ly+aySjWp`V;3fP3IkzesHnQ_xH7w=)FT7 z?t1ww?@7LgI^QjMhq2F!GGA>m&K>JHX+Ig?{LAQ#Wc-#Xndgj_=$s zMP5aor})#cF6NDbXO$K6BXys1t&Y5<^Rlhf!?-Ie+HhfndWJov@eB)B^0&12GYys1 zbNao|`s?Do2cpGMkq0{TQ}Yng&gg{$mijH77ZKc*JipDndSn=S*5T`v|F)7Ru_=D> zc)j=@nHSXPcRJrCpK2J)x{E7TonqRrL;yhYDWCS2acrmk$-&n0`|{Z^%dynxhnlk?Iy{i6)@kB zen9>-A0GZ8z85^-C;8tP=U`e!Bei{|8yXPi0Un|mq#C}kn^3&h!xrUrz2OX6>M5@2ptIv1qC+{e@j2>Ut z@P)4}=1U^w=4$9*_`t}4!2yE<1_uld7#uJ-U~s_TfWZNS0|o~S4j3FTIACzV;DEsa zg98Q!3=S9^FgRdvz~F$v0fPeu2Mi7v956UwaKPYz!2yE<1_uld7#uJ-U~s_TfWZNS z0|o~S4j3FTIACzV;DEsag98Q!3=S9^FgRdvz~F$v0fPeu2Mi7v956UwaKPYz!2yE< z1_uld7#uJ-U~s_TfWZNS0|o~S4j3FTIACzV;DEsag98Q!3=S9^FgRdvz~F$v0fPeu z2Mi7v956UwaKPYz!2yE<1_uld7#uJ-U~s_TfWZNS0|o~S4j3FTIACzV;DEsag98Q! z3=S9^FgRdvz~F$v0fPeu2Mi7v956UwaKPYz!2yE<1_uld7#uJ-U~s_TfWZNS0|o~S z4j3FTIACzV;DEsag98Q!3=S9^FgRdvz~F$v0fPeu2Mi7v956UwaKPYz!2yE<1_uld t7#uJ-U~s_TfWZNS0|o~S4j3FTIACzV;DEsag98Q!3=S9^_CHziVS4>890f-LWXcqF+TT)3H|;a*EpvFE(~h;h$s$%-XbL#(7z3 z&d|6frluAD!CcpQ&#m(_iK#wmlgEyT85p%V`eXHhE79B`tsBOVYI--UL-wMoMys3~ z=f~U~cE2RtZ{yZ?*Q1MaD%^OxwykvBz2QUg^~rm@GXyaH6PbV+#?s{tA?MW9eD}0A8+^=K5 z@0Z{;zuinUd?_WEpU!%}l5D^F(a8xfH~yM(zTT>&r^P(G)6Y4w5!2>M9XgMWUEi_O zypYFD>!bJW>k`~I#Jw=ExYU%`w<+Sl!6j!jnP#g`-JBB{UR<%X@kEJgWb}iTsULJy zMncxLyuADGoF*xl4ozEf-+rCG$^=GsC4b|xyNk3>HVt{=XvX=H7J0e8bDx~AT?eL= zsM_=m3D2(yJhg3DPiOh*#i_IEeLoEC_j2;-s)P)^xyo>};K0)@hp<_smycxppn$p!0DzpTd_PI~?CQt7(Q;#L#Ko?FnY^FAK)Lcf9^?)6cg8 z>%M;QO`r9-clPm~NB@`}_F;Yegpd=bik83F_&{YowW@Hv`5xzs4rA`NS~s@J`D66B z3yqiFTHNS5>(_@nx~T$|`%4zjf7;~yc=J{=ob{^Jlkt*Z=kc=+Zp=D6zv14O?N?@> zt0}l8-v4mcg0;&(Uoi9<%;~aOy|4S3M?=@oIWW=ke5<#0U(>5>rS=mVOxcAN=lcc@ zKCo}Z@J#ob3)kj%bnWh&2vdr&y!#(f$b?~v7PM1mAw)tFe{tJ6u`_NSoEq?^Y*`kc z7%*gZ*2~0`{=0)`Z}?j5R$4mc;?$}qMXL*kvsG=Q)iXj*zkQNX|LxP|8LyYEPg`hv+moS%Gq4{Z`X z%UZN_!?blfj~BE~@0}BR;_SfV$GQsj*6()vJsEtz!OnL=^f~+JE{zY4?Z2;`_1m!% z?_XH$kOz+&?s!9WFWby(i^jaq0)9fuVTIr94LfD;&wU|D?fLwN@>J%GO2+NJ-Qy14 z-)E2V6AZnTi!4-oUyTl%YQOtYZLNdtL;G+l{!#a@jaCV|@cnj6e^xB2cp0oKTs9=k zW~#}N8iR1{aIcKQeTlwTx{P%7T5ostc*nz2oySZHk9hM(Vzt>Ifp^DIf(H-cxL?18 z9KH7Y(GfepvYi)FTk=K)jQ(Vrq@3?m+{2=W&n(L3$nDQLK8M~8ium&B+l8+`JJzo+ zeR_g@e)an2-8<|}=(zjd2XXDdj$NjzA}!Z>7cv;>3+%ogT>R{U_TwT|DOUm*Fc6Br?kY>5t_WY8#u+!)&^QDg41DthMuiY7? zUU#EswmG)4@6OrVf8MewvGhgKOnH~!PIo=dbx4i!b+|LM=U#8k(xk<$+6H(oNY@Se z5Oh1syFx#E((2QPDk`JwpY|*Xne$uHjw2&3x*tEYH8f!GnE0-P)weEw5}dnN>|k9O z8M<`ch`#R9JB!=L%kTNW`AI~|BcjjGsa?Otdtr9w-lbJeHgl8{VjA}llRsSVF6Q1F z`7&zBw8N)r>WT+HD4((Y=}2!1bs8dTB_H^F;uY?~0bvc4<-+mRQ{Ro;O>GTp3>W_1 zZ(3gG%64alX-C$mw!AF6Q@we))5htsAG(AtBhICyzx~|nmXw>b)^*6X4y`vU?qB@6 zcuMcgnbre2>%`ja?x*_IMn2v?^mWdhR~Itkb=|l0YIXYLvPIWY!?(M9^S3&}*s1ZH z;kn-NYszPyYs|w=w!1^q(*@7Xtanzfbeho6>dC^yab2e*oSpbC|NT4ZGmG3q$6WWd zf8D)b|8I|ixWUBu(p+)H6xP*Q*Rl_0mCR9BUtciqAyMMFvhi8@N^6-LHTpN#Ho?2w ze^uKrS$MR2)x=*)qtKn_ZK_6(zrgVAwy;TgWpDqr_bg)l%zs)vk>k2zpU=LWv07L0%FTuE z(^2#-zw~&@q}^GL*|&W6>Sx4^{oErzo;~%~`Qr}1o@;ORa^10mA4*sB8S}DX!vb%K za8hf&QkX66a>Q}3dVK29+jTmmF)S4`zfkZtF+u1yVQ#>sKO$A z#y37mv(`KfEUQfkJ$u(_6Q@mwn~4XlHlOl+5_|e`m-B8z#+~ozdM#*)?TAk`568&& z>}b2=(&k&8hdkTm(yCK{n}x%H_nUUCn__V-_6qY*f}=FcdwMt4%hP`p^BgJyTai(P zUaj~K+YI_$bf=GF{;^(Z?RNP+Kd7tkI4^5gbH)?a#Yk$$GSsgdUDFe^j zI`_zad^P-f%BEj>M!Bk84ln3vJ#qW~&Z)obyP(;;u|bp5Ys`M{D;w_2avnS+{Q0D* zv2Qy^hjwvdL~j;U{N(prg>!?&`YvB2mzPfzwQpLzfWsg=j|^Ww^q0}QFRbDhwYhTh ze8^W>*vg~t-#>dDNycq1+5fGt;!*ag-J5);`OFjK)bkEr+_vquuglMEC9^s|J}6)F z#qG`R#qqJrzNT*pJohwW7%yeb@~fU3lH*->9rb!|Z!=+F=fmn5OS+F$r89GPM6N8F#{?vokxdO%ud_B{IVjXZFhQ zU^+!tWnX(6a;m*g>B}~Z?o)YR`*u9=vtYIFxi`FB7lX%Gb&N^8dtiD9*|xx$8?;d| zGT`B@-Cx28Nc@4 z>U?yD4;pPZZm+dRMr^>&jgPIeVjn%?uX{6d=E*j<0`gq#c3zoSwxc4^M~LQpwA~tG z2z*~@?RWO_;e`#{Wv{|k1&zG(W%s_+c~jd{nZMVaI5eJ_8+@YI(nI6c9KU{P>(vD= zy+kFtZL|8B+FV-{9`$+8vYEYi+?~C*n`h(N3RZCYio{+rS6i`DS#D9s@WKv1y}9GU znGn)gx<4ji0r5wxo(GRr4qWNqPvw1zl^L zZv<=ig+9D~tt++Y>7{m)seXmeytXU09S|nf=znk6){b=}V{Ny$>#l|n=XQ2f4N86G zIZxDO(L9&B$#a78lE3txdM$u4_UR&fPEeC*Lt4h1e#_&^c0OMJ>x414_@S?FzufqT zMJsW9{=@d7L1R`gW5i_IqCUin!acw2c=A}L;ac~08v4*?#%XfD>(zb8ZH0~e&eE*F zxHWfFz2%xS6Q61K+8SZ^?bq6`KMy_|Z2C*_ z`5|4#v>DuOUP`>U*1RTmszdqqm%g*gc3x#)dUzoyW}<`aKnLllhZFiOcvCq26OyZK zFS1vcEj}1Tjuep9dDn^J~#iA zf2DfJ1zoSNkpoRfUXSsAv~=jL)T?`5_WbG5k&|bZ@1^`VP7~GpQ>_NyBo+o;&bn;* zGb4ueWKaC>%08t(fBIuvN4xH~U5;(*yFKAJ*}C8BWewp4ZI&Ll^tR|z8s+rs+!1#D z6tL6ItM}90A#)~lP5t!J?5tN+!LH$L$w6yudQUrMzEE=i*>9Es zmzU+S^`-L-ZYXhC;AfL#`?UTC_T{&B*`M2V#8%t%w7;F;mP$cHn;-pL^iFou!>Yt~ z%lda#O}8J#7LV8vF>rZyS=tF%U?00aFK6DoU%jsX?(pOy&r#cBR!DkfSNM8KuDb;s z9LqS}YA%=a=7LB2K3#&}PPHp|Nr+=wk6M`0#cqy;MNFo+&8%%*ExL9~z3_g5W4L8N zVDGjsY)!+piFK5>--h+KwwSK#qE?QP^Pj#UHZ~)7QD#|(1t90bkS=%ESL$|-KJ>s_hLI5F1zQDQl zYVXx|jdPZabgg(D^J?axok1ao9XuWuckbl(^G#o>Z0&-+73155JRIFPWc4fJY~AY| zy9LjRhF!gN?~-%XargWO&om`Ow_Rr@Jb&((bxXG(`@Aet}#*7`?_tMtv6(d$={H6Yj8lQ&$+{0)y!@l5z}YS ziKt9gynM8O?S8o%@A#l$c9R?XEL?Ow%`WQ0$IFAl(_hbCXg@6R$^2(yGTtPs%{w`` zZ<}cIG4PQ2{1bL(?RLdUCUpOK_$+Gv4bz=AFGH@({ZiS__h^y))<@6%3#Gr_s7ehy zJLd4n9Y+G{*KRABSX$P8^(%e*%Ndqcnwljc+`JJ_@@#`P6uEx(nf<1s3u>aI@;ZNa zGb8^PT(-Dmo5`Wf-?R5_>Z6!BhF9LY4e!-`pDXuPzVO>ORumslS?d2jPaD{22^h*sh!E^Z~fzDISrn^Gv)ot z9UqcvoKswX3Yyn0CUO49QO^n|_WHnn61&7UCeg%zA)BC_z>L z#FJ6I3DP*|aC$OAB}l{G$_PUq$c^cN!Vt%#-yShwCGAjpy##35o=&figJ%wtQC$gA z*x6=0h*zRY#0;-vXEXWq6A<7hwixpRz)6PJVF^fLlz`$8na)e15-AnCotTwvW&)uu zyq=ZKWT@E7NgQLFW?+mbM&wR2Lmebs0oN27=|oO z5yH9QA1zJIOkxLeOag?yNYBUV91})FPxK%N@_>Sa0%Ce0Mg*oCJ>|QMNzj+K6K}V% zYp53^+aMfDknI>8#v!{1kT(%UsF95c0t7W60*&6rEU?Yv+L$n%VYFgUB#TpEyJW77 z38#a7WJ5g?Tc`_cv+q$$7Mjq{5>{S#9krxj3jGwv(&DGQ5@v}h3a`WCNL@7unI)ED z>T2C;W{JI?y4uu1v&7I5_eXfWW=Y)?xt)&FnS2Ygx91Xk44)-`v{tDL=Be#)Lke|+x3S~l&zyt{_kiY~9 zA=8nteHa4jigbim7I|MqU;c1+uDez=s)x2mxtG?fJcX0rNOJS*NGFYhfvaKHIAJO| zsUQ>CN$gC1Ch5WP2=d|h1Pv+oN%q;=ozp$Ylj9lWU+$Ugxpgi-g|quHX-)-_XQ;Bf zHw(;Gz2~PTd0={RIrMl~Qdl}Cv2Rje+1yt3RJ!xXn}emQmggIoa`uMm5RC4k}Vd6Y^HEN zyy02_mFz(-P#0hdzB$>m+DLKO3i^wSDKy$VE)_*`D6KxCkUiX48+x|4I`=Yi<1WZO zD@IDm94pf>v1ZBT0eDRVj`{+}jobp~{akAG4ptZ7&H{A~^58=acnj36cpju$UB+LaUPRZ)#)JnmdK=Tm#ND#Mc5i}>iLZ5m z?fOg9l1&!PCBlr|nebp zCy`9_P4b<~_XtC6*G@0l6{QDkfji zm@r-=AX#vQYS?NE4Q#UF9MB&Wk_A&t;!v?o9q2$oH-#6IIWTvi&9N(hX9A~Vi6!FS zi=cwCAgxWLvK|E>5y#?PJ(L-QSON99x%ZHGVa(`Q${l)K&!16^`NTQK-T>VJdc%Ry ztMVaYjPzmPei$AKJ%QVl%n72y6prwRo@m|N6CWTn|Dh+I|Jf7H0#Fogz4K!Y+!!r=Cst_%WJFG#8o9xnqDOAF?#P|*hF8YNipOy%lfOnlGG`%| zU@RXQLuqr^@*MxS=azH`CFX=d!Y3|Bn1rE4hGYn&nGjcF-?gMnM==+|XxTF0nT}NK zP2Wp&W5!(lF|UOOX!tF0H&#n5#_%UC?%f1r5kcoc9aZd|G)KY@^|L+ANCqwGdugsg zbnizrhy0eXMZSmF$iYM{oNfsAxjK@wq)u<*h8EY6CX4ITrnncO4>u|{ip!HZi|aPl zEUw$}VoBW>CR+sy0L%*(gm^{CW^YB&kp*z4S0kpfLWfe#sETZT^hA-6qO$c=MP5m? zL?oi9yb>`=Ua{ygiKKyP{O@}rXxVDupZvpm_w9#2XfA@suW)_&A^9&+Ao+)zKr_MD zmOOaIas2lWT4VBG(iuRH%lm@!eu%>)FxCf$@JBxzqsXi|qyylg|L{X|6u05u{s9?X zFHArS`1kch{!t>o8R@LUV;K#TejV!}k9j}Lpp5eLQ?wr_$=kAeg=X*B4sUon)3z@> z-|19dtNwF_x6jnRu#RPk_g-p0KWpvt+db_M)*sO~HMT$LzRDqU(z+GD78c(1=-2LE z$g-~9NnjZrf$>L4NY+VVkUel``Y^qjK0$pgdoac2RJjsGw4rj9e330sc9$aQRhc!! z{AFyokCUfkEnn@}vNM5u*1$R*&!N`EYM1LqXM1eqKqCivf zR>515&XK*AbxfKnKc5(%xV~98@oUjd4FA+kOu{FFZj#ZWn;8D7n*c?#Zo;Ls24Nw4 z3g{-!2C8#k(Ynbq&`qX*ZjuDL$t$C7ViG>}N8O~5%~>&@E@$ZxPtcN-dP1@_dEGb!1RL5qXBP`C9Q>W*HkS*;IfPMey}nTZMFqSG<^$o7>w{t zGD#0s1d^sm3uPT;uhaDZGtT&p`~U7Z|8tbX|IHYOPloZ;!;A*Knbzmmr|FG*h2lk$ zeikF?i5yY@B@no1|6%0bWjPFA~$Q5U=3ob zy75G_{?1ABt*}<{n3&hP$erf4F7jn#PU|8+YmIxQ1D0%*0M-&$c7$2fL(DAduVbzs zYEW5TB@;vZk%|SV2MbUS)>)EWItAiq_2DvhX#!5+;aDmOF)@zK)k92&W7ByMGvHW~ z80O*;{HEilSvq#aTN0&LZjIPFiHKI%8) zbBj$Mb11W$@2hB3v?{Dj{7$qE8}lE)ttXQ!v@!)lSujrQh$s~OaxO!i*U+ppu*bJv+uNS zT;WushGfiZ;R704Hv$Ia=G2lZO#PDLj|hlB0EN}K_78D9U4&eN5nL?(+p-=Eng*g z1mPn-wwtCPq!#ButV#@f1z2yx0_rCtJu)I^CZwdbtF9#C_8bNqN{B+$W|VV7gI^OO z_3&H{1t72$_k2XFD(xqiB%^Y!s=fGX@|<%{W?*I7%FJX%vM9Rj6jRXoZBGYefK3N= zr_;oz>2lCy6B6`kU&Yb{N9gYegv``3!%R}9O4GXr<7cMYR&58NnuOE?X?jh?YR*yt z*{PlODhH~KaG4ob7*M9cF^P=5fmfg51$d81aKuuK?*z0J>VQWQ+6r~RvB@xE*s_%% zWVB&Hp>x$<%|ICpGv_MYmq?HM5*UzeSud1<&LEeo4M&;~7wn#ZN}P}TGYssUA9}MI zO86JOX=m0e;$sHgmu5g<);_ACq8cQ(lmLBsY43DcZN8s+*o)c-*&_GmGml3}kK#CB^>r7zin_F%c@V>du8DV& z?hsR9?mCb(x;GhtwOh?c2rSR6j&OP~5Am3Qbr+eqIU12&c)9r6kes1F2XFCKKzk$S zpxGA8FnIra)q~O>gp_g4ZS1Z#&PGft=Mp>q9Yb57va7#`J{w$k(E$NGHIhW%d3^Y4e+w+t4ACM!x zqrtNiYl*ajD)_QN6@1OWUXq}PRrWSQt}C4Dl^J?uR{?0h7TmdSH&sGwV-RXCbU)VM7-D~BhIrkBt=50`EW6B%m#HV-Vr0Fg|E){o z;C#W%gn0GOkpq?kQCMb+j4kOOb<>$Bgybg_OH=6RM>j%!H9gOzB^tg7(U5EYT(2o#b?i4e&R8}OSsaP zvX>w!?y+MEr=Tc%0FL31@0)a|2S;3v&kBq>o5TdPG&q@w*G2iYoYhE7WQ#>X#&a8p z(+wc+XfqPTX7M_YhM9+_~vhTEoeG` z*M0@u7XFwRuch%dz-!+DUh4uopO*Q@-|$*U1-$llGhQ1x0A(5RS|c(iR^`0N=3&&9 zNp{t7cmU&hw9!XJh;}Nr$l50G5o66e5b zSPA_f(b6~H(Og)y0nHuouhHD*e)(^qxpcq$3pBUWKhfOD|3q_f&9Q!rab>Ye)FfW` z-$ip9+y84c7y8YJ=HC4an#=nOnrnj5+?b(sZ&XGkBPQK%DGY?`G#Bn(q$g5Vl%jDO zs-l!L>=U*)^bRSUY`GrIQ2ovFNc+LE@F6Zr#Kaj|U!RG!@vFKP-QAOvbRf=1Y>11X zRl<9Nj5)v!RHR#*=G3jRcUm|=D?MY>9YGXrm)sy*^4uI^z#?-d;RNBTGa#yq>lz}A zS~jQICg7kxcA#T8^l^|CNsDkSStv#r3stkr;i?3r3t%U<9`@NtRx*dqVb&ovvmP!z ziIK+QA9fBY6MI1MQEJAps+J6iHz?s~C^9A@ zd>ZTl;Fp5p4Z^W;fr4oxkX$$o z5Ga&dp3V{70RI&q!8?pdr3wRonh7V&(*5x}HEcb!OO&M`6Tqi{Mmv4MI_45^a=U^{ z@HFhD^j%BeDf+Ia?<)EZyM>-G@^v6gWiA9y<^&hy$=br@0GAzH984v=0;lj8PyOF0 z0rNJuDowN>TRhN;`?v)ZyjP+}xc-23pfr6tUCt~9?s4fQ=>foFYRGo8kW2@f7f6ARWR% Xo^;Ije~#36{2Qd^{&%DXV$Dbm z#F~*Bh&3ZM5Cf!!*NoJ_Q!`R?A0suR$rhw0b`I_p=>dsZ9Ljlq8x0_HUK~bn@!;aa zC6?}y`AtXcQt+2y9YEptLFNV-Bc>3iU`;0+WRK*v#6tM&(l9ZmqRTljlcdf+;-=|i zi;d_GAVL*18U%fgb%kW?<6-pSIE~-gnS7X0rZY*wGdPvR=P!*U61m`U$1_d3ACL3@ z?z@DgV5#`ck(k0?&ZO_yhk_?FqwnA~0dWMF zKZIE7$ZoPokjxZ$N;gn!k^{5SbR_6X=?G4Zm{NUoA893MgRm<6c_NfZdU54^1z)~F zu_4hm$u^0E6-EyhK6fVR%}pcewlnjY&sfuuR49Z}ytQd$eiP|Z%rvwwW;M24uEng+ zt}{t@E#L-D2H^BkV?7URQW<3xHuWyV7W$rzPba_%AOWRkq>+l=L^iQWAd~D77E-hRfxD0TK;nKmygG&z=A1)FuB*-T+p+&*4atD)Qxbzs|ktECc@q9(U0*fuW z0?Qp6HB6nkI7iVtnez**ZyM%fQwYr^_3?c9PZDK3;`jz2{Y{ISe9IjhI3`!E%D{FM zZGjFX(>Zc|e1zfRzHYR|_rC`*!A<+#7v6^irbdS1d-h)Df&k zt8>U=YtHkR1{d(#y7<FPx~?o)U97N`x+MmTGdZTei6xX=y;r@#?%o zI?kC+di31#Lwh@x#m!ex3*1-ex!O508l9;{M-;v7DnYNDhUU=J%z%qXY`;R2w=#iF zN0#v%o)X2~bQRiV*;%0HC|^p_=5u6-*o)}^4>!YM%mT0%UxFN-TyNBGv5t!`Kqe5w z3~EH{Trvm&(Tg|_!{-8?7B@k0aEWBpqxTFNw}TiO2iAhE2_qRR6-G@5<7spZ(lDmi zuQC}uBsnD*6f$I`^8i__#GHpsqAQ_P*nx$mGr6@9>0-h+=5vC?MtDIu# zS@~&dl)seZ7aSl1@|UUuB1D$VpLQZx<4YvsE9$D~7n z+0~xK4&`>m4th?=3{iaIN#du1B4tmF>T1tOJytF{RUVh}fFdLxyzT=p1J%pj!> zQuG?uinhc}pnHs;h?`U}*;ZO7s{tBwNY~3MtO7#x1)2Rs$D;oHAK? zbF2<%P(kPl1d^hOoAA^&Rh=KYstig7`30fd+5&YQt|jD|5f3m$L8$VEv=Wyz>|tcj4n2F|z{D3yz(m~R> z2&}j6yMo9ryMoD0*gC;JR{@>H)q$QWIO}zDj`Rdy8F2>m6WP{?gG?ha#u2Zlv5?^r z1HNM+df`yOLLx?n0Rl2mI6^d@#zK12Scm|y5WQfEFi14M*ocKh3nQ={Mq?of;9t&> zGKu$67F^cE`^nANg5X0?8j&KL0=nc)s7E)-L1jja zJ!OTSH|AbWU3*B)L4+326c?&u$%=8mf=??dWE;jpnjot^Rk8fkxDuM@3C;5)SHRHa z)UDGITEP2S19IV-Q}+{@Q}^6%cj8jkq+hCl`toj=hPO7DVD){Aj+w2;mWBk6aUGDN45g^r$GN29N1+6 zDCfm-ora}roAdmmp1@d&8E`+>kXELX(WDFhAjwUPWe|msXAbpj1wHX8jP3>43)&Gm zbgPo-Rwd`MIdC560B!lj;Kre-Ra{E09N-BO%f!ZZQQC)GV=bt{hg{sUVjjB!>H#%o zS9CJmjO$En&s7_EU>Q5G;G?7B(Mmfj(@-?&lDHt} zxtmz5&bNQanRF#Lgyca=Z6Kszi7n4@_!(M^pL1T6khs-YDzjufnp8!%aXZB}nE8uy zUQ|LK1wdV(&Cp%~d@aP9oaat@aY1e<0y+M)6kKnb#vEvJu(pqDg3HBa&=U4R&QiEm z;qR;PxElMy*luG#SeNmbMq1(s;A2_>c%*yqo!AXA9gZo@%FyUj)Oo#c>;Yw97H{;a z0*iyp*kiCRQ3gLzE=4sB5iO;1%X*4VQh7z&L}gTNk%#CHD);e#SU#0kwl{VZ&W)A8 zHz7KOK2tvr%>%3C$9Ei$X%nBwrpNOQJ)UnMe-vCJp)RR{Pq-hjSL3+P6x`KpY+LUicZUq$^uO&i;2U0!J>cy!J-fxLJ7VS^Ei}nV;&OB5Z$6} z9wwZfv%wy*hFrb`%1gLW@R)R*4pbUJwxf`QeGoh~a9+XBlx+}d>1U4zKnXh-thER@z#2kQhQv$?-@pKZ{*sl=W2W6Dfj=7Smo zGC2<=m=iW?mL4r&8PGsJw6GuHH4hQ3N5{mDjTr@fzZBk!t?H0D;$+^X;H;2K@D_wh z=fGZ=%&QP611h4FFm{>)Sxn8gMIoc7foBBllFEPrhb(4+$D)wfY1ktPza#Pvik|Yr zif4%^$uaSL5`1BrsfXEcUJ#zCSA?YL1uq~xPfWmfK@{ZmSL`JwU@FCQ#d*k2$kZFD zxVO-$+cbCnhFc5)MQV3fFckN?rv zb}WqXU%sEcZ{+%Z_VUd5{QbMxSAXvCa^%jO%U2+8PEuKq47sjHlcJ`3P-0+WmAlki z>Rm1u`Z4{2U>$f&TEds_a;Y`w`jcgCJ@^V;ci;TDt*kV%d%3ttDaaBhxhaa|WTdyE zSk!$+c_x{JwGzMqz~UDY8#U}=27EEAqf|9ckx9iYHA5OGB~wu`i$sCJQod9b5={7u z`WLhG4#n^>c#?GDSfyDNN}4!vY*K`165(IWs^kE-KaTBT68<2Kgi#zHD1*DBMxgW* zWQh7hd;r8*IPRg7lO8&-vjJgN!>TYD5znjwhv}paaDKVDK(O9h#+7o2-T8v`F2rV= zKz^G7!KP?wKN+OK>f)R)_+uck88}E{H}H7lIG-=`l(r>y)AUx5YKG%dUzw*>;BZ7# z5u1r>DMRKB-^sQ!K2Z*OYsj~Qx2~Qz543(VPbllpT&|Q0=c^zEBt`LYfz^(YW*8w*lA>3Unyb>m2#WIPJKiNLaO+n}X% z?Vvuc(m}Eg(iKD%v~mbtV>_8Al+jX$LBL7Z7TT_dlFek`U0vh=y|AZ1u$`jiSrF%? zE;1+SDO~RM!#&O28!(n<d$YJ|iCDUX^ko_B%Zy zPDCB?JMj|g<^p3=M?556(mnW0T9hBv&;Xnc#B-TDv03I~6&S3Ot~0hJF%Vkq{=G-o z5PL{e1E&M=EI;a0p|q`xCoKYQ4)Gka49*AhyiUq2cEshi?@)3f9zVkn$8+ z1qQ%15>~w)KzYHgRbT)Ya~DKC;>I5EF-;_~po`vpvDGXPd!=we?&9;^hi( z%##n62=K%n@#JI6<}t^URuV*K1Q-rH9}WE*pC5utRrDgVIPkf)d}N>DMXg2AEqJJ z0+lxrA?!I2cA|mEmSli8wmIkmfWWdy+UGtJP*m*k@@F{bZ_h{yXYQ5{zSHz!O6Ceu zT#X!jnvdJ?6pAQVA3SGjxB*^AE59?)(Zh>#~ zUhuNIg({-*`Z(hFV;o8ns*J2)xgsv9bwe&8@UwhhPNEHcnF2u?RW zIB;?Vzm1`5L}VGqp*g19;rySmZ=MV!=}f5hP!Oju1<9GDSh_^jN{z(LgpZdvpN z&=+ipwSwAIW;#;jCXAb@m)WI};v6c7L>!BTcO`I^g~KvHX`l={#yjvVhm?2k&cOG$ zid4Qv{U^w#9(twdI2ROH9=Ve zh%CYupRviJWJQ^P#Kg45KDfB0Gw3rh+c{0B1dVQwYYg=?bzX0qTpe$w7pq2h(=ZLJ zHgMjTfy|)iRZu=!56zE9;?uD4 z^K5Ig$=AGgvw(VL+m4t z^>Kt)a?|V;<(!gEd(a0|p$dJ(17BUO65QnBjg8uXnr?AX@D5vLCwSJE$L_Yxl|T*k zW&N&QRE+sL&G;v54*wA(b6}_A!%ps& z#BpOMZUyZY>v>!SDfdep)WwSaMF~!EpwbC+*rFF)@Of&=sE!alR%hYXMwliMpf}X?o%YVFPP4{zVLPa3}_?_c0P|287HC z5HTHbLvWKa)4S06f^|XQ`b*hnuk{ftV3*lb%+_;4?|=?G8mN&*0N$}i>h{@YZ}ky* z8nZ^n{J=MTvdymOBj)4_Ht7oB%MwgO`%17d+MmLSSOcpvi((g9iq>e@2M`69U^A@T zQTc)^+p%PdU4sPM0OJ9Q3PF~49eWQ7y$=*2@J$D%rE{Q0fD6@F;<;V~zVKTiJb=^! z?CGFzgFY)4?D%sxb-rXw0_Cf4M#-4LKn0#jZt=q=za z%=F+}1$Kd5VmnO}a6hMRH^jP#bL#dOFc*Y+Od-PlI~*A^u_Eb!V8_KBcf7IQl*0i#{3{>L^#yx`whB?c zoJ33NvN9IeWesz4#Jvxr%g>L>hS{3pyAyn!Q+T`B2B#F?UET&Rtc>Q+A}IF}d@CCV z`-2yXDM&+$izRA2|@Vt)Q z>czJTOn?hK0(tPuB);G~C6qSFnglzPRS(dZi@ShVS@j5wxsU=J;EQ_^&b8>#P9x{) z6U&nhWP(kNM?+q&P~$J=l6m=p(-9(gce8jEZ19EBwjgixs(~1YI7-{&t>k3l2!)-3 zOoCs*!1lO1@yxA6i_Jb{6l>Hd2b2_CZ_77X!yxA6i^@A<0 zr>W_bmRPebp7GrlU;TqEeiW?O1rbk04~6i}qp=5QTl`gQDE&S|4@jX0!1wEi9{AVx zI8atA&Fax;4sDZv{D)26(#}Ue*yLQgmUK!>>>oCHOFL_Su*v(-)O1Qq>~A)??cZ$j z?zBxlA$D9$a2UFzC7Gd9oBKRPvGXO#+~kw`$?T zv;r~B^PLvbevlJ98|-a)?BQm6n~K?Qw3NY*aWTvkZlX%s04peMasTZ%tIqw+=DzYb zoBP(^Z0-j}oq)KKj%4cROx!~`r~-QFBSCQr0EA6y^xXo%1p4BGf6=q_UH>7 z!0)SsGqlJKq5*P5fnPOvAR{HL6_MRU12Xs$2J|7SXicJszT^7`^%QxJ0{i#$V}%9IRAFery*qsU2r`XeV8gfRg^LE^{+(JVNN zgOK)17N-V@X8quqEDMl|WUy`+1Z#v-M6-5{%EsQwslr4UUE_(|`T!MtuQp^x`Akx} z9`0JgVu2XWOJnvQa#Zb^I@(y zGz*ZW_!}+sfHGmq3h@1Gq0`G=6X4Myzmf*ASq#~0iEY9=&~A#dx%f)T!Z|&BnH$$yrb%uzw_u!}k(shKnixOy7P zHA?CTzRVQldVpW^QV4sJl=~5Dy3`0Ze3=d>b6jJ|(kLTGH)Gjj*zm16dqaPnkygS< zIHa=th!TU?;CE^)!C3d^@6Yk~#=@CwmHpm*F_8Oc{ zsHZ*~zgCy%q2J*-b|;K`{Y*Xqm>S-tm36gzW!)?BIQWe+pscGMPu0De`+feC1$f@W z?)^7EX0Y?|9<4wv-YuXln0b@lIyv%V7Qk<>B}u?@!Q3wh_AI0cB(n;(k;qR?v9>Z{ z4`wHTj!ygUmK%3r_#%J_r{HD=6w#;oYXH5_7iJxQ}sO*@7Ikk6y@(u!Dm{pj1Kc?>{M||Md;{02=leADX_(l)cEb_@ZmOB@piW8%$ z3%9wGmH^$FoZ5}#E|$TG%6k+|q^)CBI}xLyWV8#)pf22DPFhAFVji@Fog17@=3A~` zHEAUQrPT6?G{TN91FtYOfKb=oMnIbebq0M|@EZF|EUv3No>Nz$%U4zOhLH?%S=asgFU4_M^G24p$7k{yj}l=yjTS-aSoOk zn+Elq2cvPJ8jZ`QQ{aipGjK&Dl~>~=T0rILn4%3-UcHM5zHr-$Pib)OX*&HHP4hcW z7d-}hN|&Th7U9`sM*6I9sLqg@>-qB!^PN{*nQ3-F#7(Gx6z#A(9kD1^Q9=8^K z8T;JQKbJ%rHRq%afMz4Pb}m1e1DN&!1xoDHJxHN-m&(1taxw7$Y}=iw06#65=ZNpv zohe(|JxJ~+t_Iwilc>d~Uw~VC^S$WZ*DI$^2|9P;KJZ(!iQEOhb=Xh={$%-S<=jgS z;CXgHS&+Z$A8THKWA3AMnUYzfhu`GBC8 zh0G@=h~&jEH)#aD&%|7%StCmFnV6g8E?P=VC}!1jF*lCwVH#r~5%yqZWS|c2wls=< zP$Y+VPZ~vsI7ZR&ZVhV-jiJL=V2#+$SOhEFWU#BY@!SA6U++!8Y*|kN9@+-u=@>~T zj^_*3{x9y{1TMzR3$^PGF` zx!bwto^!5qe$_{ew5s%clL^mdbhf3GW`m?T2-}0T0Vm+mlLkG%>W9*{6|kQ-`J13y z{v5M2Wy6qJP4cVmm$pSI(!1O6Za*B|L3#GPu)Rp36nIHf4)|Xy(por&bSuSOrEe30 z|ER*87U{{u_TVj0^&v2D3NUkxXM|-ZBkIX=iYQoB;m>J)k8ry?gy=ZT+YEUz5zQMM zprvS^U$xJGRe=ilS2@7K1L=PPym1u*?fcZuA+sxtvOlD0Ex11XbD|tHmO}gfInD2h z)((dGvaOJc!}b(uvk5=aErQ+xXQf5ZOSCoMKF>?yUTDLgf*vx3%=v>RyoBJl03*+FW6~4fHWh0>J33!>&>2_^X)U1j29zZ)Na@43 z=S4qx8n)N0^-kOco%c%TkeMczXuk{23u{U; zXxKQ%&}q3HlyIE`)83pS58ap; z19&Om9dw(Lpoh3swfT3-k2SrOg{O1=z)~By2HsS>Cm~Bf}iUw zI4uxoQ`#D}Ts!2WyO^9pkwc9fjv(M)XG%AuDKmb^GG)p)L$g$DSN_+3I>u@EC3A#R zIuqiXhF^+p`l9Hj;g{-)Z5n>b9N83S*?%9`G`zwR)pUdY>=I{e~SSy+I0w~LArIh9RtHUpqIDJqq%k>q!?La$Y6onNKi^!r6D@8Z^!$#n+ zQdSPnsDp@r&FPK&0JV)7@4w*_yU`-O?k|S)N|BaptP}4Mo?F(1AqiXB<#MiP@=W&> z>^^9Wq&9-_5ofUaPzOo{J(EEVJ(G-{z|KP|>XMXJx}-K-m((_%NTHhy1C?hm-4#&c(LnIzTCAj+UI)(idGt*@2Te zc3WA*UE_r{IIm&5Vb0g-YCTcRV@?&nV?KLEPh`&51uH#~8;9n1%x9vW=(1qDp}kk& zSi^S189BQZ*(3Jl5s-Vio~UmPJ(0SGo`_M-Qv=ciQum~vgr3OLq9^i9-=pmZ%}@{M zfllIoPgpPd)rLan(<6OTSTC+8ib&ss=R>%j2vDg#Qa5P_;NR2y9>Hq0{2dD{?pfg( zFal$6~CrTZ$$kNi5Q%g%*J`cao*k!C2+MhWF+E7@2*6@k_Sh%*>WkAzlX^)Hk znH?5cVdanwR@3L&U_k?JvB3)BZL>p+zph7rqu4gP#&i_h=3FZqtl(NUSOcxw=>O0L z3tqXr?M~F~L)(SEleb%VvedG@nqg_Twqm85y zMXQ|DYKPyO+hG}txNSTGJF1(T1`2JFXr%X4g_IzHDnN=0oCoUOKvQcY22nJCApuIa5(-$^g8 zWgGROHVmzTEU>s|g=YXp;;_)z1!5aa7%|V%247?wwYpXt6vs%*7`Uxg8~j9QgJJ6-r%KbN zrLIahq_@rJl(C2F+?b!rFA=qq1*!rQ`_R}ZZ>MUPs6UcEE#v(RHDa5}XTk;zrQYah zbZ+EPZLVr2B0P*^BZH1&hXTld=d{xyCb}~|m zgdd|^BM;*a8w>()xeV(jX?N|<7%c9R+_?U7=W)Nc3ho8xFnro2dFxCya z#Wu=3$MuY)Hj;@VzB8WAWx5O1wlQ-cVFai*NDY{o@ba@@M}KWA%M~LVoRQ^VcacHQ z!yD;Hcw^xYV79sW58-`@~aj2p4+KTiFd9P{HjAry}{SjD|nLX%ym}c z6><8giLny7)b;Ll6H~Y2v9ZczM;mD$$)oP8DooLMW&;qlI=>1!#a24a1UAW_eMm4d zuS_b?{-XV5zVVFWxY59lK@@L`wc{yR_ZDoJ$SYio+Jl*a+540K!s{EeyYkRX_SmGr8CGK;-xp{}OBDP_3tL*!ZjjDEn%9qckGp@DO;fNU<5k4mJMDOZ zc_K6#>~$V5wDV?BC)pELk`Em{VAZYzJ5p*UUj@C1SPLdog{R3o{{Gz~*A+YktkEb_ zH>4}mHwZNg&{m&io2)Lx)4J|G3svimO|5IXw5{VF_xl%glF$5_;}<>ifNs>n#rKM+ zb?+ZthRv{^&t*UhN)`Y(sI^rZ>X9vgpRye@k7eBu0ghYm`(aj>oMqzn1g%J>6lZuc zD=8A|6R_m(922PP-G8zuMfQm4BJ}vNldx{d_#7|9YK~V!ethv957A51(DtzTG9x0$ zfI?fuZ&A@oCzl>GbZc?tMZ93TiRd5w&(3Sj4x+)8-kPSt{g|KpSA5 zfsx!wvLL_z70H5}!lNRE7ivm?+H0WlOr%{bi@7gQnPig`1!-eVotE+*&y@z7JR=P@ z;nEknv*Hz%!~QA54LA=f3X4bzCa^~f$LDWf{D1Pa$$=(9IgR3l*3W) zfao89OICTX$QL|c%(7&W9FGT^$>DGfJd*?r92?U(FjtK$>qBrH){DI>cVO>w_y(T) zR7~E0LptDBj&$)%C1)pJ@cd)u7rd9_l1o$^4%c|uPe@Vd-!w|(Gobkp5l+qFaE&;x z0U{j#)5s-ijX#@N8xHxX38$2(2#$?*`5g3BaC|sNx}!NFi6kECH5-j5n+x7m2>96D z7r?~TJX{+Ss{8;lv4gl~5bgDR{=c%esG zL{eFZyi@Y@>n;!QS)wF35jFyR=J}Ue=U;B&a|DM2KO=F3Ws&|r?PPkXpCV{kkCL_u z{gKqtw26q%r_QM0^CvaUk7i+A;OOY=5lAwgoyxdgkIo|x_Q z%*KCAdtGf>x1qz^Bjzo_ip#YP8h zc=?hI`H9PnO&YwlVNvp&!fo(gE-3s&ZJ+IyVgE1kchG*}pm zV%e*L;FBxv+#1qT(^1)W%+*L z<$j37i#Qhrg-Uya{D^gGZ`R}Z_m?S3?B=g15mPJC>JKiIRo?JNZmX`(DY%07dkq^m>tep^YaLSu;GwsFx%K?274%dyI_Y-@0Md5Lyn z-a@B`8?XmUVM%F0;XUjE;xmBWx8&J>q0Aq7E*ZFaogc&Rk8`jKmwh%HP*fBIy5BVU zRpiMwifQKiG_>=kH0^vh8s{!=UEFAK@H{UAi=1q{kZp*~pfMy1GtRUVC>d@=GSy~s zUiIz>Lk$zp6Aaz6Xa|eT_pEj}Tuk?9{0W;zc4K5U25T|IcI4J#hKWk7ahvVMLIv9{ zi(SdGqrHSM#sUqbVch*d0Z zpQZhh>|)ClhzD4ZXbK}NWQCRRAF;w(`R`g`#q6_KVIk%z$#DO~3M=OPf5HkY=E=Wh zg=Ml>Vc|@)!opdGV1-38YfURG+!3v?x>&5RI1X-wMNjLp!W#cKR#?%Z6;?82Uu!F@ zsV}y|s-ZW}HER!qb}>_r(c@~y%s@q8Nqa^65{l=VmYC|0UOG#i!BTh4LOX#rZt>C4 zx$==poMWF*aYUgVh@cBIQsUJDthLmbo!gwFh-Y;^;Cb1|x}}Z_`0TE!<5Mqp~8_tJ{1TV!gJ(jEesv z)+^2t>lME|)@vKh`k1+V)Coel(tD0!Zb{OxHc9q4>~PrNKuZj3V}>(x4kHxa#FKPL zj_qOHmY=s9&wN-+!hQsrh&D>T1J7#gxast}sm_S5%5ZaA)6*G`=vqo=69jEP z&N2`MWh(CF*ga;EX}tB^f3Pxk0_KXBt~wYWKA1Ors?mF$0^SgiNW?kLJH?i9#VW?C zRlCSo*B)a-y1`GLR?4j%Jc9iEpcmRAtMp7S36jHOqCfmU0$19n?hMk94{y3E)z6<5 z+pP^x3rsiZA7{A1@=2MtGh<;?K*q8RZ(UpR2Ffsz4+CP6MrY?_DB&%ngl9r&)X;g_ zjY?Q`0Y_LgxT7RHlVa_GfUxLb4V8j;JqUI}dhJb&B+=PQ!GkbUIYUoYJjb*jAyUm= z%=*h%7~-wM$H=bIp*Z9tfAUEhosK!ae7s&&Re-}eBRX?9on)EJuNW{JL<<=gPJ;xL8Ay*67L}_t zDWf*)vH-(Gyt{Dj0xu*vXp-#(b!DIqzCDe&Z_nG@x5rfNWfT3q*~KTif9ipt5t40$x$RpBD!UfZ4sViNYI~R9fdS4F+FuRitTPGMr)F6bp7V%-BO^dXo9xu8pfmq-JywCn@KQ%Hed$cB=l9h zHYAv^a(WF|5w{`b8jo5rm7u7mB$!gJ@u(GV(#mN}p>6?cvb;F(GmaxEDyEQM`+o*# zic&#uAeXCjn?#F;EsnWrq?@#c?s6%a-ywZjwi$Yd?m$u$U59_@WNU%m!J+vbXnWM! z%nA5SvDYqvwJz(9^TZa~&A2~1;*9QO%)u#4ccTn^NN2^qCghtUaFCB~%;f}=z|EvJ z;P%zDu=INgru2y^_Qt*jd4?h7*Ie6jaBR3NU6YZWWJppaRU{eU(+Hcc;WFKCQ{utu4?$qF1UU!2}wp zHc)Jg5Iw`43X-9zhKvZ~t9>*}_={>N`GX0%D$9%@ww5jmV}iv#U>KiAXJuUVlly7f z%iC*UJ2$cY2F#{SX<3km`e2RTb-o*|N78(H3S?@mMn*vnl8b70G5? z_6ah}KL4WJvrxPFAIm*+r$Mi4fGkqbSd@Frc6a|=?s*e(&*nPio_8@5`8bhjf2Cc+ zwdEu~P-J|wETsKavnu9IF5|Q%87GOszXaE?c#B56)Ea^z$>5srlwO6M(syzju&TL; z9|Oy`QdqvJGp>^58%pc{jOE)07RxuBiI#64#CpN*Y$n4;us>VP=X;vbX zWOSX(Z0x;H8nz;)v_uAb((n(+j)1pBoCWEHvlrUXeNd+j9o`mKUOzC~(8+QgS>C|@ zbg1(NTG^v*x7g1Owb;*vTVxh)lSUpDHEh!2gy$ChYm!jM5rc?9cA1v`4r~sg8=9pK zcJhk|fMs?N{ARJkjiOgUP=x-7_Hl3%Z64Kg*oMBW?v!>1Hle~U4Q>@Is+I0#ahHZW zN9$vkhE5~9Xxi)iu}i}p)O>%~goeQ;G-#R$J)J4ZPWw49VN}N2IHR?3?Mhs`Vy_;o zlE?JRP{T(F=c!1GuaY+qrS`EABg`{w?Bx&(sXa3^9a8(KnDM%qTyh`6CHGXmQXUwse>wQ4R?Sl*C~Sbk^s|=Ba7@J8w`SU->R`&69>t zl`Vodk7FyWfWHfCKw$-3cfCYgAM41u*gZ9m`Kb#E)?#g4$}mRQm@zq)GiF?nC*f<} zPJfTldd=I9uciA{Qp~wF?~sgAGzQXY6#ZhHFmQ`fX|dgE z`02>XWf)^pZba6nw_H2m78A3l?V8qYH{?0J2|mg6Wv(U9@Cnh$ zfjhDj$J@R}-byz};a3Z60!CRjUs8?AbAj7 zJ7w8r+3AlMk7fg=7;;@$V_L7vM)ZiJ#ndZ-8!j~ZXL-Yyw31tIO0@&Xa~CP}D}aTx zPLn02e_`+HDOjhPsmD5wNK=lT0b|6-+=xUPgUxr7?T^Z!%Nd?fF@dGSTkg0PS8=+q zcu87v;uiWQS74NrRglzkKXFY=2DBpRALAt%ETtl$0#XT$!S^Az%+iIKv1W`<+Cty7 z$B9qVlLSt@4atemR1{^1K;_GFj-vgpD}bav=V&E%OV7hVt6OHqKr|+Y=0Jvy+L*w` zNz!j5>F5jZjmo5_^yM5;Bvg>c_m&XI6Z>ntdx-AkK(dcI3}0_WI!maCIiq_*=NQ35 zG3Q`WrwN8qqPsEQr89FXsfIbAs{p+4#R_A#l%5ZKg1tdlQiS$*^d&ZlQB>DwnY2$y zzRE_QH^Meno`_ZmX=#%al=D+mNhCGc@}8OB5zZhJE2vM*GLa03Ih>`YU_ZEkDN1k? z^2CT1qTHr}q_>319k@Ej_7Pz+DP7uJkZ9EjcGl|up%=hcv3 zwZstS2J3v}1}nPF=g8D(SyIqHyA%;$4FXnk#8&|$MSB&nVF%iJ(sD$5jUqT-J<*~4 z4R{b$!(^*xiS}W>iLx-?IB-zL5O)DwL=NQnRchQR!Bv?QvsTDbCfEF``D$aq8o9CP zGk%1*A=;|D_Q4iuX(M2h&H;P@$^m<^O$?I??;(OiJ>wT^6k~>lA$negE#T~RA3rae zDbbu*64r#ckyskmgnN3_ii#@S7k+tUQ$&wJWK$2wVR#e2Tj>|FqH?&u5x)iKR|ZoA z)|-e@Q!@f9QVDaTx)HUHB9=zMS`;(cdb*EM$3P{1%i&qk9e%_g8;y5+>bmn7sVV%o z%vgljsZ|y^c_93o60sY*s2>ba1Xy&0kTD#RGE)z8d%P*4{ID~}?Ieq8ia25lQI7CN zA3<`2f$WgecGi+3c8YRDH!epQv@_6$!O8;k+KEzvS+^Gzp|O$@2181S4IKk1VIb`I zlDL!*%%y};T)($IX1Fd%lo7_qU?w2y;XgE9BHxll#wl#rL@pek^wrV#B7V#uzbwb_oq1`_~$G*XsnS+93siCCwNp=-CIZJ9jSEWzoE~r*dZ!LJwHeN`PAp|JIK1^bF~hpO zgh{Y&uNTwR5IpD`Eu%D*6?EO2$%l*h`5#~_ar|M% zZ{dNQt&>aHI=m;XGoKD@1#B>i@ZLJUW-n!X0l$~S*IjOV0n5a4+W|O!{!DquSsCCs zZVN3^c96r@#qVl54uaoDp9?3H%TZLPRe*~jI4RCienbg<%=L@vLyQWNi~!=KPwaSIDx~}Wv3#S zOn{w8e_pQDb|?aT-FSx+Ue0T|qzT9STs}*hQ2HVb!@*}x?{n-=hI2UZliQz!K%2_a zor|jsER|wdp*lgEAV~~OcggZj3Q=P&rt`?+nqac=&+^tEiGMs+l2Q`?7yL9MQXSWTG-=ux@Bsu5hIrl&OsAUuIV!ou~vjm)?yG6W4y|rhmUkj{qo>>@m*zPZv4qm@EM}o@Bc8p30)DegPOb^|PYc2gpf zBXD28;HJX>JR^U8C(^VLT=?4Rw=JGMxF_0g0;8Y@sbLd8uzI$|?G0Zv4%0Z4iN0Y4 zZ!c2bUv{PQXU16v0S@r%(4W(|^LKE(ZBIn|0z8dBU76I!3wWitMZLc*zBXvrJ1pa} zl+n1^m99go$7MSL*VsvDYE`R!+O3saCv8hN z-)KG7ijC(Hl}cfJ3wl2{Z9vYY)ofUE>pUWt+H6VEvjMdwhS@)e4wQ({g>^~7CmE!w#Z_Kw(M{0(1;qr z4sFrDWQVq>wjEl;*JOvLJ3(`^ba`68jC3lig=j@3+I1>FXouf}O&cQfqJSLB)pFya zIg*7kzliWfCEBuyWUIG@o~vQ2cT#x1q~3hDQhE5I`6Zj=#%RPUC^1DWlm#~`8FZ*q zLD6yO=jIe0!gkQXlbme_>*p^z-hpegmKA&C$cw!e%8e_EZ1^4SIm>P1EUR9%Y~upV zc5qgjsm|tmR#Z`u8?JGzt6{t$9m}x>R$V!7Z`em=9j#XZ?q5usR zVovCf$R##1cF_x?mfX*#f-x^>9_3wwab|Nq%aCh?Q~*z;Bx9m+WIJ7Wu5oec=#DzX zJIu+#{;JUi?EYi;Fj~$1B}!<#HGGI?olD@eULrTj7Q04|!FboX1XzuMi(PeNvP_eX zOqBaJgAafQv;$6UO<2SAD8^ZAdzpHy+L7s@FUzlb2+h&bQm1iQ3g>lGp-FnA%&+>X zRHjW)H_#?}#@ixhhdRIND?Re$iQIbXa{${Eu%#H|O+~WD40oI2X;a68xNkQV`0kkU ztA5w(4YIrxwPRkQCr0kc`XgDmdyVB+l`8>FT0I5fn&|DkZn6QmN{eI<2^VqC+wmao z+f6nQEkqOH8p|B>^olZhgYTvGkP`lo=Gv`h5jTwil^({$RbIwT569@v$2<&y z7I$7&mD9Y`ht6C=xJl7;${Z^_%G|0Ti>(k(8^1C5yg5n_VjG{d-sZgtf$q zt31_(bV2DZ#SaZ^mO3dfQ3@H;SMAbHu4~L?OwoIk5$3YT})g_3M=s4YINTM+oNwj_~Ni;<_g(OkxWoQLreroo4c)LPN z7G0EIwTF@HnJ3$$mT9|`qSm35uwJn%`s0PNmz$Po_jD~Npopw<3igyT!&pye+p(}& zzS@oV5=*hhu_HGIkY0f7)fPS09(_T&?KdKnk}G@U~(Ec32#DpQm>5furg z<6tk`Wmn0dPtFJE=M?Q>1w~g%?Xw5(ax6!b*hOCpu?iN_zT=?i4F5gHf8jNXH>Z_o zKUwDMT~KgEj}#d*Ol~I9wa-i!Gnj=WXf~P za`owD!#pNy(${79rm50)W$Yv^Xc|_z;k_hJWl7%JrO*%_)0R>+BD|5hl@05e=$X#6 zOZmN0y0-)Oj3L>nxQOuL^tY- z2u7cLpk12eov0GisVLU&OHEP$n~;`Fysy`m@|7tW-cp0-uC(J~T@m{!sx!sVr?|q_ z5&J`Sm%2mYlPV-O9ff-JX~?vLD~hmK?%R3f4!tZlAHCwz(lcupw^8qCb;| zH6`1XkvGHIl1&HR2q|Yn#?R^@b2L(>fIQp^a7AV~`)A=umo~!;E8>lRF1(ys;~n?{xL`9*D|tMb z!`Cg3b6W9EJm{4IuHAFyUuL*uqE)1q=09?&^}M8dUPXLj5q8Wj(vG4HA-uhP<9Aq2 zlOjDNal9vy;{2|cJcs*>ie9A9Sur|DK99hWjrSbw8^6cesdvW#58kuG)~&lMYx2`Z zk93d+{vPS^%pZEcq(S2?^gdvG@jj#TGv@b2Sj-80+3V8VDtAVDKsUg-(KY62uj37R zOZYT{gAa3{8Mb`*+*Zomk^iQYrV4r~@J(tiZwuhR zc|!aH{n!3C)4$ZY@l*CPuYWIb{^NXcP5#ro^d(;ZnsGih1zgh?)IVp`zwToFdr8%h zHADS#V(kOgXDHh#QkCsWG7e|4_RBCnWA6kmgL(JyYo7@DvMiTD7M8%iZYBPu+HP3E z;+|u$7RrP4X`kF!8wi=gN83SmB zyX+?Nr3?7t4!*Qy4PFlA=~(xF)%tEZ+OvVzVfeuQ4P1!`cap$&Bq=x77LWfI#% zMT7(Q2W55ZbJlxa-*w2NF{2T;+`hcHZnIiNXPTz!% zST%;hhSf)-##&A!^XKa%O5R^9YpnlK`uJ#+Y@}s;a$ri}^Oste53RpZpYDV83xD)4 zN{k0Mh);paCW*|G_h)=OpthA~ZchTAZ1w1{z$+Q}Bn3b1G;WNIw__s{?6BO7^i|E6 z0{vANq}zpXRhr+&Gkk0S?}PD#!*X~D(*a(`;p??4fwf{i0aW+}Z<)%Dj~iWB-Or`e zuX0>M`ReG}?b&|SAL-fwUq2Soqk`bOB3{U~uOJZc=g2YovE=iF|3zAMv(o8xs zSu^>_EX?%Cf0g!Pea-Nii6rBhYHp$R(7-yY%c2cTe`Gp?-o9aS-;}Z<6GTI4R>z}pGv{Lb^3G)qR#yv?&G215>{BOH1{(m43(-@T&#j8m(WEirTxh$;9 zTH8c_OzW&|LM4YXJT~__)uFl94U7(237UnPFi~E16}kj0*khFWx69U-k&fZttr1yP zbI59`{{N*jJv&i})t+^<67-ob_^;B{k!N%pH9m@x*eUxCf7EMl^O;K(8{K-oy8F#P z`i~h_)V$-7d!M|O@pXRYUy4mnMz8Al=-1uH-yG&Rci^td&g*{+c|5Vwxnr9p1<_}F z8h-rt!MeA$H96pP=yyX*PDR$OHur8O+`slt=qAa#<9>ehVZ!BaPa7Kr+eNMEv*M+R z<%(-f)4mzLBV^RT8J&;L>a}`FFRzzBb3)Yjq@>V4B7Q$~zSn%)bA3F|Cl5K_Vo0}D z18#kJb3-4m!l9Wx{~R;7zTL;4g|iQL7kKTQ9ME-4%0T7M0Y>i(yNkp6UU`2|mo-i2 z>Lu?V+qZ5<(bIiD_h~(=yRTcDR_h&<(+>>U5nd&^*8AM$qiGj=&gro5@aSuE=3jng z-0tt@xp+UyzB{1a)v0e^t!OZN)zr#0pH7I3+`Xtthur>;j&4gZ8V(1H*gE#$#QF2X zhqwAYw)OFL^&dKvpS}3yd0o0smrcDp`~8`$#+DK~W$V`muO5{?_^)>z{XT2hIA(Ij zox$<`8@2^~ROr>qSiC5`=y90cE2kgKXd0E)-Err@5r2&t`cVJsj4RVegtcF@cK9U=D&71t6tx%M_So$ zYwqv7{PA~3_I#PCba%sc27Vv?I3Tvsj;ylsfKDmXwPPE89$Zopw|u~j>1`Ga8TNYp z&O5j7xzlWr-<;tkJ{z*X9rAgDk2*Ddl>AGgQ{VoPZ=GD$?ESFV%~$ewun+G?9XmCr z_nz6l_d+I3o78mVpCcL;Pr4DGynlo9vSxE{+Wh%MIc8=5#g1e59k~7a-uF+cqU|rw z|M=4EzV7*9dreD5ocVO_YB%Sb`~9;{q%J-`_ulgD^{2J!DjPO3WzfM_7kjVX7L_^m z)dh}{-!8Y$p8r{|$|1e)H_41STsgR#Z~pA3axfh)i5t=TxDMVa^d?RQ2`E^id&QvTsP`vThf zt#t62v}5?|+ZxH=>oe$^F&%cEGUd-Kd-B&n|AgQMVTDz_!!~K}_WtzDr5@{cIKC=t z*lqWTt8)9wm+o|2{ioOI@D9l#;R%uI-9LQuAn@zhKa&0mz1M$l8`Bk2#_-~;`9Ju6 zxG_DgetPV->+XxcpMCtxrP0A@fxoPAJi7d5f4`S(E1I1f+i%smhLev?dHLbR32ziM zh;xtb>Yn|1=+m9jCN67&OWsKtyGzz&`@8xNE-&w}yov7v-G%8Zjwd#oQB*J2OY1Xi zf82xVF@MZx`1PdLRZCuv{iyNesnXr)Pa~XEA2@#N-roE4TPgA1_PN~mH{XTPhH<_J zS3EBG{-G9A>g|14+Sv9+zsfIqK3yIi^xEUA$M!c4svI{)wtS)e`7GtbIVbDo z2E^+A>|XZHso=h^tzS6gyL;8y$`|Y8D^9Gq~ffv9BD8GkmJqbFX~Dmn)8pT;Y1G@bieNF@s(o zHQ?0L-_KwEB+k2ek6+^#G=6Qfe`C7`67N0Rz6oo1?q0hiiqwStyQbFvc+ESP?;o1` z#&@PG66c@il{~C0^ma?W8`k7$1ec9$y8|>qIJX*c!%BPRsEHFKZ`Xgz>#-+aXY)1V#?SB15Uwrg&#cwSSPTPI+ zhaL*s%Nd&a;p>c@e~msqsj#Y#W8vFv(k?uWRQc^*|L3-;`+YmSW&7y#+)vsZyK($} zmp}UVfBe9}e8;k`Wrj0%9_1JQzR(!kue`iq zQ~K6xy*fYC*PG-!Dq3;4+05|+3No+U+M1Mh+b)0a-qeZm@wr30OQ$;w`1Re2smCkh zhWh+vT2gqr;S%q5IjtUy+|~8ScX3YFKX-rfo!sqZhf^K9bbS3*sWkZNg(JIV^|y`v zGu9zL_xPII7Z=a{p!|=E?`X<%r}XVNBkJ{(rN-Nn3PUf-PrMd?y7@~#tlXGrQ#tMa@|#PJRZF6i8oV=QLA#isPFr6Z z@mf>Gq_A95NW*jYe=FSa>%N!D^*@Zf796=aclp`GdrxFO6Meg-B+cyQ*RHZn+)|^l zDAzqPwL%2Ja_h;((MyM z+HHAs-DdExIW0c%o4&I+IF{{j^zE;AZGIqf-m4PR5B-sein|9a`G z3#VKjSM4qDGq1a{&lerybN9a9b&JN6_1u zIVkU+$qNJIv_Xb>NImei)*fJ7fFW4auX^x(qmx(J3io`u;p8#rY+&RdaGAXYw;=COj_q z^r`lZ7MUaV1U~wH|Gw4r%KzFo?af^o>*Kb6mhr{GV*@`8woknBr<=_uCx>_L`&qKn z(|3kBx%DmTJ3Da5o%+9ya@(ES<8<7DtVPGyo*Xl%_hAQ_{>q)ptl`x6TF&+Se($l^ zNmpVn?Dkx>eS7xG?m@Q)91NJh3qNmYHKM+A_xD@(o6xq?$qoaqF5UHQtH*7ZcUhJ` ztF3*bA?@EdY`^`^C(DXPEet%dWZmcMWcnPJ2MaC_@=vl$2rpj~d2GkZrSEiW)$Gqk zj^kP?*QVvn{9(z_&M~X*7Hlr{J`fX@dFIk57sv1Ha->~{@kwr%FMWOaw^GHL6)nDh zclf94-aK&TvF%4!I?ZaGvFW>gJI0jH>waO%>m6OcR(PkrYiPPz7y5z!%v1Xp9Xjhf zCpE5MSwy!Makp(U_S_lO^@?iYiQ(M``07Xfs?D`k<^|3u9Uk#@O1G68_ZZ86U*zYq{{8b=kEZXske&VK=8_X* zi-*pz`_cB)-j>D6!1K4APcGkoY-E5T?!AJoEB3kkHAP;O@m{A7KI-%Cz||KXt-tnG z*J-^k{*v5F(X*(Py;E%5yBVR|uPpsOZ_2>E$thJs{hlhd|A{%EjsK||^6)J4tQAG~&Sdrqq_etoe0e)G+bj-JR~`{OrNzHWbbUb!-7&8v?J zPQJ75kyoegAAQ= z>q2KQY0~PS%U|1(_~X~gE6bgic#CAPA?xM9*X=MjUq zyfVNLR^ZZUTmFSykBN;ZzPBc2Vq&W7L-jSk&b{rH4c!v_TVP+8Fz1GjC1*Qks=A!q zJ}|_oYs#=|AB>!s-+bcR4`cc&`_K4vY=7rrM|1~PY<{zJn`y<%8~V>XvL@-QKJo2U z(yC}TIMij#nsZHqn|5FKTWj^K19t{Yj%mHquw{C&)2L3fe!8>Zt*p)q)^|Mn z?#)k|UCeuM=;a&lrM~GD;+ByxKl+Ogo%=Ph|6;IX(DM6zf)=K}pSjQJ&is^-0kO@_ z3_UkI*DdLg`@QwETVLtW_MHGzd)vp?8m}6edvoO1U!8cZ@!pSioqjspcg@)*4?iyo zXqH#;wx#(5j%rtJh+wf#Q34?=qm$%>M|gq z-Q%U(*F}Gn@6uz#<#_3Y_Y6NYcNpgG=(l-FY}|}TvC%`CE#9k++i=$wnDgCCz=*lz5?@+QAHj8C2HZ0CP%`>GyA+Wkw5HvBkh z=98_w5RRLYkS%p{9se$>v#H1 z^IG`DsRHM@w`TtMa|89LEibEmXD@749{%?EsX?7TUZMEKcGl82UANiqb*u_;doA;? z?*|9=?B3yeTlt}0e#1X))TV3O_s%SEZ@*)BhQhHQf@>jc0o3l3BY>Q$)N2n7y*E8Bgth!rNBr*^$%z z_jtR-5$=($uUXL@!|b~R)y8YSCj6@Up1nEU3g%;`+rn?shf6>=p6L|`2Hvwb*ZG3q zam?QPh5YCt^&G(I{(HQUafExab5|?6r!o7N#+d1>{ZDkO>w6Am;qKqWDOm?jH_pND za1i_#tJdRp&@17SpzZdZ$nR|MJE1l{^EKfOmwM_6Z~Wim%|zOf4C@S5bR*1q7x;11 z{UQ84>w9uPjW`u^0Nve)M=zP{iKw;L;F(^9XMK95igZs0zom6!71kYwNIj==e*Zn* zB&6L9cD{40=-$BWeRjSGzqY<-F0UtQPB$;x*0t%zGrgJ(VEuax73p3Mem8j$-V~|l zd`|b@<1LIM+{N~xR&<{S{#h@=@72=tErQ2-i?AN!5Y!**8dwdh{^-T^`FJT)60Ayr zwPL~Ez>hU(mxd+w1|l?H!ND4c9ETEzlzD~P!yZ8GfrHmWTmv4q-25Yz!dC{XTXLxDhkABF+qaGo}lBV`tRG_^C-~+oEb zD3nJO_~J+T|C{m<;OZ`qs9NRWN7w(LJW%3)TOO1yuRm7wnd?^+)h}unV%d%Kkg|~+ z9``Kmj@K?;Zxpa`pn4^xcj|cVO?Wsi2R4AiIk(#CTOIcnPkB96q7ITRjT-fIB{5Q8lDRYUFvKAu$$&RUGo&Ab}d9s((_t2koME&i^;c$)e ze2L1({6V{J0bf$?3yHi@JMw&HeOmAj70Mhi5jezr_55$iSHgcVU!G08hEqA9?-0t% zx*V$8i6`t698hL0#PZ^0S4Vqsz}kE>tj#;Ko_!XJ<7_?^|;? zDQiY>(uOa+*O|#1n#+v-QD2&&J+ww$cfr2{l^vB!?GKex*Op$Hgb($G(v>odTlTfb zEsQ^?3)qj5sd?HeCXbLaA^vsEam$v{pz#2Q8V5W{`M5O=^=umO!#ux6KPmJ>a;CEC z|0Z%c>;8|RY%m@XUMZ>?jYo2rLJ&Ud{*R}Q^p#9)8IR-$gz{{pAqFF-TE}-fai_2$k5^0S5ed?uYn1w1?N*69Iub@mxNm z44%j5h+6H~On+@YZvvk;fnLt{XZrh^e4ftXp3mp$R(yshwbeLNH=pZD*G%Uo&S$h0 z;sMHTH~74pr*GYUcx~YISA}+y{etJ*7oOvSPZx)ST`JmFO@DZ*gYe%%ADxXhu;#gN z3f2`cdeJ8=lf%_*|7z~%c1KIuZq$=Qtl2A6ujOoT*Iz;z0`O5y6+aqXQ{-Nr&HUWh~Qk2UpyE6!l^#WYx#)@^q@|29wPQNNSH_f+QC zUe4cxgbg~9q?p5ipM6Bu^FK|JNWTj06UXyD)IK}8c)R|0CABqK8=3TE=afl73WIkx2b8mc>> z$Ct)4q*HS|^R?vdh4qWd!>IC}M(Kd(Pk8?7sIPG{i9P6fLghi_rIJZpqj7NfI?4}x zdjkJoK9A%){@2Uz3Cd5PrzU@h<5b?(a%CF>KEWf(wGne0!n&l^P z^-xG$VQ=cny7XNn@R?Pb`vqKkl0QZr{uk@ej(T7`2*9{E+6;&GS$lkpHp5xXcgAt} zy2cF$0Vm!WM>6Ge;V6}AKM@{N1@h;*=h7iXOb-=pejh<~a|qgTlP5MV<2=SIBF4F2ZrI`bhAF+Q%Rl?@N3< zQHb-=n)5nyeT;&>Gl!3p9?S=H)zm{%t&s3~>jT|oGEc{c^SyhW{h0>-InZxnoO}!I z;4QTKm5^iKuzD|23f*e$ej2vQLZ61Q1^wl7KVrUw@&Et-e60Y#q%=;V(?wSKhu06t z*j$dLI*NMtw;#$^h*wirK2FM*;Gcl;&!V5^rN#U8mq@RVGMfb7kY~%Q8qR&Jj3tn6 zpL|F_XE2?s1^LiIkH(iJz_#0G9D>53Ump2fIHZTZu?VniybQS71t$QW>o24@_hs^S z7Nj^qdAjp57t53KXK2~99c$cw0OT#AM^5Lc2aT=Y)8jtoPApz2`U2>g687dw!1M7F ze6BfuVx9$f0}ihLL%q9;dUuz@*PQ1G^s)9Mb{GORtNJmp+T%@<{ znXN|i&)dh&z^0@4xG0wdbGmERPtJ!pxg-jEKLrz?sT>gVCq5bnhp(et5PK&WM-zg$fJTDjDju2?R}SIu&XN4dnKT;eg_{VDK|RiZr2 z?VG=+eV^i0z>nvlfu((cm=)gy%8T6$m{-ErQiiyyGI+sP8f=TJ3^v6N;osACqI^-+ zm3BB9I@xu0cChQja!Rh4qT0|?_>V?sW1-0sLzTzdNA%Qw@e;&NWn$FSF7b%)X^EQJ zQ`a652!hiNq=(iS>vUwTSf_(i8DSHnoQ~#EP7fiz=>hnPRw90ojU}>aiU?U9(=?MJ zlBTnG8~V4zGo3}zbO!nNl2_s%%EF zOLHp?EPSLTrl<*ec}qOe>Ie$Juei?o6XcK z*&bT=LJThI?--qFU7-U~fe^^bKo}k2O-}uKG;752P-1+? z!SjptxBy}kZr>32b6Vo`3F|jfT!+?<)91}Jh%M-)Y6Si}Fh|68lbH2Q z-w$c_3Lj4`e6OJw;e0`vnE8H}$_4AHs%RXbOI24nAuntqSfwgqdX$fz(_eS_lmPZF z4nF@zKf}<^2=%QdpUmZBV(rCs)fbmfNp<;1lsx^q%ja%&`S5gkJ+abDnClVg2#|Mi z?@!bx524;6|26nk6TZFcIh9JRZ`S$%f|4^tp0ge(z=3}Adf-f#W^4I4qGHJ$f6aL#wQ%BJvR=i$G*G{}{sf)cAIRSv{vJ{Ue%BguG0n1QJ;qgRM}Udolo55$ zeyIOMzV`ojQ}4hWFgJT9j)ED^{7?Cs*09;P zEXq9>Ixe9caE@^SGivce;~q9fM_r?%8_I3>SOr@%PQfC_qi(YMHnte|Zg%xyNsx)i zANMBcyD7ze`>rMT&1j$q&}nXp{8oYAH~`0JY=cxXT7ZMxEUd@2=4B^kTT%bE65Mm@ z-&PJM*1zhsYMxO(AGB)rD{B!lJxyHS@lOgM@*5_+c+ST^wM0nY zdj7~$2R?&@S^8JCh_7b4BApTw(P`=I`IEu|tYw_ZNW* znPvBvzrz!;$aosjEQ+o7d0D%UW`|R4uonzs=ote|JX)R!%b{eEMfG;AOY_%x|Bo#X zc>St5u2IWHnXw|hTH?sZwZ4M9Nav6rgn65}-NHUEPR+jL^+2#ks;!?A%G1P(s2!># zl7Z+~2BQ46%(0mRe@`y7A8F0@Zp-OVitPa~Sf_sGBj#t>e@A{^PT=`*;rSUL^?v0z z_~mU)H0H0Qr|wJgE6LThKa`)nism2wBYRY}q{TR;#3l~&H$M07z|-aSsiC#wVCW?_ zq+90meI@&hmwVmw{hYDT`-u1=IUMHo&)`Ga=qKWVGjhC=t--k4f%1-WErLDlr*-Aa zL)rWo^M2)~-j=UFGCJq{AwCGakZACX^U#I`dEnsbTdxaItw0SWI4W}vi+Lm9Fq3T` zY|rHWV%i)dc+KtOIGh9X;c(XV7{dmA0O!M6_4X6t6o3n$>-U%(T4Stj@Y;woTDK6^ zPrK7O=&!ww6M!`fh1D949OFJsPWv;*&O&`x^KvrPuw$WC0v$TW8KM6N4Eq1)ezH+d zz|TjVpLW&!ME<=wFU|bKGm)R|z|VHbSIu~}G@27X0r$U;pPC@CylDI_l)?sC#^n92 znT9d@U*Mk!bKt3(05jdXrq5A3Nr5wchX>Olr@0R*$<|nmJv|E)83jjaKF5$nu9fv~B z?92Hi^tYJ^TJ;_BrM2dTCR40;O7IEC^FG9<#xmtZ&L^3;zGK}l3jK}*Yfb>dJjjLx zf=@i2bd7xrs_h+=mVOMpM8DIHIkaEN-^J_-{QW|TaX7WMJp9Fr+X>O>?xf7;bOJB% z@c!FUo@P3!{LpWLP9^A6a=5zsQH=FO6X^mY8SBmIP=gLN zrvul36X+1&n1$} zl-o6Fq_`Jqwigl;x2#tJKgoO9s7J2u%G;b?*jE9s3(>2rrkD4OkVmTNg>41kJZi|_ z)pCmi%3z6^UJi$AUJfD+^ru0ePB0u!#3z+C{Tb;}&p{s{UFc&t3}_|&D$(1diH{xK z)1$0_U-}$+`8tz`pZam$uY(SMj@?`iRNm%t5ah9%{tMOlqOsVVFOtWkbp1@eU;}5C zw@?n$-*pFl3jiO$Mmx_Cyj`+-BDSF5(PEd%5=>b#KWSD^?)~DVKSCZXw3}mr%)rFE z?&cL0%D;!Zk1x-^*^jVh{o*|z`e%1bzTiuU^lR!lh5nKIF`9iCMSFxGy2koUpvWiw zrwOtuFA*P=7wWaSyo7WWqJ06%%3NN}-F>*7pWf_0ShIc!?}_z^mzQ~+2;~C)3iCUH z0#3dzC;Bd8Er`m$8~HM#9n>ioSmUczzl3y^%!9*N(c9MDCxY%_%|_r`P5$y86aBW2 zg}-L~9_Y2$g%A|cOK_skB6N5}ZzNqKUp+Zv#UHbs4W~uKSBt+wx+>;Vo8BJoK75|1 zH~S0L}+{UqQen(Y2+26#8@0cRQGU6k(rA^d`|Y z=&jSH;bGra^S`X-9> z7Si?e<_Vl$5nnC-3h5f2rME!?pAF2n-;p@;_=9sfll!7V{>7A3E5UnP)(~S09nfsU z4+CS6+&m898t`U)9f!yKf#(l!4Lxw6U!!s81LPCqmi>NCr@;T^QmhvwSnb~;F+q>J zT|0kcFIghpXUVHDg*Is+=wbyO}UrQa4^M4p`V;lnB=+_$F`gjnO z*bbO&ff;X%9hlFi0*?%O2E2iHWc;9Z058tK{Z}3oXPL03B)^~)DGTw-jfq?oh z5=o=CnsNCm_k{PDl$?(kZ-Xz#);Hr3ME?ZOrNH$IT|=I1@aNe2j_^~)8*!i?n*qFi zB5CALzY4DAxPPk1j#8J)#=8+Wg&-s@6Emonbqxxj<(P5eZ|+bi}!)~BH14PGl7{5!V3Bm9=> z1aYu#um^ab#(LJ~D;3;fM3 z;}_)=^iG3kV2uyr8)SI5Ne46o{E*s*=hz;+!}_* z`-q&wy&mwy*J_9G9K?_#B|Jlptku)yA485b^Uopj=-Xf)bOXjDY9|NXyU5PUk0^|d zh&Sd-n8!Q7cT)g2gxVn@4`^_thfNf7uK3Wz$A?IG1|OdjgU1kkJVLxwgeKIxERXJF zJdE)Nc%U9NJhX9wbmiJO7>9AkCTPFHFv24;&YQ=Rl38Xw$1n-cz@sDv57@OZt|mOv zkPBA0JW ze>}3~(XTTeF2*A-rarXvUEtjY9(lln=A8vG^kLvZaM!0iYO0R{3C~GFzCMBkZhQs( z@yL}&pJP10YmxqpnEGHYg}p=grq{;|;4u?$vtsH)gVW^a+Y;Hwhc1uKlJG2P=Q>1C z@y+UwM^)S3@Ps`1&}CcJu9F50-&6+ z@-X%p0hfQ+4}ehtgm=O4F>@I2FWxu0kn#TFeWTi*P)$FOPy0iG_X;CV`b9sTNq+O$ z{!?2&6EXE2d;UQ_wY49?xCrknhmX02@&4ldE}Izd43UF+jMc*sKd=EG$f=NLwLLmH zOjk2$zYg$@k@?ZD%gN@L@pfwaZ2RGjlRJH*nJ3PKOX;hF`-|N1 zyR(M7o7$MugI%4`&Jp><;B%0P4Zla=3)*@e=F16v=gY_oWxE?~?V+~T((P)AQ9gWw?9RAD-=oQcX1U6Q_Zq;mEuaZ+ zwDWRGhc2Gy2Z6Vavj_EOiEn}lU%;640fhH$t}#7~caeoXh)oQpyuA1pMBp+T_8{hV zDIy@04=`(rbbAoPzQh`iZ6-NA*pC?Phw^FqN24AvH!%D^q5ZrJXJucGyg>628*@Ef zs*nD5=7Yh{+w}GW-hKqChnC@E-r;&M@Q!UiaDDDqu#3^{J&k&>9$(2WMzeXj7~U%I zE=A4#lXmfhcXHgAPszS&4aP!{;xI9&$WORhT{=8Yivm1kMID~1%km<4M_jI*;lTp{ zKOE_ISpwrF!KcLKCNVrH9`KilK~o`hTVDh}BraEGc!}XN#GsM5v%Uykj?0DpKk$bp z9O;h~gR&t#T3-a87MGjN@G`^a;5?N7w1(F3cgE$8W_a*7q(4Cn;$?BmD3rb{F4xWQ zE{4AX@E5HgPh9RqhW9Z1Hvs>*W#V{A>3idHr!u^k;co@}*^9v^$K}oxIVJZ*>EZH^ zzX0CL@KJh@F9r{Kn9uN0dhlEfULKsQ5dJ~%A@oNF-D^`^js>cP&zl7JQ-Gbj!1&XE zXViO+NEr1LLs^yUtPZh^%s=xrGs;8Ib9T<`B*HzTDmgy2AGVS z(!Ek7WF5R%x&?!CZ{l(Cs1z8T(AY8c%bb4=$r(Y6#Zh|rCj5O~G~JDZbMNAG zz4|y2i?5ch7xu`i@S4(n74`=oT@2spnYl+u)&RdO<#scV*ETIC^dO}{StwWwc{DOk z;T}zWg|LfwVMD->O>m$`1!pE@5uJN%;s)qLZs2tB4sr?k(V#Jx>A{x`TwxEwut-cs zfLyH6kJ+Avp^-km>Gl^lV7>hYbcR~kUTbpS*kS%%x*Y=Y%kSgAjr$AY&%}I(_wr+2 zMDy5us?)yx3UKW2fbo(9?XTr$;2&!bN4`Yx-I%lJ@H8$D=)Vm4^8S54eyQK&I&OwE0LLJ)>Hp?O=nwh}I4nCU$OX6zieva%^+fgZr4r^l8~V<7FlPn~lva>05U3n^BXU3bGUZr{d>W9mzTTP_ zW9LKdjPwYXHF{YI7vRmb7gNr-KIPQ>_LQQWM8^Z-&3Pja1MfoTSUc1I7t}xSE)T|w zYa)1C_mlEC(8_lW%6ARu{yLE~emu%~ zjaV`GN1WeqmX9Y$7g=w|Uj8rix004Ib{y3Q%A1Jv9dsX=cSQL?BI;r&XwtDRUO3ms zLvAwYGLiAfiYd<^^mlMn=q3`e_BWjR9OG;gStw5y^2zGMCpjlxCmgY0HN(cU9wSS_ zGl*nFgm`g-v2`edHfI41VeL`)5r{55W5#;9JUD9=`S5(%qJK?9IYy&AWA*lBAHo*p z9*uIl86VAm(VPM3ckFfXB7i-oOCt3mi#f=DO-#KI?;Hd=PK5tMz@wcfA}opYLz`HG z`dP#Atooq`@ZbYbqwGXK!PFA#>!}NV!{wB-no&+FG4Wd-&ye;S)KdlokaM1RPFGm^ z_p*trz}Lv%ya+%*p2c}hWs(tM1RrxjUA^o`a@ob zOFma;>-Dvs?or{!x-e>m`BeensBSL>9Qn1`H~?I>0T-#yc(D=in+Bkx^=Q{k_@5bo$iH+c@E|?JT)GEc<#WOH ze}HR4xH|%`(PJCv@i12R|H}JZy4@1*bCIym#XU*7(i42sJG?k!=LMWr?c6SoN%+kv zPHl6jsPJIt&JPBAagyS~7U6hKU>{1W@Q>%yO~-`KwAhvvRK*2F2(X4Nld=Ef%F$z= zCE9`COd2=1zmGGqlUCvpmDS5&a(DbUVg&$TRH>XKhbeOY3ml zZZ5Hy;bPb2!0k{jd_#I0&{pkOFK>4XaR=5x-{W{5=F43%dM}b1Y5#4PP7{DJ>raew z8NW^7r;A67{gCu{s6VYg4`BBLzoh#(-^rYBSq%MB`CZ%&`_^EcncG2Im%fAV&Fw(r zJkmnhsPAsk{V?o9JM2R}?317$1if!GaHL<~KFAj?$R`M>e;x`>u6~g@`|pzmJ6|$4n}#ZjZ*gbc^<#i89a8 z%Sm(|%b%$(T};1oP(Q_30R8rQ7kDr65Et|7<1zI@wL;^(1M>yIqOBJqv{Bn~9yW0t z^>ZBg9Jg%mAmAq>PqZ=8Cpf-0wKume(mam(nM`$zc0Z1GKaO@kj>>*XBV=@s`PaF`of}*@3RUCRq*wl4zxt(d9&u2|#i)m6n722Y=fI^}w`vmdSR2`L9@ z%G#I#nqaz+AoCft;N|@1ThamQ0cHC+mk;#7>Eao93qy-BP}(`-p|G=4AP8R@TzBu;Q;I?g)X zn7^6to6w(&3&(w*$_E;tdHO*6x@MA28#3o@sL|UPE>^FNoS43E3&K*L`@J#VqaOf2 zZJr(N2QH#3uGdsue&A=GUz_EEnW&>V!c}6-V{QX|Y5C(F(sdzQJyqmC4fAe8&*1{D z2z~^{*IF*PO_vMqhg?A8fS_lji&-A@7u?&CF48c`1p$V$mJ0&9TyQ_+g8RV_9>RLr z7%rzQXg`UyeLwPrOy@*@_aMNRFZ5xs`$35Gf5Yy-)H&{>FC!PAJ@1PuA!q zrazPXMS2YLxL@RL2EVxj^?V2LEfaV~ebCBG&+Ea*Bqe@hkn(EiV%KflN64sm!r#{NFS+r@Z)OTe!czC*uYF`p;}9AtdX7chVm z5}uJS?o9K|Z1`A$WGW;05sZua_wbDIQwaC}3HhVJkQcv-{E@amt4Blrz%wu>c`9Z7l}v6YZNin-o2x}V%nF4pLV#=nK=EAt^+JP6sMCBl~^ z_Ioy2d)E0SjT8Yp?G{kw?#KWMYhk+MX)h+iCZ#U5qjcYtEThLKCrN?Qe z3X+Q+?uWmN>4(Q7hI4Vgrv@0$T7c)nj4$d#q)$%s;2HFJ26~KRHKOJ@_?vqUKE;yU za-lpy4rG|UiWSO*zRnA3`~-Od{U&d-!zW*tD#EWdNmX4{-YVC!F@bFCesOs7Lh0Qi zFmanizahubn0X^|ri;f4?Kbv5(QD&^@q4IE(U&jUzL0OWbGcC-u19OWupRA7^3BVj z|5K1}9)^6gn|YulZbv*_&ueO5L%xy3Dfo<83HfHXseNBX`@YKYtlRf?Q~Q$q#QGJB z_MH)tnNkJmy8m#q+VG!G{SNJ} z)A4!8O*x?BR}fE7KI8h%Q@!xP4?7lW8;^_W#YOjk)8vxqIKLudx1q@|dHw7*=!_xQ z*7)+YxvwAT5#5#-j_;-Nen~m119DUc<7-Vv9YN1OFGmFpK277hreB%qqx5nn_Hdin zL(RG>RFSc?m`(85vs*3uLE+$eW>UAIDI^$ zo?($0y%-khM{J8c5&QcA_v3gx5KIx!9!z&-Bixg|OgX^+?&7*J@bAknksf%$U2F8R zA|K*k_fffH%BkszqUD4g6v}xQ%6S*yzck+0PH@jBe1I46oq^^ZOMwsTPdt)79-z%4 z>jk_T%XBp+2!6NDBi5PPDU;JR<8R1+MX=i>*yy-@0QhVE*sc5HN~&YM{!%$zJfl7P z&TCQ$hT&3~P7d@Nw>@0X%S~-g`94q28~WO10Y~6@0Jd3~BK?wNP8aV0pDN)Q`I9wl zwBHgu>Jz#Hfq8R8A4T-OCeOW%$^crPV&HF4k5eEwaT%a*%V)U0bb@|`JW^5L8R#3s zAvdjC?-5%g^)89c9M4?uQMoDAH&skQnYL2hgKi!~y+6qC*7d&GRPQ+k-WK)Fb7E7y zKR7_W4>#32<@@>o^}ZE2K8SkHH`V)Aw-CRCWNITlyXJot-lSsuuNsolHhalL$VhLf zA}ET-G|1&6J3mqFTjaaWsiGt&Fd4f$xGecKyN?Wy_OvF=Bh{pvc+R6+f1 zAo+;sT`QlR<+9f;&xu2s0QV}vLO#;=4%@}+NO$!B@OBIMUod>%`txvqcpd!mt&4pB zMU?JXzF9p0-F*{u_ZH}GH2iqZ;`FTfLOuLqCKDX?KaS`2_1LK2t)^%6k63$eqy_lR z3?Jbmwpoyy_JN+sZXL`bW?qRGC>O2cX=`nFWB(|vOAQ3qMsQF!--mf<2Eoz!L$Wqc zrKD{159`cY-Q&s7XDnNlH<<7b=JI-8>^P8%4+xOM*L$qK6X(M8@SQG(1Kp13zmJM}J*TJf@zfZ8g7OIj)Q=DgXC*Tncrwx( zNzcglGRoGtKub44m!smh0s7x;g2(=^g9sBm=|bgrc#n8k^EBzR?BWpWRplUwF0s|| z6WC1O%ka2Q*X+#oQ4siOu=nEkC_)(d)gT;V{Iq?m(_-X%tchSWS7@4EyX9 zv4G*M{BB|o1^5Nd>=G4>2XK=S7~aeWK!f0jQ0}3;+M2){e!F-B<3+C+KkqVS2l=;m zOKwkt3?CBlb)rxz^mO8`=G%4OuC1uA2%g^W5OSB&>d6zkLc2sL)FTqq9DmaDUO9Z= zTZ#J}lK<-BE+yQMr|t@M(06ZwI{B>j{d*}L>8gBnaYt91jFCkSclG%9JBo*OEqY-} zsO=SzEAMK1At;)QuTP0z66`qe%H)=Fs^<_;3p*2=r1p|#arMF=RQ}s~Lh0I=6{^6mqIyHCn#l9M8p_7Z zKmBoxWjDrc!}+eYk03qg4ljA6-y&Bvg>!r4s>2n#wdd-lj`huQ_2CNmAG|Tnvg$kl&(o;l&uZ3Nlb7 z#?1=IHvSEE*hGPsKpvXN>6`hA7H=@xeHCcR3;I=b{<0qQO7;o79|`=pZ~rNY@Qp-q zIm21=jY>Ujzlm=&fNwa!H|QLobb4>WH@2bwui}8ScyzvT6T@5c4T8N1p%tMDWl<2e zQN5Ba6UQ^Jr$qTiCEyysH@KV(UnCm1Jl1@pTjv{N!8eXFoHgHg+{8DYW;koU(Ez>? zLcMpRo*qYd8u;{>_(nR~A)Vt}^No=)@yLE&664TsUxHuB_c@-`Iu`5^P3x%h3C?mI zRl*u1*)3wN(2&Q=K$Aqb?(unSF915D`nAW5FHyRs^NwZEqyzLVfiCT)ac&9XrPDJ& zv~dp2fNPM^9+0n>q5R7j&U&0%LT$$F27iNqi?yRexda00*Gc^abT6yptF70STgqXyXD)iaZ#vp#mB`FWfW9(RD??)$nk~D*S%Powa)cm1 zheh~IIKCn!91h3l38f$$+9@fQwK|$mETv$J+zg2FG0h52L4Zsi6noAQ2MdFHHGx~T zmL74<6s<`o;hQywSC&q{55|94ln?#s2QQuYY~9$hAHVmv#~!}q{?@;nF4UB;P5SSh z|M8o9KA)E{sQfpXH97eFIf2kOX9i`nc92S1(2x366?*6eI{%(ojDM@dvc zKBxjbER!wau@_x~=e;_TXa;;U!&}K8nW75)ZU2DZ)4+QXU(cKD%rH|kt$Q2xtz@ru znJ%Xye}=OjZ-_2VgFbNHp(dw#a&5GKLAD-eY=t=A4|^^U>^3(-HjOomkJCo>4Kh9V zms2^82k~ZeJjv$7uY^!YaXG$2k8hT$Go_5#`h3TaKe#zS$8xW6MYVWuWp=93Mi3seG7I0&k;UQFhF~mSC(ReauXpJG%n- zk0SCnF5sBq8!ASD_s`@yNfZ+>Zz4GuCqa+D5&Z>LI(#ut0X*5gBx1bK`V4GqWCVtf zJ$KK5r*&Lf%Sy!fR1DZ+Kf*=e33fWcGd@^1LmV3aQY`tmmiA2Wa5KNx(jRj9IN%SD z4l(#s+Kf-6pBnhY_EQ7@Ic8i456OCcgwmW(c%YoD7c<7;gIDMCUZJ3m^#QT`*9Y7X zeE4SYN{<1DaToj`c7924;8z}-4ffWyEa1a(kp-XpFU*~$fbV*7u5SdcKVP%KQhMI) zdfTURde;1u`CpM;Oa;FvVmNF5mnya=y{o4u0WOoCL3gkLqn@Q*81K)c-(-Oo{FiUI zm?DCy*LmEwF$|A;;0fFC=moexVO<7$+!2T1T_vAU9!0WPhw_nrDwEzJo+}y)^@?>Q z#NCYFF_2lN(wWPc-#*CjxCi_az(?`IH;i!)0>5<(Klh3P&c`XXBHmt(hj;TeKVlS* z<1#$Olf=7RCd%v9#1A=crYOaD`6I|m)SolyJ@UOZ2G4H--k8TaZPZ^g=^1cXcji2_ zeDICSBcVJfcccy}KggdrJ9#w5&(Y}jS7Tn7yg0E@zyp0|U1p71r`G?KXNR!LE%X*j zGo(UybBXHp4{vv$43hLvV#XW~0}?r`+t>@ig&wYc!K z^TTG598Y%}#jRq~3)MndxAXk-ViumNE>kukt(E@U0=EDwPJI)lzAJDy%C4nuKO+2{ z;g=3mT1eH=k^}50RmbymN2zX3P`W}%Jz`fnp%ng{ES)r$Je+P~KSl}=Ja zYdolUcl6Ya+18`nj^78BNBRFed>d1Dzw#(S)fLt~i0}R#JsMQan40^Q`5ar1$FQXw zCjyIi4=Ry#d@n8$%8yqM)>=w=``Y*=D4(I<4b3WGo5{5jultIa`KsO{Kx;1t(MQ*y z|E-h6d7R_?F3x9uYlCFsSDGJ)RN`NtSB+mW_k`X@M!!WGK39Zq#e;9fgT7&vWhvjC z2j40H-zw72BWF0I+joBZav#2xNzdTCef2{{#DBoI3cAVh|(|6!wBn!X7blw)}tH;ta z>ib%Jp|32Va~zw<1$-hx8p2B~U*?FFkQY`W9{Zj^{p)su@IOH7c=!$xv2v4r zg6EA$lg@XLZ0k5Z>v3k2IZ@~E?!=Vned6mTdMyCE9)1S3-qC9%O*5A|E zwsax!L%?kYekie}Tt;#X_?m4g_}Y4%4-)^ut~pD&>~-+L*TE+aFnp8`J_tVeBaUY+ zmmT1ER2S6Nlvia$F1s0YecodeH*tJxxoo|`+i{NER)(`K59G1F<+1H$c+2wG_TqfH zz2@>@J&xp3Saey+mna9|QK#l|px=lh*kj==Fd1_Ks=uX{@}(1ehvGYJkZ)~2;(QSo ze0(8-+lRQ(cChWmxbl)1nf1$>6t`5DQkz+yIa`|To8yBf>_{-AG&YKR9jAwFky~m` zi|bo&6xpS59}}O z3g)KkZ@UAPMvRT3a^4b78%aSf=bKMi-UM_%+?zh>!j_ ztMCBv+KP+?~hLA9qktdU2-h`|7aHML*nHb-Mb! zl;UeqBIWPl(&n(zTlHr3S>+eVTgLsl>I=9Vq5PtK(YKR27acu|uf0ky<*-6)cn-4u9B+I{9OgC;i6YFpRM9yvRNP*1 zGPQl)nWZVk^IbbHJOHRhdF`N<*7M$t!vA0GOW$?AQik)P|GY#jhVrPxFxG| zke;I>y&E&Ck18IJ7WtQm|D@l=OT?>~LoN|Nt)OqzQY-yu7!Gqsv2yV%{FP=p+Mh=4 zt=-gLm(H2Mx~28#1)%f9n01k8e-!H=>#l+KSMv}pbs1mJ>Z;@-L zuS?juJT!Q^)uD=mXPU%4j&7{u9A&iXF=rjMsD+1q-qbfX1RgQrPvqfC%my+T<)r_#Iet6a@i10s}7oUu{H z_~Whh*LShbD>Yo&;#q6o)ZLNPld)E;myRhdo(s2xBHRHjIJiMG22OwWsbfXj{kWjoAp*7~3C_;PJ`LHF|z_)8m?D@#uo zWo+#)A09xtjI!krE}kr#wvKFBF56Q+e5drOYwxxQ4?1`yg{BQPxN7I@Dx^o$A~(4HcljM z|5UYMoDg-erqbRAPjBm~6FATHZcRVFHMBNqb8qw97EwOMpSJ;gXT5i=*xXw=X?=+@ zcmAA>B2Z8{EwE@K<|~0QIIvPw&W0AfSE;TnQmO)rjxT6&LW{njaxP>ovGsJL9Jnt0 zh8ou>u-i#pR$c|m-Byf8VemBn%44s=n)e&W(zNqj{p-T?yt=Y_i8z+73i{R3GoRa< z&*!%0!%hk%NpE#Hf~}6Q7Zy|Tj+XhZmUFPvDswd3U9F2ke3I-8SSaM;1lc3dhifOu z`j-0-aZU#*2iRQo)Wp?_nuB%skUp7guGZQ#tE)}j70{0xQr8+4B)}Jqn^#SnSVT5S z5qlJi^p#{7r;<_^-ZLjTg@ti^8~LOJ5zf^|I2-?Vnl9W;-o_3o%$64s-!M<*Th zr|W_|MLIvn^Yo~^#Unb>Z6kdbbX`}0kI{O&7(icrcVr#NA+&yMxekQ9o#J}bQ5({Z z`1Uc`B~69AFqQMMwoAIg9FO}2^r#q@pA?R<>tO!^DuP!c{zVU!qHhIP57D!XeCI>Dv ztYb9Eis)lp`otZCM}$85*ZUx}YmBWD>Yv9e#vjO|ysYoFsDDWCx;+T|9ssV0o-z5( zMGCqW7xaKN0{Ee8r}QWw$>kz(+`Gmz+2y;XLTP*Om_rr8(_36rj?=i`bAhA|gYv07 z+wv#gNwk6Wk|($wtm)zj=&fJWX(JVMJ_`D}A27VN9&{A+;m@s+#0UDiMjGHpFuawX z(g&?sDnQskZku${Qkg;_|UJvLwYdiJLe+5 zUvhqydNA=0;*jsapP_BibiG~^!&~aXL=)(^3Gj0X9_vaUK&StXX`Q1C_*T1x_yh8P zyWT$0S>u`5Rex%mm~@DB6Hbx}Csl!0A8QeyN?Y)B%`yB+%CZiJnjK!%jP=LnR`9!o zO(T?2@ZU?7C5@s5^UiH*{fjFKuva3~B9u*?$;~oz!z-{B3X4&`pFwu0b@w#SZ4#^I zK?bEg66u&9Ak|Psv${s?`*81C2by~c?shlPo$H5ZumfY z336pM`#@U9{1m(y`W2H6Ys6;thCVi{w?^eqcgn=y5`Cd|5SZ5XS!?!4F;-6_Qg832Q zl}PvKwn!>Ezrb853-h8h%#X-sUeK6dlqG75lEjtJXXYYIz*^9B#OH4o^OIVP@3k1) zuiRi0+PK1*NESEkc*lYKPgM(P5Bw>g7^*jlYQ0TH%QU)qIh#%lSUKV zI*fH(9&@?v_yD@V7qeau`pKp4^?| zw{4aGcnR)W-`$SioPe*RM3pzFC-APyyKwu88)}I|QGSOjudPvO7TYM0%bnWW%Id(D zsyKSs zBX+}}KbF@b?VZT}Nz*OlCe3abbVjhSA0@l#*+hpCy3_C&1X?5ciuOpsW)CkxV;sjO z0-(D9(<|PAE~wYh#h4*pdPHA^6Gj*=mTzhLF9~{Yg5`BIJw5C7u|mLQV;xP2lur_5 z&nE`(OFI?t*XZeL{mLp`8S*O9)NM3N+(KLrd+B4f6;hLn{5+Q@6`JxgL-*_7Ba|XXXBo`4p$8m^! zSv#M8x&__FV{U+RT6MjGlgFh|mg}S1exBjTV>;0x#@c|40DMOE(SvAoC}VsDeJ~Ed zi{bDLei~~xKsaOE#+-dP(p&<1c~y8P{nmg-eZ;~;Z)?Cu@YeYJ7var(8+n`o-h@x= zdX|J|)Fjm*~CZsyQGIr{rrB; z->RR>5ZI&jtKn(YA8CxEI!@Jm0qX71!Rf~8rHyvm1^9>ce*QMYS*0suuNl(R`?;ns zHtU(_oDOYWT{}ZKy8kQIP9|FKZ|L>4C9q5 zssR`4A5}&^2*9h-pbPMWz2FC*fgkJ!d?&rP;0Jp>DeOlz_PfTfh<>+3=qHu;aH;jM7CPy@$&qi6hW64HgyYzg}EPc9?ad{;m4$h0-Kx zI?i*h4es#Ph#l$;>KUa)UNPc~qX+hsfz+@9-S$jqm@d^;sMlaMyYm8O&}XnxTGSx7 z6hhZjR9O(1zELROQXb&<(~b9myaqWiV++h$l_Stl$?F9)RC6}Qy#YE>#`~Y&bT!m$XxBD_7AfA+%Gj!2QrQ z@!Kj<^;)&3!*Q=uIoshlMedMRBery^S9%Dv;C|nTO5A-U1#`jf4X<|0$8)Y{cd9Tvp=;jUmngr;tXd$m95P= zw~6{Qosg|>WgTAw?W;WZJoZ&C$DGxy=U7zeVc%t#hmyU^K>T_y=9G&tr<{lR=mE?z zXF)Hr67&6~5&6PoM>?+WJVX}r(6?a!0{b7t1wUA-^PhXbf5?84?L;m3&jIR-(7W9O z`KiEwL;vk7H|{m~3+A09KV-)6l_=ksA|O|A|4tR;|6?URZ-~kn60#FMSne^;cZziw zN6AiWmZiJ^yilGg;2ERwyDq|$Bylg|-RlOxeWA`584^shhB>5IYKW~`LsXLjr!NdC&Lc3n4T@#J$gQb*6ziY z{2g<4jN_8d*Nf3EiWr{zY`qKmC>OL*>LfS#!vz)EDE!avr;)m!HBy%w8mV^p>VP&X zAN+PyjZ2xtE0>tFQ5%^wN*AD8YK2DW0yInbWsTA()+lB7X&cfgEi~H3=$%?UukcLP z_W$`r4d^gdp7Y@u!G~{=dNtt8^{TZ;BExC*`V{)zQ=FdlxPK7iKG_kmoM2J!_x9;; zvGxbi_Me4xMsqsW^pP6rj~f3dvFwkM*a4n##3#XLK@MmrD$AFkpOqke?ynFwqV~%q zmqB}lKst~T>YE;n>3}EwQRV>h0_AZ~mlrVpi^s*3?4y8(9>Y>jfgU4f-ZGt@`^{U% z>OOPecNp`OnMiMxh5SbIl$oZo(R#@j$@^^IfpXQMTqKtdG*8(;Wzy;YHipBq#XN=N zXk(tD&3mlmCCqyaxCp)yM3XPZw^7HT5%?1PKytDMhq(&oC)ykj`aidbOB2bLW)c6n zbHwB}N-J2Y&tJ@PbHx4`{ua;=I8VGoek>11_>CbC6TH?hjQKCi!xnU3gz;qyr(@l3 zxF2wSkV`>7=f$#Nzh-#o!4_X--p>P__a|x`0!fIt!@eH1P@aK3Jk2Gl>d$CTr!O7b zR7u~P1(jn0(-zFp_wrm;S*Xc*tFM5c2En^|W>v*CgZTuZqb{GjMb_-j zI?`3xU0rlvPn^E5XJud?I3w1DAU!2vFORP&ze8zCWNt~eTP00i|2E$1<9J8HZ&7F7 zJ9);QAAub@t-SB2OoE%;LH z?+)Bs2_F#H6$IRPSI{GYUqBiJZtaX)SUQpvc60+b;?`tWc2xL(9DYsPDWu^-yM;K< zqmGubt|%_FcZlCZ)}r^w@jLoXqGv)$F*}J~#JBJJe;;_a^52$-SGrVb;YUlvnTlp= ztH+xp{{^g9h|W~9Z-f0?4`Ev~x^8sMqjgXB*+=xKQi7dA1=t~U7{8AyAMpPL*b`Jx zcUbv=_YoDINRP*`i#bjN7VjQaBI%S4g5>?_>UkQKEOuUd zWkf=}s}mAp?OmjUz12%^qy9IpS*kr7YZ=$9)MA%AINO6O$&si8X=e^#1)H82A2z;n9Ch{hLQDj4@AOywdar*59v+jC-c@YQayU zb~Gpg(&+%7*va{r#|`2)HvXQ0|b`tLW$SIV8As#&z^M$+d^WD8LYPV_$M zvU#={huNmRSNO+jwh)+U=3u6I+5AbEX^yNMADE5bOL(q1f#;elEAtdJF#8zKDt$$j z@Q0vVLcpudHgKL0>>bFW3%0~LgaLY|X3rHrpgmVvZ4ObJmS&rSCEXb7F1GEeXWOo7 z-}N-VWr-HHUdwLbx7+AlgX@?bb``kOWGbRd|BTDH-v9q#KN_KTja>eB>_-i|O~ZcF z+J4K*F7*3M|FQZv>H)$>zJ_(%RM6|g3>T~aGvH$EQbE_J;K%hDhR+|Gz`wEnLA8Di zoe$&_muWuc&A>XUMLx0cbS|;B&yde029qMmz}UfS1I{j_i#;^h7JxwX^jy|utS`d; z0n)d{@-MRYN5K98&ll_;n1FidLHsYVzraRvOd0XNNc(H@wt0Vn2ljx357s8F_?(UO z_nV;Knj)%)lD}fWrO>&6#=M$STud`pQPW-9PiORQCt6rp z5vXSUS26RXmBf=42P>Bcs&$TZUFB+nBdy*Xe!D3|v$bXQRdNV8uMx^i)t@%X)x9@h z&%=7`RaS>ViV@X+RCeRuxexPVZzyyC z>L*(qM$x2Qp*_J+=wE_DP2G?ZpCdXBbmyeRPY7OpqT|4^hYp4lm8(zOd30yhq3|U` z?ml{8)2=;Fh&&P6^8;@}Q;sj`d0iqp6+ZA#V)$3xp{t+4f2rf1?qZfsPOQc}U5NV~ zlGFyNE*E#<#9iQzx%Q(Hp5t~oLLxWes1%0OWw()Mo2ySGwWJT)@7U?S`h*+4!J6P3 zY>z8Jl~0>%DD8#!cZJ?TyqwauP*-m#^nsXg zs@3s+PtxSy9@WA}th)c*({`0fNP#g4nWMaQAXi?2S>r8KnePyXa6 zJeck2>JIJsgFu-}DLz9|ne+BL@=)fkg^ywGh5x_X6-reTnnY<^+XUc!2lh*}Iv&yU z+?g}fo=th$a~}!sy=z?Ou!F_zS*j>o9sX65x*GC|Mghu8r5mpeE)kp0E4%5pQNF>u z`8@W?3wqLhkiE+G8$I;AwbdjA-PZ7?@h&0>g^YJmZ)D5&(>r74PAr2i;t#)EKIKvx zyMya^{;@7*+{Qa-jbLw#vEAx>RQ@Rge;~dW8$bGe1?*cNyU2GR;%~jk_bc)JV)1$1 zkYy0rwwTi~^PhMsE!iXEKllyyvONR-qw#5L{<9biYBtj8t6$LgF>n!=>;808*`Hyo zeu46{q26S1Z-mEZh!#%IdK`JoWUs7`A6Dn5*&c(V6saGaj>vORjQ zBP;^<{h0qZ$a^mcvG)SxQ{=C44zvzm#5WccOyTwa>+ItCe(h!l7~~HhzQGo#gZrE$ z${7y*ZA5e&M=t>!z6I@q^(7eh<$mcA9Uy>4$v$%z&2d2gTE6D_3#Z2U-m(@pKV-XP z(tqV}y3l{wqxy8hl+Gn0yWH#HUx)oL!>89o=)op3eE)T4(t}MTy%N?6Zvh^pFN@s= zV63|);!K=G>~$f3dzyZ41@uZQ+(NXAf)T{;1eE0GP&mBei}_2fi)SG?JP{V5T@#Mh zbk$+g(d@4J?jm`+DC(Lm2S2?xDLv(0N0+1h6>jT1{9oX6OI@)yS=B}wpxzf%L zPPmdwauau6I3e0gy28mNE@^-Aw3%JX6VmR|fA#JueZX}^+n|!kq4uR+_O3+-r%%n? z@91*2SNyBeKJP~zofS{ybM@cvA`10oWx5 zZ0D+2*5nmqs=qAJFSqG~P7G{U2-TI=uL0dk(Z~tiCr>K$4xG zK=L{0fbftdtN8)wg3|^J<^=?oK=zRUck4xCItk(|dC;QpBtQ>|K>hMc&(sX+Z&?4@ zf_OZBS@gFp)Q7Nc)dc!YWjJ#`;?%icW?&6%F4jn9V2xxl)=2WOMv_B)3gsreO6eZ@ z1!T<^utmI5{L}hJGy2FHl(!V+Ed@Nz|FYQk4?NIEGSFAxUk+)ata|xxM!cIjUhKGO zjKf<@dgC^Rv(_7v9XqEFT=pAqm`}vowcs1znhjVThq3|THT!rY-D)(>JtA$v-PrTt zs=JYOkTzFJp|q_B{MSAk8Vz{a$=sB#OvTE~P-V7xU50m*xv-<`uO=;h(C4u(OV?{%WvSPK9*b~;KId>T6hGQ;(QuLPw2zL; zH&FcL5n7koj;`-H^DMdgTp}2i4|yJ*PtWMr`Q{2nJ@k;BFyk(Zvz)KBe0UbHj}t8L zQwM}|#iZ9WAROgOK5UU*WW7ZeU7&;C;QRq?99@-i3*r}w+`>AHzpfa(hepo;1?|m7 zA06bxFW~tMaMSo`qj(sr-3UTB^HV(9Ib0bM?vd`$m~>$u&UKLpP@9mjR%18pnjC|t zJ}bt}yB2gKkWZq7`}w?I!^Zv?C>`kF(m;QS(9ddhirkS7^Vy_?sV|5tXXip5q4;(Q z_n^xei1uFw4neXp1xEyqG3Tlp-~ZWP>OaW;=VIRMT;!qUkNdv)Qyp>n|2Og+w1B3oGLjSl2{p}W}dN$KPm!}UO;&(&EaL|vCto^6T@Si#y zYc>7&*X=3OmWhspe>3Qx?vej(C=*iCuDs8MJxMr+Fs}sP)$>c|{H*zKI>YqI&q&Ys zrSf+ToyNAjD1Qpl!~ToQ6OezB0YksQ&5iz_iu~PZixk3<)5i`;taFhc-{Aw`U8v*j zrtw0@I|K9nks@{KF~WZUJDP>CqbZ^>2>sr}<+PGFGc_H#jJ5)8X!NMEeEPC;pFZY=Tar!7L_Msi|B#27^H&((u(j8&O9egN-^S7!Gmmqt0A(aoG47sX2Cf?7) zWRB-WeIQWzKOZ^m1u-7-v}#MDcqF4DzdSu2y%9cq3#zyo4}C?G*n@Ki+i2c3&RCBn zoN@WE!IRSGJQ7JB1Kf~_wdl%27(fVyT%+3B088oO*Ony4C%?&YY__Bf1jI`cHz&V2 zM94Sm@q|sb^+e+A5vmxk>jRMv!N=?)BfSMG*ViUgisM1>BlNxW^1t_WE&r1L2g?6< z*YpIV&HlH3#b}&&<`s#cuhJO0(RY~p@NK=#>#7ynKyU03P({6|Vt){{ZQhuk+#?(xtM&t|?o@P2O#!N$ua3Je~K-tWg=iJu&G- z={gbgzXw*T-_-E4B}%vleXcXo5?wZ;wLQK)1JhdsR==VAUrd?KHg9XcFn?PjaNI-v zg8K^BlWmWL=l*nJOEhp{Sao3f|9krM`8=Qgg8nZi>gxpD|51Pe`ak-1l9;IBi2BCJ z;d{Psxs7Kk|80Y(FC$)vvQfLMefH0C3!v_)Zn-a;%ft6M9Ism1no_hu^IKpfFMQQH zFM{0WN9cP|{y#~XzJct4IIFC)ENJw1nmOf(P05^ zjLDY#=ieV&@%c?dr#Fi7;Vt`}g7(b{x(EL)!<}~7#Q)jng#Q99%m{kK4-YuS;e$?5 zb_bQ|k8$xBTRwzu!)i~CaAJJ<5b1m<;eA3;*B^~!AjbxLAL{scgy=?KuO41q!z0{m zCpT4Q{m%bUGB%|tSWT!L%Gw>>x`b_O$zPCJyECwTNuHH<$IybVQCEDv;`6_c{QJm% zf2`-<|KAK)cL}q{0;g9hR0D~#s|t^XPpd-nd*vS%Zd1L1p}~$;___#owv~!tFyt0( z3so=eQxCtS`t;M6#|18Jb#!-Y8Z#^ui{j}aMeW|DsNP(tK0*~plRedGJ4(MUk^Frw*q^3|8JPF~B?I9ia6czFh%@&Ab~3@yxpU}8xZ}%4{~zKCTa{~Rg5VY%t{xKKv(j|NY z-`Ex98t69;UjLYY{xeOF$MgPj=+Kr@-_ozIOZ!uQ6_7V;`pBCYzh8o^>E(I27c2qm zB>mdu>Yx+rs2=bLT^{*bmH+DD{|0__HROc93%!lN|0j!$ zGVeY0Oy+WJk?;(-SpK&~FF$y?jDYgH`p9uw`CYK3j{M97SPdDcCj4k@H z2UalaQsWv>OnqKfE#VpMzYeb=S47T^w!gPe`)lR*;%r70X?eAN@--^IpmXj~E@<No@IVJa?H{>9_Vg$Z|+L`Ub6NmC6 ze67k4`ZwvLxV>;zu^%+hLvtSZU3VGu?=tLl3CaPzU=ZtO_)c_Ph5)#DDnHUQ$`8uG z{5@X8lP(Wu6AM4W*Qorl^L~T=p_dRVJdx6`=36oKhKwm*H{=^q~ahV2?B9(XU*f?+M-Rx}?x^srPI^hqN7yx;Aw>N z*3R>X)sB~6NUvDdvG<1umAAq#zwoPyRpGtoGn-4B)up4_O5@a>=fAB|ye?Nb>0~y= zxgPKSDDae|eAt%szJNdf%P38}8P&1(ZxzZZ#_{DBb}cW|unWSzvhx#la7WUi!O&E| z)4sZ!e%AttB-apEV<#19@wsPwJq>o1& zLunVw@EGp1pK>blHjOaF=?2Ij?@||EaxX9pzx=md%O`1A1X>#8nXY{o&i@eg zH`mch-uGYbZf|}0GDoZ3)zbR1>i<>vU-Qv>2|jMxR}yTYAs2n^vi(HDcn8fEgRaI{4{n*wX~j6%SMD>eqLS;k0H%3pAySQ z63psL@PxW8??O8->?mz13AA8uMrpIXs-$Xape4Nbo`k^d{Qm^a{j|OX`C7lLQS&7D z(v|C7dG6YhLSOBfJYRiJUc(l6+1;j23rdBp@9@@k*Qu}-maEdM z)0EFUl?Mu{M^wp+2P>ZsE%BzeD-UFMDi2*#HN6_|;4ve!UHM>^@_9&n$E=AeqRLw?QBBmeZ*N9j0kh9IMSQ_Ydz5;0Izd%sE1y@*rkK^EG+gKQpw6=t{JUa>^vdv{)VB{;Zbp)9r<3QlCeZ2m#oh~I+G2(%;Nm* z;8oky8macCx`O&C^*>@8J>m8gr@3#!e!;1sejHjgPib~k9cPWG2`yDM3_AQ}ZL!k#nLhC}Q6FZksvk~vov@atWycXbN+ zJJCE}6E+sUu1{*duFLWmeyny=ujDRwD01ag!H#P6oPWK})TuAN11%I-zq`HFkQIwO0;I={po zv4fF4Vr?;d#5x0y9&?g+moBk7;UC^D)*0C?);Yj#G32Z77Hf;yE!O!D?-pz8w_B_& zvRe$PU39k?m0sH|*7gtW7HjLbTP%rpizW2g38oUQ^w|sMH1>UMkt2J-MjP);d%+5g zyLsorrxSw7b!uvg4Ia2X}ZQzUW8kzUcG(mm1s!zUasD4jR}r2Xf#y zOxr<&Q!uawb{YGkFRh#y@amrE&HHEm3JPrvie>W~1^m!wlV;D(%;y{Eh z_I^dW*xQdT_I_!)02Vr3?EQ!6VsBr%@cqvJE>T9{H*_)?^(OyE$1+tBeydpx{DJQH z-Jcoe4+Nv_HorCw(BOSbMc~hLe=m5UR_CQjZV~v*p$PofcO8Dveol4n&}q%I>kwlk z@vaD!LDq}$wUDQAGSJ*R82?61>V>rCtnDQ)?!03u8{9nWOwxrcoE|J_&u&e3U3c(J z*%s#zZ)JVFMV8%hy+UMx8s(y5+{M4ya<2+Ht?OP<7+ADwVqno0xp|xF#aU4$9g4c4 zVm5msDZ&35bpyUl4M9!`*~5--LdCV%3pF)lryWt@gut~8^7S}nA$L^!Cu(y0`)Wo( z52W{G%waNCU*hnt^CpR4=FSTjf;%sC29wus8<}$C#_pXL{%3C zctGIIJwcR&=02IhW&^fOY3XTmd@5*0gs&r$P9hC^G>xF@Xx;2hYOTsd52>RQA9h|i z=iPZ>uM5yl#Bx$v4cRG2wyH?0yUpN_p>FceWrX)%$O@dPe7EEyaDi;XHrKRtcdT!DuEpN%DjS5hsBW3jv9D!&;4jyvTy+Vsc&cSH zrMVjW(7L7Lr7bg#SF~*3h&fivGl9Qg9>wR5HM|+hKZn)cV*+>8EiOobL2cdlWBtO} z0olcaaBhHu&JBp36F@uucrRd2?UnUc);`O$CIgRL%&Ok*Xgwi=UJwH`NlUcl@s4F3vCwd>G1Q;-#3WLER3l=VZ>MdSTP?v3*V;aKo(L%DyUzq@P!a3 zWT%l9{U>|9*nqv`RllwNt`BL(5s#|k^Lk0jnGc0fMIewggK zxiwzMhN9lPp=Ukjt<)|X#kIE8gEse28U3Gc5Ua=E5^RvSxL!pIY!t7QG{XzgChvA{ z`^m!+oeZ%w@0yVv2UTEjSXzB)xb?6^FlX?Ee#;V8r(HX^0kcz@XM%R_fP^pV9|mdr5%@JO|*s33-K+S`RHSOi>gU{F4N`#T{%ICN2!TwZ%1y5sNszaA){omwWuJ`y3SKdrA|K<;LDPHUf zwF3P_H>}$1g%E<$!e1=m#P3h`03(kSssZK;@3tu^()!ARI2FhH7 z=jv?qlsB5h@^RP|4D5_m>osZ}XcruuxVU{bmVsxhg>TjfqGkkpO&d6SN}j*2D-W6r zv?IOE7^bgn`EY3{z>@R6pEj@eaPOY1&Y`Rsc5g=kG}QJx5v z&%k_oSfC95uMd>fOZ;vgeGQZ`g`{;T&1LW)aaGuHFd^(X>S$MvFAh8m4C32s!Wh-s zUGCu9j;?sO8tmA2VZ@+ds15!mQc7f%!oCap=eH~08=F#+-c8>#6O8Za-Slpkp&L8H85!Ym$DE$4Czy9OlPKVU+PQ#yu&Ux|Q{{m?Hpm6=&(Wtmn`3b92 zLj3Okntv9#*tWcM{Cv^({<8F@ZNFdeACr4)zdL&C?oXGE|H0X-5)`>w-T#9b9~Ph4 zFvOX@?ZTffJMxX$&>k=Q&r6F=9ep_ey2qv#zgE+b_ulNoPcI%*Q{}v}g zZdV&zwL$Xui{9{LKF%CO8w$<$FKZb}BT~EJpSSzQ`|ZVl`0d4i`0d4i`0d4i`0d4i z_S<_DN4zd#85Q#YlFzF`NU3MBptp@K-BS|o{~<5Ct}o+dH*mV2mtBhYzvpFlPsGbE z?sPA^xYNDt;!gLni#vGPb?IJq@uYj%-NRmXrHGeZNbw|n!=vu6z+;^|dbGKr$)~Pv zVYE1@kAJ@l+WBM5b8PVM=Pi_U|9)Qh_e0CFJZ<>*lLqkb7vImnUmQFmoN0|)Hq+$Y z4h4#Lv!}hnx(D%nDSO%jRLvOdB`#%8dwM*ERX9!r z7VjQZBI)>Ef-LdOYDsG^<=5BRH9eFuQq`&ja!qPoYTe=5|GT>Hk8h&N`o3vnowlhA zrD%Yn0SdNIpdeyFz#&u!FhB{lW($s?U`g7hDNsQL*=dqOsA*G$bx~0Ril|#F>bmN# z>qbDgVugyRtdF}CLAFAD9#+}a$MxBLzvoVprlq*P@4J7zZ~M8KJ9p09xj)Z6_ndRj zxz2sv1}3GEU+!ss(q9X9s`^|Z*rDojE%;4ztNB8U&|%ytLK=V^?ru?3v4w@(9-q8 z)3<+>`d8r7{zp^T*B^i9zvhAAY3##+Zw#F1VIQ(@?(@w(6B5OQWX7&iuW3(M`aT;} z+gN}2s-f+a#_I81DFI&?m?YFH9YfQT`mvAkVe@dLUG;``9r(r&Ycc!qYxcA#dQ#cP z19zmK_uy2>OCRy)E3f)>e}cF7Pqh2`r=z^3171r}MSY6OSc)nnm>)|^k(~^#oX0WW} zh>vn9aMQ3QiWs}m$Tls>^qB}3Y zA&k|#75JRIl^32js<*&4V{~*ZSF3Tn6SGl-cfOD-T$x=J0{tSo!!(lDFc0>WA(t9rcR8XLrF| zR_du(+9YG?2D$MX?Ts*pto8@Nkr03SMdem9)&S9A=G1~5e()xX;=w9l7aCjd3wuEgK!VHHn+Szw( zN8Pz|53bL^Hex$On%Uz^( z50c|Q%-jk6KmYk<%3alBK6$lF)T}sekTg4vRm0p8 z^(WSPk-QPWLoO@1rcLumFj$_6J=Cb^9*VC;%+sba>rD}_12@?wD$}u#*(<-w)2Bz% zl|1ZYu&w~B-Fr8lBY#HL(XdDAw~{$`%Ma2Q*C$}MfL{b&51#P-S=5H)s@UFa1@>lZ zX1lQ~yB*;td}rq3JF^Qmr~D6PV}ugA%eDqHK1dS2)BA*k!cAj^rEs?$sZFKf)p2skru%-|XNJpO829`b(ATD-P2| zFzZZ_R`{LrS;e&h847-XrLx?r=P>_>jxqVqG$LS8{*;trhBf)hygPA9BHL2QYQj>w zIh{B*d-9ubPgu}iPGgQ7#Wyfpc(g71zfkgPt}Au-$4xO z5B!s)AxwZgal!)z|J)pm991N_Qx0~lVbf;+Wu_d}?>{KPW*$R|)K6y;g8iQn`suP$ zgZ*Eq=%>rOBG`Y|VUlxj`O{s2ZO>IiW6vDYVoI~fKG{GM2iI!Gb}_6@xayKbmcG%! z68L>NPW)uHtNY$z$vOk}LyPV*cylBz2ZML4U5oh)H`ozEKEo^E`(>F7pDXO{T0!!q2Ezt#3iHi__lJ4#{xI+NCIj~3hJ%8% zXd`)K_2>hv@C@3U1P;R$>TceWC5^#B$3z#D?x3;)p{vSN@ ziH7+vNWuB9h2_rR{1-fqLe1h_O^8-NRyIP4FQ(nyo&fmY?am?At(*_WXXRYJY2uz0D*)ZSp zprgzADSd4&FH`7)__8<5KmHM(^BU80)P=VLuh;QN$&fE#csg&ZPPAwdJ=5xsyX{)J zdrpJ5(^S=#5!06e#R`n zxuNJ_|Atv@OVVbM%zZbYy6FXNx$r^zVl1ol=Y-u^ISyBTjxg5Y4vlRpNE@4-;~y`& z!s9LQZ~!de{*@hEH@T@v9{K7z%+6gqw#a)S zg^q$buvjkFnym2U{sQK->*Awe8#ND-YeQK z%75hi=4a(|oZI}mx8MM?wjZG#MH@fP99api4q5#Ie&c@6m)8Z$!J z3w>F%OPJTWko^~3KVQg>6p`KV_loElafM4d+c=yDH7xo3Fn`*+{}Uu_956}l)c6*a z$(66V9PU%@4Wo^to6wdQzzO?aO&D(+-!ubfX!vTdIL5yi7tv|REgT+|I>N?kHdgGZ zvv0s^odx+~0q0+2@^$-qzHTqEO^`P!1(6j?vQ5SMrExIs4t$E$b)tI*rGYMxqdtO~SQhv0lNIp{M4o+sM?2Lkl$Yb7F69NN zG5ggk$nOt%5aOPHc45Ux{d>4ipFbw7GzYlPA3Mc3HzGu_Rhmu(Vs5+P;CiHa^xv|= zKXb0bS6}Z`PR3S-$D=C)aUo*s0M>=*NT?P!*eg|63Z0h9Ep((0OP12WCOS~@N#wcGtF}~5-2Wz z^+Wt7>QCZ1O-hdOoQC`$&nbudlo)FpsoXI8T!iN|$u>@-$Ca}Yp3}5Ao|E~<@|=kJ zrCIkT8v_dc4DBL}sd|&(HJoI{tAg+yp)A*EcUTAJyW~&=(r|DX=<(g$uKR95sfloc zfJfSjVm=o(#c_i2YQiqnwcZUW4OW-_fs_uh6eoe92@0#DD}@$A2V_glfzON-Y!4F- zsp*qCu=N|!z^N;59w?}}l#8bd) zTFBqg_(y={{G(Vt7Vx*^CzSIM{?S{uuAclO8U8~^FVn?lXH2tHLJn6JG8=N6vIPh2 z!QkTzvzk>xAFeDkdxm+`%EazHj&-8sLMuhyN- zFx8#i4*FEnpWd^?+M_GbW(7txAJxM{UmsslEH?6*8r z1pS97Hd?L*2U)}b>oGV^iI`yZnPg1_@|8>^Q|8@Us554bxWy>3xos;$+sUQ7y z_2v`DoZC${`_65DE&cUFV_t3f+%stFt<8CFTvIjaoX;|8#m^r8ZDz|qwlBTvhG#z5 zz5b>5ySiQ~V~?gtGgm3KW4sd{l1imdL}g4~UhA(-zrX+SU0d$i^4k~Q$UD(~Fu%lA zoAAaR)g|}6@w;Cp?$}9l3xB#Tm(AYt{g97N|K`Czo9F-H#&E^+fr7UzGfv9#-~7i< z9=Y+Y!|n|^rzd{*$=GF{XE#|h{Heb%-to*Tb8t-Kt}hmTeZz}y9m@O9nU{asdF4X& zngw(WNJo@gw6L0ML_?I#3*8P)nfw-0DvweK%ae0tNqrNi%ixvAN%%$@jI_3vlhkTm_H zZ$=&;Ao-qo>Ml!Qv*-76*MDCve(}je%kFw&#EYl4ek(RDIeg%{o4#!;oBZX-Tb9X= zr~mruPwfv{FT3fM(}#n53r-}@&C8tlnak!gpi3YK6LmSfxBk3fAH>0vywJnnSaBX-AB*d*17EP!4Kame{Sr@Pki8B{La#Ej%SwF zzw_~f)7!qfcRy$UL-WLaTBS$Gxh%ahuGmu-{FLaTsRDw4oYWVY$b z_)?uqSKd(;4ggm|FXAisyS8xLf*J2-6&++a)oY1mqHgpWi0`A7zdM6}}Y;)UbmL>WO9;P^{^q~1?(;8}BU&dDIs?Ir;jjH>M z^B=r^Mxs=pxl-UY+WZmdx9N_riH(O5qjs@9qP$M=Rjgt*_8`pq5W!I1lxo`piU{qw z8DICKZduSx@EYg6zRu)i&O$j1o^pcIsx(z*~5OKMJ z>JRJSs>`Htf1ZP$h{pZloQ|KyI33U|0EY%FTySJKrv^M(1GqNfgmslXEupELvHuh$ z^F0ns$ZHDCP4BYQGnwrPui~nw`$H_{PZhlK--Ykv6b`!6rIo&S;Vw)L~ENH8rog>wqjpNL$CC*IYhs0$Ie6?km za)fHdC4L_UBV3ty-$+Ff+B`;!f1&lUJAIrGJv)Ju6T@>YI5+WFPW46!XBqmtnyOU$ zw{>05nCOIre45Th)OGE$Lwi3R=OdQLr=w>h>bkyU*jb0<)0+I=EK#gQ0;GYbL25q> zJxxfmVN&dX9>nnoHAjp?hK!%{JkLy%M8U?eVy+0)!)f+JL zv>E*XAJZk6g53TF_#5pghbfEa^JSR6jk6F|%Jo;2f4{n}FFMi+N+Y@2{bM>bO$M}a z*RBpa3jwI~x~~5YkG9iWPtjXxR7&}CI@@r6*K7CVyh18+tw<}_#7q8;ot^{?uL_j2 z23Obhnxwsjyzs@DVan~b5OPaRgD*?@bHoH_WH<{xwjEGD{*h-O(y=cE_TgtBPW3zk zF*;VKXpOY~1@A2E+NkL-(H=S*HYuiOk9Cao6OR?URdC;HtD(Q7aoz{WCi!~sTYeVh zy~_ujVW(dH1dv1l12YpJU494m{mIh^u<00_WB_$_H|{tmJt;E#=M_qEnohGZ`-^8U z!R*&&{!~8m7xS6F+_pdtD)z|SUuG-T=y&-3y)U}=uE+PmBcEGmB5k8Rp#B)R4L$}> zZiiq`zuL0iww&nV7-#!p$gLc*j^HAk>g9p9T3zw>fWe3IQ(J&D_Kv5mmK8saCro(= zan<@}wHOH0s$Sq#>zmXDwr1sbkUe}N2cz%EK@R!5o^Z;ukVWh#Sp=xRj%&Ux2|4&>$s?I$Mou3p^v^@)vIsAwt2n!_9u$9SKpQ@OXK=>W~^^l z|LDH0<9%BjYx@S?w_E${+edic{yeU4pX7ZTxS ze7%gcSWm{FdZtvYmm4E}`hpV*xsC!)f0%0wf@hf96rAMunNr*# z@OX`l%hVEkYXH2(;A!=8%=e_=_wey?K{d!WPJ3R1w5w}?1M0d!J!3UHX>wXi^-e2d zY*BS;q8;*XCuXs2y!SLtNb!_tY~>)(;2*j}o@HLUb)K>SS7x<>!i;cwm#BSvQP%H7DC_qkl=W+Ily$pC zSraX-wN;FA4o4~H9(41m?tYExcjlsP^4lvPm?Bm^FfDI`lrK(epAKIg)Ov-$r_V#% z^L1HO1L>i}_m%`k#D}y3gT19d^)(a(yhoM;H3?UtFz?4vqyD}x9^2|Nc*FXRIz@Klr?~U(+>d~4_bCYWC zoz+iy_w$kw?>z9BVNJ(1SG_+=eS2!<<`4cyxAh)%%C5iOlb5*1Rl9m?+FIu$Lcxve zGH0!R$mUy#=fM&HrsI2Fge%4p04>ts($D_yGH&b;f&*0=1&F` z<(&KD+J`S|_~(b7%O`$VywYHLS#IllzM4`nN7G_+^fJsxz4H^AL9)z8?@**>5K>>~1M{ZEp54FIV=Mm&K*vm&f+Yw<(1` zI4?K!nwRb#^YY${=A}M1F9R3N%LgL!Qr}}$ zR}HW0>I7cxG!2oUCsd7-NOWC|-QeWEv7=uN3J2~xWQF!y`Za~5*@J0WJ5RbotrQ#I zJ(O-Wi!jtvc+}WbbfaS9Ut8b-?O?Te4d+>rjeqxM{@r4)p(C2IcLSjR~sXkJO@ zHT{4tSm{r{06IxqL1KzIh%o%c-T#B)NCNiafgaNz-P2$6dO6|(f*#=iplc+g;+mM% z@1*}vc?TEI`+C5b1x?1ktUhwXBWZ69nw0nK&AXI6nbVg1=0ft64`+^8lX#+H*xQ$l z>HYoe4{q5oZ}0hcp192~pMU>caQOKz9=`4Tp{lE2KAj#uIc#Bi^NQtvSv)t?@V)85 z?sT(=8<1F%RJW4{;&`=cp%1d9H5) zLgXAZTGTpRnRGWR#SX>wP21%-N1azP(UPl-!a3@1aBgE1&QaftbJU}uGvotp#E8y! zhCvByT0^x+15FDq^bFwNPG}9W9($c-h1NJuhYfq3^4Y7f*D10oN)6|9$b7$Z2jA}; zuq~HAP->b{4xFy8wqg86|SLZW;kf!<$T|uY>w@NqT$M_Zw2hpS$1jrQod0%LKFR#ll~4sFpxI zV1K2-c0<;uV5NF4(}y}Eg|h4$5CgB>~sq9 zMrRU&9w7mNL3XzBB5CWdBX8t!ZF47=*ly$!Tl_*|i(g1=@yjK)cz>M<(%2I*X>4BA z9xjayQ5vXC`A0}&o8qDVP14x1m^2o#h%^?lh%^?lh%~m0OJn!Pq_MBZI6_YbBOD=( z&llqXfoBw@Lt}h`S2VpmjZYBABM?ijjPMBh%U8?Sfk&_{#v^FqJc75P^CrS0_$BV6 zJc67{@CYo0Gvasz1(-KQvlnCDNPN!Z^EtDK&zak8^W_FbiHyGz8??J3I(OI!XsNNe z(VIuGhVPrJEO&Chsz1adz&;ZDelNjAcm%g|9>D_|kAUyBMNb<}RDlEQ%>_rG+4i{d zEVu!0$qhUmdwwmQG|83c;&%V9{NV2YApbJP*(Gc7A%}pDFZG#81Xk>67m4sT`K|j7 z!OrcB8M&V!MsZWr&KNv4I%=U#fmI=PGy6Q{y+3lPM#y>hwukpP6K<7!^M}Ap!6((!x`Co zv2q&NcUFWSD5oW^oS$E^oMT8JGd^#$N3jEw2UehDdqIE|Zb$7slGP zySx^(D?{Dc5(L0WlppAX>_|IZMpuhOK_{_!u!n^>heLjVv(a-#P$KMMsyc~a*}Q?8 zpA5toBLJSUKJ83m>6|y^eqNsm9Bu+zFy(&OqgsGBVt@2)iwSnA029mG#Nkbs$k!B? zuL<8@+-JUaoe6oGc>kN29dJAzSAT7s(zo$FUVjs#{3L4ISp5k~fIY|v9AKk)GArKW z_3zoP(R@vSwMX)m2(C{%lO~i?Vp1ORTJppBnUNoaOz0;Q_n*b*(?#W!mM#m??nRLmqa*8^$72yZUY40wlF6WZvL>r+_cFQ1L4waRUKM_v7 zXq@(4UOV7Lt-QQHO+EUlyS#R^E6YPZOAt6+p$A=Y&b@(4{$X)PN;CH(YdeZDVtL1y z?Z9q}MXc@^$LEcA%pfjl=Jl57Pz{StDGB-6b?hFgGO@A) zd%BQYbnlDyQnn*T~zyqJP{Sfnz>hpffP5aBN_E z-LSy34J(vuL2)eb9IG>Vx*F`o`GI3d?`l|q)BwV6^PEc4I_}Ti6#~Va*$M5vZyCk> zi0uIV486J#`Y+L?L)ki-Y7OxrNXl~#}h zph|#txv@$1WpDOx0Z-fS$Cv?TNjYFQ2DWX{@sR=9M_{`@A@FQdLG+&5wpV-ai?%nP zUw!*geh)r|hOis*pMuZAFFD+z{HK72XdM6s(*7)y*_Y(EI?trLO%!#vakxJ5m2O&W zUJufZ;BbA-Ym;t43}1-4dpTTR{sv7tmnBCxf_zcK4|oHIi}%YE^Fyt>4tT9W!za2& zFAc}wuUhvgx0?^2;{?~sZa!fYT60$LywCz*U6^Sn&$bgM6u}sx0Kx{B6vr86v3Tm0WzDG!_|K!X>WU?rQ|(n))_?I`_reSp->v?FzjD4n!yhqpf@Vvhvzj`Bd`{rJ5< zI{KEYW#P4uA3Gj%>~@TEBvdi7=kh#o?`pB7Qq@TrIb=~ISs6G*a1??2+^-V){G{sz z9n2i);g>;+vJ$%T)sEu`pUTxrDeXuX$HFv!Wklx;ra0)0WgLe5G3l7jG8lK!bym;~5fKUz zIDF4}nLzJZ1a`_I=n~0CTAsk?@8CZDr3vf{D>uwNUOcU*bk4=>JNtR9)6K7c(=ZNcHtDkw+)KKq;eAfvo!=VcMt<7gAAClliH zMV{CgpIN*1jACCfDh=BkEl=6KQ_Kgy(|FxH!2R9S&*asCci$416?qOGG!-xMyc=f4ivsV4MNdi? z_{MsxVhh{{OgKB<-}e=@dBV1?-4lY~VWjv>0yH&DLEm#ZDX?(l_3begRtHm`0p2py zlC9IuUjyIhCdJ;+tdzEfqHi|%T4AO1xg3L!td!b~(K*Qn2ifaawU~QSO{R3&xQIQW zsMW?ASd2nB(y>CQl`o=|gwpR<=qe;fhU>KZH11Q#dToVTu2xM9cByQGWd9atYLW@F z7s@s4@Jx@9_~509!6!X~R6rj zEW5EZ!L~+GOTR@~3o9TE8fmIKyN3lYsK(S;D>(#l9iL?8Mzq3Ke-Y@>Mq1arCdlaR zQQk9QGve{NRJXcLgzk}qlSFf~9gahCf%9#7uJe#Q7Z|yKFn(QEQwXP(4$CuKiS9)C zdVfH*lVuS3dVa2IMvVjuU9#F?9gE+vb!;`LZ{mC7jfFUOwQxCU0LYD`IU$GDWQ=Ig zNo^fFie@whI(ERC5Uhud5ys@f&Jbebh4F}yJ^__E+Q(GhuC77lZExkQwGB>t(Xwyv zJ-=VEp!UKIV^75Kt$RcV3^?<%>!yCxu=n$?7QC{^R+qo?#Z9TLM@`x&x#X{g&+(Yr z2eiTE*|15aq!S$xr60O&R5@ll+rVkFgId6up5UZVLQ6(SFAaAbbNHME%C$9u<0Hpr z=WL8VQ?*!r{yofu@=!$~^mp_f#(=(K5VQ&>E7NHFG!*=K5LV5}u=s0i*26-V7@!N*fA{WcVYB<{gw ziyz-m0vCI5aZmmi^DG>hXEauS+&r8A@0n-MnX(4^<~?txd6pw#o(;Bl!Y&)1e;(Ou zvX<4I-D{#bXx^lynm6%zSfOkv)!C}>n`qm_=h*p*LO#Esr$e((+2sBe&D6*o3!nvH z9U60t+6Ut`evSdZU&>E-H$5&gTE8ykC;aAadYpA}xW4^_cg6V$kHKO69vwf*K|g;Q z_34d0`5mv;dwP41Mbd?;E0YP9}c70oe`K%MXeq^*E7mL8S5FIaX7>a z5s;tArrnfTz(m-{(P$A|v7W*hg+Z5xFb#3|1lMKKd;DF4E|%_R6nKz`cN1BX z#i#?{>&h$+m!~svINYQDivUHmjz;g`8OqXu_4Pc$x5%Fn9S0xt5BJxxhxvPb({)eu zct@k}nmK%5^j$jmGy-ep`x!y2Z?FA~Rvs}@Hxz^RGlWH4JKWo`3L0C09a?;+i~--N z!OgyNJXCqch9N@}-2;OcRB^0vtlyP^c`E{wq*eMG3u=!8TYoE$UdPHA#%G3h7SXH! zB3wK%sQ=<{MMxP_9s;xoYj`-L=+&PL7n_6nlZRVy-=5*vy(>(UCqfiP@k1#6HB-Gs|n zmB%NKMA{zQF$BGduGmkBRohW6R$Pm62^D9siW*V6_4eaFgPGmmwSRavht{yaTt)II zQ(6_vN~?Uf>RDHjyU5;({kVwzcpgsh_&3vuZkD7DD@-{ciGgiTB|l&g3K!)!$${z| zb+f;bpZHEvC$wwNf0agMSd`z`R8JUB3^aucA265)G|hkeI)D9P=#6i#f7{MH<~3=F z3+;{D4o~o&2IpsbaL{fsIOIh!@W`_%JS892ld$?xFKF_Gpk{+d*LW-hdv&!!FSrG&-9 zbFqibZmSQAN0KQ^`V%+t1u6_qrmgmr;; zJMsf_J-T4(7UL0Nm!RB)-4{~Bd9V53(_vJc0pzDR zXyrk@?)GPNFl7_1JsoJhPPAC}KFqBjXROR~|GtKe-@!F1jAWl;yaP3D_7YX0!>Pe$ zTm`uh@~q4NtcaoR`tmE~Sy*Q?EAwQ_I_53#uo~6Jc0&Vlmgr~Ecg5PudkjQU2taIL13iVR-CNuaYfnDrSnc8k|{j|AW!#%4|* z13Y(A4?HKbnUG`D0@s}c9(n?DWl3!mZ{@}G0;ke`l62o)J8>Gl+WE;?(PbEqIf#?z++Tp#h{V_$_QXMh#0DrsBQ zbyXpdUHjq?y@hm%(#a}5*|P6x$sX-BnEHSA#r?k&{r?Es4Ez9g4(*m3*Z(uo|1Y8c zCH{Wj{h!<2{}H}KpZyPBgTPjR=a7mpfOr(K?`iAmVDvxn38YK({|H_-Z9S!NGPqBE z0=N(uBb<+baYAqcOB=Z_t2ckZ0AowZY%cK_^7KsMa6S1c7>#^?9pwd-7U@|j;CjnB zwDnm4*)1>=gAXTspZmR`I7%`MZ)!H6b zs%F7T)wEJcp2aOs)o?3SvvBIbu34!Xz_KzT)~R+xt|4V{oRz9+nw2WN2)%Ki;ix_O zENg&m*1xn;l`gv{Wsr?3*d@+^TlIb2`zHxvA};d`KopIwY_5nQz2&}V>q z82$FJ6@lP-$j7?j@&V_h{#}9qP2Q%l0#4itAnXdbL&T`DwgVyCl`Dw;eqp}{b` z5ZXs%zps>4G}*3(?nU10vCzF3Y|B!na{Y^Ru75FxpRrzQyH>7Hrnb!S;4HFbvJLy| zh|bcgKpR#`+!afX1iyhOMnfB8iiNLIdT7w!8aSl(@vAJU2efS&txnm{pr)LCbA8b8 z_oW1wWtKAg^~m#Xy&~RTGpEsdBcUg-6VU!FLv#;WhOl{-6t;)@iuV`pIea{SjNU-N z9N)=q&28au+J3^sN^)C}?@u%c>B; zAL_rqMdT5LZ%n=DcP)QYPx^he2+d3>TL^Zku#uBZ5{LeDemc9!+%LN|zqKmll5&I% z4Ekj2t1q4OUl+p4` z<+>KcP|G};Z=h{&^wSfG_y*V!aGBHk1TJARJIzE(7Z+NK+m0cR**rh_NCw4uepU}g zkM%ZgKdnr$aETG#vdxCXNODTb?2GCZCZEPJPL&S&b4e`*A5I+xbA7tvCDu}}!Dq6T zB0bSt;A16AJea@M5|_cbAN;EGo&x9`j`jKuu)-W=!fI0+dS4j{zR^ltO*9seQ>Fj1 zhJYH<-h`eWIIf;%RkhxVwJBFM70*R^rQ*5DvS#G%4dBU4f^R5L?aE9|=Bg|UUJ3mHH~Z`&`sUos>FF490BLX_o3XO3)U_l8n$5 zn66}+hsd`onesy3{(OI|okLwGLNg$Xa%#Yg#SSHpEI42v1Fsh=g=X2CSH@Zb2c}T4 zUFXUonY+rlnHO1oTM%diO+N?ne*yA;#EXl0N?}YlOupzH;9$HHCt<>FH&U}~BhHrS zB?Hct;5wduGvjJDLK=jcO;d)NGv#7ssJtLjvrK9+oRjTcEBz&FHSq^)CG!$uOvP$- z3r|_}1GOsS5cc?S2t0aYFZmCxgv=taoe0}B`4-#B;UIs49$QcOlLmJkaQSeonH!>T z!23PnM2H{-#^K`RGhPlKkME=MI31^zIXvF;R-;U`CIkLQ(WAieu!Wb6-A68spu}u{ z_P*R!d*@c!C)IWN*cRExFuTIVOM|lS0Alvyr5MlqJ#CQ0ENzhe30ll!@G)yP_QV+R z)}>jAbzOROTbFM|wv?r>0}nn}@!X9UGfdStj9*iN8Y;hay!0m?Z zEGseo333E4R4I;^&18WquN2cvLOmH9IU{>>FTbxyD+8^g(Y=UY^c=bdcy8kF_o;Wl zz}dWJ32)CqTt3F=9{n|%`aSvwJt+}-Qp;jKW>Fpy0r{GQAK_<~6fNQ_2R={c{1)xo=On}A*!wyk!sBLXX z7ajKEYps(@#KKT~U=*xe&f!)LkZtTd{F-KlMsCwcGc%Fv+ zPeHT|>1xrV9re(=yP6HN>)B9hci#5N>~_4j3h(KA$$_K$mlb$#C<2G)??4Vp@WK^0 zj6!c*5&4b=FR+0Gi~SBWBFJ-zKbI|J!+s}?f1+S>^MZ(xz7|gFhv{6?5A*B5M!vtGAoSP9-m@}P~TcYia^Sfl1bzJoIB2){O5P?PMfEi&i|;u(=u z{}wr?JVTy~^&37$f|(x7_Qs7KaoI@VWYLaOc0@1nv%-0@( zVsj$nbt=ZI8SrrL*QcCbw4^a(WSq_coOp?GZ9u+r5QfqGioj0>JQHw_>9Uyt&qx0( z29|>`nS31M`$8e+63O4?nS6Ys$6ZZmRh(4JMcmzl7Bm8*{HXCAIOb)Gh6OV^U>0d{R4wL# z;wHNY`j+%x4-WO>yJ<(zMd_OOELn{Xiuq{9WaDi*)8a+d{Z|tT@&}I$QQu(Eee0 zxIEg~E)REhPz;*+gQI#sQB1I$L8>QNM`6Z57H1s<>W|PPL^jx@E2B+ z=$vymfjc|{GrtFa81*0yF-{YKGQ?A@1{k-)`qcXo*>a1(bz-<~4j<Ssmf75}N9LXQopAc;n(dS_-V?sLC^9;-(+7n~8U>!2Cz0l2{5aGk^ z74>W{=F(o8cfIJp=z3yi>2tKR40^VZ-~`-b-ASkXqJQ}I6A1G#_ovX>gM7Ju49~YO zx=XdQTTIFc;Z25@30BHU2u<@QwiuV+I|x_4DNup zh;9PN2`oZETciH!8f zn1Fj^ex(Mc{aTX1FiK+Cly@FR0rF*w5dg36p*O|L#oKKp>9FVPSq?&fgpugaG{E!p zQBi$+UJ6dv^tXc^S*~XTS?0}$d3lnVp5O)C1CM9+D9^~Ad>OzEWf@ncqCIGw^0pv& z(4A{BPWhO`_g7&&ttv#|@O|@*`^Sybp@189iE+Au&zCeNAm8|Ldiy2HF$Vp;6yu{m zubW<1f$>p+{D=1que&h{A2&Xzy@g(UDy<(eUwOW`i;a&NGs@5&RQvwX7Qr~(6r;gS zy#Ia_{f+OA?x9Bz-B%4}BIQT;LPP+Z*o&Um_J0@`ynKKeq{00^onM4cl!VCsO;F zy4$bk{OE2!UOtSMe^>cTJ@7Q-wpiZ;;G?;*bu*do^K&ku4`{zR4DzoWi~$qsi-3Cc zBm6$OHy%$!uI6d0czrRp2u{E~#+4agjQvH&RUYaq_B~!MBVe3^3nMUgJ)OhTsmQOV zoQ0r(6LAiYU{yM+w-_PveGbp=PCb4F6YmBj|I>-OuwB*F;~feqmaR->> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK) +#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops) + #define ADF_DEFAULT_RING_TO_SRV_MAP \ (CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ @@ -156,7 +167,9 @@ enum adf_accel_unit_services { ADF_ACCEL_SERVICE_NULL = 0, ADF_ACCEL_INLINE_CRYPTO = 1, ADF_ACCEL_CRYPTO = 2, - ADF_ACCEL_COMPRESSION = 4 + ADF_ACCEL_COMPRESSION = 4, + ADF_ACCEL_ASYM = 8, + ADF_ACCEL_ADMIN = 16 }; struct adf_ae_info { @@ -182,6 +195,7 @@ struct adf_accel_unit_info { u32 dc_ae_msk; u8 num_cy_au; u8 num_dc_au; + u8 num_asym_au; u8 num_inline_au; struct adf_accel_unit *au; const struct adf_ae_info *ae_info; @@ -231,6 +245,60 @@ struct admin_info { u32 mailbox_offset; } __packed; +struct adf_hw_csr_ops { + u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size); + u32 (*read_csr_ring_head)(struct resource *csr_base_addr, + u32 bank, + u32 ring); + void (*write_csr_ring_head)(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value); + u32 (*read_csr_ring_tail)(struct resource *csr_base_addr, + u32 bank, + u32 ring); + void (*write_csr_ring_tail)(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value); + u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank); + void (*write_csr_ring_config)(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value); + void (*write_csr_ring_base)(struct resource *csr_base_addr, + u32 bank, + u32 ring, + bus_addr_t addr); + void (*write_csr_int_flag)(struct resource *csr_base_addr, + u32 bank, + u32 value); + void (*write_csr_int_srcsel)(struct resource *csr_base_addr, u32 bank); + void (*write_csr_int_col_en)(struct resource *csr_base_addr, + u32 bank, + u32 value); + void (*write_csr_int_col_ctl)(struct resource *csr_base_addr, + u32 bank, + u32 value); + void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr, + u32 bank, + u32 value); + u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr, + u32 bank); + void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr, + u32 bank, + u32 value); +}; + +struct adf_hw_csr_info { + struct adf_hw_csr_ops csr_ops; + u32 csr_addr_offset; + u32 ring_bundle_size; + u32 bank_int_flag_clear_mask; + u32 num_rings_per_int_srcsel; + u32 arb_enable_mask; +}; + struct adf_cfg_device_data; struct adf_accel_dev; struct adf_etr_data; @@ -282,8 +350,10 @@ struct adf_hw_device_data { void (*exit_arb)(struct adf_accel_dev *accel_dev); void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, const uint32_t **cfg); + int (*init_device)(struct adf_accel_dev *accel_dev); int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev); uint32_t (*get_ae_clock)(struct adf_hw_device_data *self); + uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, bool enable); @@ -298,6 +368,8 @@ struct adf_hw_device_data { void (*restore_device)(struct adf_accel_dev *accel_dev); uint32_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev, enum adf_accel_unit_services services); + enum adf_accel_unit_services ( + *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num); int (*add_pke_stats)(struct adf_accel_dev *accel_dev); void (*remove_pke_stats)(struct adf_accel_dev *accel_dev); int (*add_misc_error)(struct adf_accel_dev *accel_dev); @@ -311,6 +383,14 @@ struct adf_hw_device_data { enum adf_accel_unit_services services); void (*pre_reset)(struct adf_accel_dev *accel_dev); void (*post_reset)(struct adf_accel_dev *accel_dev); + void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); + void (*get_ring_svc_map_data)(int ring_pair_index, + u16 ring_to_svc_map, + u8 *serv_type, + int *ring_index, + int *num_rings_per_srv, + int bundle_num); + struct adf_hw_csr_info csr_info; const char *fw_name; const char *fw_mmp_name; bool reset_ack; @@ -320,7 +400,10 @@ struct adf_hw_device_data { uint16_t accel_mask; u32 aerucm_mask; u32 ae_mask; + u32 admin_ae_mask; u32 service_mask; + u32 service_to_load_mask; + u32 heartbeat_ctr_num; uint16_t tx_rings_mask; uint8_t tx_rx_gap; uint8_t num_banks; diff --git a/sys/dev/qat/include/common/adf_cfg.h b/sys/dev/qat/include/common/adf_cfg.h index edc4813cb69e..58502c8605b8 100644 --- a/sys/dev/qat/include/common/adf_cfg.h +++ b/sys/dev/qat/include/common/adf_cfg.h @@ -76,4 +76,14 @@ int adf_cfg_get_services_enabled(struct adf_accel_dev *accel_dev, int adf_cfg_restore_section(struct adf_accel_dev *accel_dev, struct adf_cfg_section *section); void adf_cfg_keyval_del_all(struct list_head *head); + +static inline int +adf_cy_inst_cross_banks(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device->num_rings_per_bank == 2) + return 1; + else + return 0; +} + #endif diff --git a/sys/dev/qat/include/common/adf_cfg_common.h b/sys/dev/qat/include/common/adf_cfg_common.h index 68fb5e8a98b3..65fc60fc8c3d 100644 --- a/sys/dev/qat/include/common/adf_cfg_common.h +++ b/sys/dev/qat/include/common/adf_cfg_common.h @@ -27,7 +27,7 @@ #define ADF_MAX_ACCELENGINES 12 #define ADF_CFG_STORAGE_ENABLED 1 #define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES) -#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000 +#define ADF_GEN2_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000 #define ADF_WDT_TIMER_SYM_COMP_MS 3 #define ADF_MIN_HB_TIMER_MS 100 #define ADF_CFG_MAX_NUM_OF_SECTIONS 16 @@ -87,7 +87,8 @@ enum adf_device_type { DEV_200XX, DEV_200XXVF, DEV_C4XXX, - DEV_C4XXXVF + DEV_C4XXXVF, + DEV_4XXX }; enum adf_cfg_fw_image_type { @@ -158,6 +159,7 @@ struct adf_cfg_bundle { /* contains all the info about rings */ struct adf_cfg_ring **rings; u16 in_use; + u16 max_cfg_svc_num; }; struct adf_cfg_instance { diff --git a/sys/dev/qat/include/common/adf_cfg_strings.h b/sys/dev/qat/include/common/adf_cfg_strings.h index 2f05dadadc45..933ffe0ba6ad 100644 --- a/sys/dev/qat/include/common/adf_cfg_strings.h +++ b/sys/dev/qat/include/common/adf_cfg_strings.h @@ -22,6 +22,8 @@ #define ADF_RING_DC_RX "RingRx" #define ADF_ETRMGR_BANK "Bank" #define ADF_RING_BANK_NUM "BankNumber" +#define ADF_RING_BANK_NUM_ASYM "BankNumberAsym" +#define ADF_RING_BANK_NUM_SYM "BankNumberSym" #define ADF_CY "Cy" #define ADF_DC "Dc" #define ADF_DC_EXTENDED_FEATURES "Device_DcExtendedFeatures" @@ -112,6 +114,8 @@ #define ADF_CY_CORE_AFFINITY_FORMAT ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY #define ADF_DC_CORE_AFFINITY_FORMAT ADF_DC "%d" ADF_ETRMGR_CORE_AFFINITY #define ADF_CY_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM +#define ADF_CY_ASYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_ASYM +#define ADF_CY_SYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_SYM #define ADF_DC_BANK_NUM_FORMAT ADF_DC "%d" ADF_RING_BANK_NUM #define ADF_CY_ASYM_TX_FORMAT ADF_CY "%d" ADF_RING_ASYM_TX #define ADF_CY_SYM_TX_FORMAT ADF_CY "%d" ADF_RING_SYM_TX diff --git a/sys/dev/qat/include/common/adf_common_drv.h b/sys/dev/qat/include/common/adf_common_drv.h index 3bb35ed55da3..7ec380540336 100644 --- a/sys/dev/qat/include/common/adf_common_drv.h +++ b/sys/dev/qat/include/common/adf_common_drv.h @@ -203,10 +203,14 @@ int adf_init_gen2_arb(struct adf_accel_dev *accel_dev); void adf_exit_arb(struct adf_accel_dev *accel_dev); void adf_disable_arb(struct adf_accel_dev *accel_dev); void adf_update_ring_arb(struct adf_etr_ring_data *ring); -void -adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask); -void -adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask); +void adf_enable_ring_arb(struct adf_accel_dev *accel_dev, + void *csr_addr, + unsigned int bank_nr, + unsigned int mask); +void adf_disable_ring_arb(struct adf_accel_dev *accel_dev, + void *csr_addr, + unsigned int bank_nr, + unsigned int mask); int adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); struct adf_accel_dev *adf_devmgr_get_dev_by_bdf(struct adf_pci_address *addr); struct adf_accel_dev *adf_devmgr_get_dev_by_pci_bus(u8 bus); @@ -238,9 +242,7 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, - unsigned int ctx_mask); +int qat_hal_start(struct icp_qat_fw_loader_handle *handle); void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask); diff --git a/sys/dev/qat/include/common/adf_gen2_hw_data.h b/sys/dev/qat/include/common/adf_gen2_hw_data.h new file mode 100644 index 000000000000..395abec81b9f --- /dev/null +++ b/sys/dev/qat/include/common/adf_gen2_hw_data.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2021 Intel Corporation */ +/* $FreeBSD$ */ +#ifndef ADF_GEN2_HW_DATA_H_ +#define ADF_GEN2_HW_DATA_H_ + +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" + +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 +#define ADF_GEN2_RX_RINGS_OFFSET 8 +#define ADF_GEN2_TX_RINGS_MASK 0xFF + +#define BUILD_RING_BASE_ADDR(addr, size) \ + (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD + \ + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL + \ + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), \ + value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ + do { \ + u32 l_base = 0, u_base = 0; \ + l_base = (u32)((value)&0xFFFFFFFF); \ + u_base = (u32)(((value)&0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_LBASE + ((ring) << 2), \ + l_base); \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_UBASE + ((ring) << 2), \ + u_base); \ + } while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD + \ + ((ring) << 2), \ + value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL + \ + ((ring) << 2), \ + value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_FLAG, \ + value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + do { \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL, \ + ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL_2, \ + ADF_BANK_INT_SRC_SEL_MASK_X); \ + } while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_EN, \ + value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, \ + value) + +/* AE to function map */ +#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define AE2FUNCTION_MAP_REG_SIZE 4 +#define AE2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, \ + AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, \ + AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), \ + value) +#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, \ + AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, \ + AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), \ + value) + +/* Admin Interface Offsets */ +#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_MAILBOX_BASE_OFFSET 0x20970 + +/* Arbiter configuration */ +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define READ_CSR_RING_SRV_ARB_EN(csr_addr, index) \ + ADF_CSR_RD(csr_addr, \ + ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index))) + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, \ + ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)), \ + value) + +/* Power gating */ +#define ADF_POWERGATE_DC BIT(23) +#define ADF_POWERGATE_PKE BIT(24) + +/* Default ring mapping */ +#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \ + (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ + CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +/* Error detection and correction */ +#define ADF_GEN2_AE_CTX_ENABLES(i) ((i)*0x1000 + 0x20818) +#define ADF_GEN2_AE_MISC_CONTROL(i) ((i)*0x1000 + 0x20960) +#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28) +#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) +#define ADF_GEN2_UERRSSMSH(i) ((i)*0x4000 + 0x18) +#define ADF_GEN2_CERRSSMSH(i) ((i)*0x4000 + 0x10) +#define ADF_GEN2_ERRSSMSH_EN BIT(3) + +#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND) + +void adf_gen2_init_hw_csr_info(struct adf_hw_csr_info *csr_info); + +#endif diff --git a/sys/dev/qat/include/common/adf_gen4_hw_data.h b/sys/dev/qat/include/common/adf_gen4_hw_data.h new file mode 100644 index 000000000000..c0ef0c92772e --- /dev/null +++ b/sys/dev/qat/include/common/adf_gen4_hw_data.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2021 Intel Corporation */ +/* $FreeBSD$ */ +#ifndef ADF_GEN4_HW_CSR_DATA_H_ +#define ADF_GEN4_HW_CSR_DATA_H_ + +#include "adf_accel_devices.h" + +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL +#define ADF_RING_CSR_RING_CONFIG 0x1000 +#define ADF_RING_CSR_RING_LBASE 0x1040 +#define ADF_RING_CSR_RING_UBASE 0x1080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_ADDR_OFFSET 0x100000 +#define ADF_RING_BUNDLE_SIZE 0x2000 + +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), \ + value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ + do { \ + struct resource *_csr_base_addr = csr_base_addr; \ + u32 _bank = bank; \ + u32 _ring = ring; \ + dma_addr_t _value = value; \ + u32 l_base = 0, u_base = 0; \ + l_base = lower_32_bits(_value); \ + u_base = upper_32_bits(_value); \ + ADF_CSR_WR((_csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_LBASE + ((_ring) << 2), \ + l_base); \ + ADF_CSR_WR((_csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_UBASE + ((_ring) << 2), \ + u_base); \ + } while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), \ + value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), \ + value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG, \ + (value)) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, \ + ADF_BANK_INT_SRC_SEL_MASK) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_EN, \ + (value)) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, \ + (value)) + +/* Arbiter configuration */ +#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C + +#define READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN) + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr), \ + ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, \ + (value)) + +/* WDT timers + * + * Timeout is in cycles. Clock speed may vary across products but this + * value should be a few milli-seconds. + */ +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL +#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000 +#define ADF_SSMWDTL_OFFSET 0x54 +#define ADF_SSMWDTH_OFFSET 0x5C +#define ADF_SSMWDTPKEL_OFFSET 0x58 +#define ADF_SSMWDTPKEH_OFFSET 0x60 + +#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE) + +int adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +void adf_gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info); +#endif diff --git a/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h b/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h index a8afb5a4b377..7f6e80eeb431 100644 --- a/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h +++ b/sys/dev/qat/include/common/icp_qat_fw_loader_handle.h @@ -16,6 +16,7 @@ struct icp_qat_fw_loader_ae_data { struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; unsigned int ae_mask; + unsigned int admin_ae_mask; unsigned int slice_mask; unsigned int revision_id; unsigned int ae_max_num; diff --git a/sys/dev/qat/include/common/icp_qat_hal.h b/sys/dev/qat/include/common/icp_qat_hal.h index 3a7475f25333..68e12826a7e8 100644 --- a/sys/dev/qat/include/common/icp_qat_hal.h +++ b/sys/dev/qat/include/common/icp_qat_hal.h @@ -52,23 +52,32 @@ enum hal_ae_csr { }; enum fcu_csr { - FCU_CONTROL = 0x0, - FCU_STATUS = 0x4, - FCU_DRAM_ADDR_LO = 0xc, + FCU_CONTROL = 0x00, + FCU_STATUS = 0x04, + FCU_DRAM_ADDR_LO = 0x0c, FCU_DRAM_ADDR_HI = 0x10, FCU_RAMBASE_ADDR_HI = 0x14, FCU_RAMBASE_ADDR_LO = 0x18 }; enum fcu_csr_c4xxx { - FCU_CONTROL_C4XXX = 0x0, - FCU_STATUS_C4XXX = 0x4, - FCU_STATUS1_C4XXX = 0xc, + FCU_CONTROL_C4XXX = 0x00, + FCU_STATUS_C4XXX = 0x04, + FCU_STATUS1_C4XXX = 0x0c, FCU_AE_LOADED_C4XXX = 0x10, FCU_DRAM_ADDR_LO_C4XXX = 0x14, FCU_DRAM_ADDR_HI_C4XXX = 0x18, }; +enum fcu_csr_4xxx { + FCU_CONTROL_4XXX = 0x00, + FCU_STATUS_4XXX = 0x04, + FCU_ME_BROADCAST_MASK_TYPE = 0x08, + FCU_AE_LOADED_4XXX = 0x10, + FCU_DRAM_ADDR_LO_4XXX = 0x14, + FCU_DRAM_ADDR_HI_4XXX = 0x18, +}; + enum fcu_cmd { FCU_CTRL_CMD_NOOP = 0, FCU_CTRL_CMD_AUTH = 1, @@ -104,6 +113,7 @@ enum fcu_sts { #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 #define GLOBAL_CSR 0xA00 +#define FCU_CTRL_BROADCAST_POS 0x4 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 @@ -111,20 +121,26 @@ enum fcu_sts { #define FCU_LOADED_AE_POS 0x16 #define FW_AUTH_WAIT_PERIOD 10 #define FW_AUTH_MAX_RETRY 300 +#define FW_BROADCAST_MAX_RETRY 300 #define FCU_OFFSET 0x8c0 #define FCU_OFFSET_C4XXX 0x1000 +#define FCU_OFFSET_4XXX 0x1000 #define MAX_CPP_NUM 2 #define AE_CPP_NUM 2 #define AES_PER_CPP 16 #define SLICES_PER_CPP 6 #define ICP_QAT_AE_OFFSET 0x20000 #define ICP_QAT_AE_OFFSET_C4XXX 0x40000 +#define ICP_QAT_AE_OFFSET_4XXX 0x600000 #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) #define ICP_QAT_CAP_OFFSET_C4XXX 0x70000 +#define ICP_QAT_CAP_OFFSET_4XXX 0x640000 #define LOCAL_TO_XFER_REG_OFFSET 0x800 #define ICP_QAT_EP_OFFSET 0x3a000 #define ICP_QAT_EP_OFFSET_C4XXX 0x60000 +#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */ #define MEM_CFG_ERR_BIT 0x20 +#define AE_TG_NUM_CPM2X 4 #define CAP_CSR_ADDR(csr) (csr + handle->hal_cap_g_ctl_csr_addr_v) #define SET_CAP_CSR(handle, csr, val) \ @@ -133,20 +149,17 @@ enum fcu_sts { ADF_CSR_RD(handle->hal_misc_addr_v, CAP_CSR_ADDR(csr)) #define SET_GLB_CSR(handle, csr, val) \ ({ \ - typeof(handle) handle_ = (handle); \ - typeof(csr) csr_ = (csr); \ - typeof(val) val_ = (val); \ - (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \ - SET_CAP_CSR(handle_, (csr_), (val_)) : \ - SET_CAP_CSR(handle_, csr_ + GLOBAL_CSR, val_); \ + u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev)); \ + (IS_QAT_GEN3_OR_GEN4(dev_id)) ? \ + SET_CAP_CSR((handle), (csr), (val)) : \ + SET_CAP_CSR((handle), (csr) + GLOBAL_CSR, val); \ }) #define GET_GLB_CSR(handle, csr) \ ({ \ - typeof(handle) handle_ = (handle); \ - typeof(csr) csr_ = (csr); \ - (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \ - (GET_CAP_CSR(handle_, (csr_))) : \ - (GET_CAP_CSR(handle_, (GLOBAL_CSR + (csr_)))); \ + u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev)); \ + (IS_QAT_GEN3_OR_GEN4(dev_id)) ? \ + GET_CAP_CSR((handle), (csr)) : \ + GET_CAP_CSR((handle), (csr) + GLOBAL_CSR); \ }) #define SET_FCU_CSR(handle, csr, val) \ ({ \ @@ -157,7 +170,12 @@ enum fcu_sts { SET_CAP_CSR(handle_, \ ((csr_) + FCU_OFFSET_C4XXX), \ (val_)) : \ - SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_)); \ + ((IS_QAT_GEN4( \ + pci_get_device(GET_DEV(handle_->accel_dev)))) ? \ + SET_CAP_CSR(handle_, \ + ((csr_) + FCU_OFFSET_4XXX), \ + (val_)) : \ + SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_))); \ }) #define GET_FCU_CSR(handle, csr) \ ({ \ @@ -165,7 +183,10 @@ enum fcu_sts { typeof(csr) csr_ = (csr); \ (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \ GET_CAP_CSR(handle_, (FCU_OFFSET_C4XXX + (csr_))) : \ - GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_))); \ + ((IS_QAT_GEN4( \ + pci_get_device(GET_DEV(handle_->accel_dev)))) ? \ + GET_CAP_CSR(handle_, (FCU_OFFSET_4XXX + (csr_))) : \ + GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_)))); \ }) #define AE_CSR(handle, ae) \ ((handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) @@ -184,13 +205,19 @@ enum fcu_sts { ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val) #define GET_CSR_OFFSET(device_id, cap_offset_, ae_offset_, ep_offset_) \ ({ \ - int gen3 = IS_QAT_GEN3(device_id); \ - cap_offset_ = \ - (gen3 ? ICP_QAT_CAP_OFFSET_C4XXX : ICP_QAT_CAP_OFFSET); \ - ae_offset_ = \ - (gen3 ? ICP_QAT_AE_OFFSET_C4XXX : ICP_QAT_AE_OFFSET); \ - ep_offset_ = \ - (gen3 ? ICP_QAT_EP_OFFSET_C4XXX : ICP_QAT_EP_OFFSET); \ + if (IS_QAT_GEN3(device_id)) { \ + cap_offset_ = ICP_QAT_CAP_OFFSET_C4XXX; \ + ae_offset_ = ICP_QAT_AE_OFFSET_C4XXX; \ + ep_offset_ = ICP_QAT_EP_OFFSET_C4XXX; \ + } else if (IS_QAT_GEN4(device_id)) { \ + cap_offset_ = ICP_QAT_CAP_OFFSET_4XXX; \ + ae_offset_ = ICP_QAT_AE_OFFSET_4XXX; \ + ep_offset_ = ICP_QAT_EP_OFFSET_4XXX; \ + } else { \ + cap_offset_ = ICP_QAT_CAP_OFFSET; \ + ae_offset_ = ICP_QAT_AE_OFFSET; \ + ep_offset_ = ICP_QAT_EP_OFFSET; \ + } \ }) #endif diff --git a/sys/dev/qat/include/common/icp_qat_uclo.h b/sys/dev/qat/include/common/icp_qat_uclo.h index 21a1c2fc8ace..1bdddce1d85e 100644 --- a/sys/dev/qat/include/common/icp_qat_uclo.h +++ b/sys/dev/qat/include/common/icp_qat_uclo.h @@ -9,6 +9,7 @@ #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 #define ICP_QAT_AC_200XX_DEV_TYPE 0x02000000 #define ICP_QAT_AC_C4XXX_DEV_TYPE 0x04000000 +#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 #define ICP_QAT_UCLO_MAX_AE 32 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_CPPNUM 2 @@ -17,6 +18,7 @@ #define ICP_QAT_UCLO_MAX_XFER_REG 128 #define ICP_QAT_UCLO_MAX_GPR_REG 128 #define ICP_QAT_UCLO_MAX_LMEM_REG 1024 +#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280 #define ICP_QAT_UCLO_AE_ALL_CTX 0xff #define ICP_QAT_UOF_OBJID_LEN 8 #define ICP_QAT_UOF_FID 0xc6c2 @@ -46,22 +48,42 @@ #define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) -#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 -#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4 -#define ICP_QAT_CSS_FWSK_PAD_LEN 252 -#define ICP_QAT_CSS_FWSK_PUB_LEN \ - (ICP_QAT_CSS_FWSK_MODULUS_LEN + ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_FWSK_PAD_LEN) -#define ICP_QAT_CSS_SIGNATURE_LEN 256 + +#define DSS_FWSK_MODULUS_LEN 384 // RSA3K +#define DSS_FWSK_EXPONENT_LEN 4 +#define DSS_FWSK_PADDING_LEN 380 +#define DSS_SIGNATURE_LEN 384 // RSA3K + +#define CSS_FWSK_MODULUS_LEN 256 // RSA2K +#define CSS_FWSK_EXPONENT_LEN 4 +#define CSS_FWSK_PADDING_LEN 252 +#define CSS_SIGNATURE_LEN 256 // RSA2K + +#define ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) \ + (IS_QAT_GEN4(ID) ? DSS_FWSK_MODULUS_LEN : CSS_FWSK_MODULUS_LEN) + +#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) \ + (IS_QAT_GEN4(ID) ? DSS_FWSK_EXPONENT_LEN : CSS_FWSK_EXPONENT_LEN) + +#define ICP_QAT_CSS_FWSK_PAD_LEN(ID) \ + (IS_QAT_GEN4(ID) ? DSS_FWSK_PADDING_LEN : CSS_FWSK_PADDING_LEN) + +#define ICP_QAT_CSS_FWSK_PUB_LEN(ID) \ + (ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) + ICP_QAT_CSS_FWSK_PAD_LEN(ID)) + +#define ICP_QAT_CSS_SIGNATURE_LEN(ID) \ + (IS_QAT_GEN4(ID) ? DSS_SIGNATURE_LEN : CSS_SIGNATURE_LEN) + #define ICP_QAT_CSS_AE_IMG_LEN \ (sizeof(struct icp_qat_simg_ae_mode) + ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ ICP_QAT_SIMG_AE_INSTS_LEN) -#define ICP_QAT_CSS_AE_SIMG_LEN \ - (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN + ICP_QAT_CSS_AE_IMG_LEN) -#define ICP_QAT_AE_IMG_OFFSET \ - (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + ICP_QAT_CSS_SIGNATURE_LEN) +#define ICP_QAT_CSS_AE_SIMG_LEN(ID) \ + (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN(ID) + \ + ICP_QAT_CSS_SIGNATURE_LEN(ID) + ICP_QAT_CSS_AE_IMG_LEN) +#define ICP_QAT_AE_IMG_OFFSET(ID) \ + (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) + ICP_QAT_CSS_SIGNATURE_LEN(ID)) #define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode)&0xf) diff --git a/sys/dev/qat/include/icp_qat_fw_init_admin.h b/sys/dev/qat/include/icp_qat_fw_init_admin.h index 6f88de144770..9c1482158443 100644 --- a/sys/dev/qat/include/icp_qat_fw_init_admin.h +++ b/sys/dev/qat/include/icp_qat_fw_init_admin.h @@ -200,10 +200,6 @@ struct icp_qat_fw_init_admin_hb_cnt { u16 req_heartbeat_cnt; }; -struct icp_qat_fw_init_admin_hb_stats { - struct icp_qat_fw_init_admin_hb_cnt stats[ADF_NUM_HB_CNT_PER_AE]; -}; - #define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 #define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 #define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0 diff --git a/sys/dev/qat/include/qat_ocf_utils.h b/sys/dev/qat/include/qat_ocf_utils.h index 0cacd8f0a84f..30a7e9b7f8ec 100644 --- a/sys/dev/qat/include/qat_ocf_utils.h +++ b/sys/dev/qat/include/qat_ocf_utils.h @@ -45,8 +45,11 @@ static inline CpaBoolean is_use_sep_digest(const struct crypto_session_params *csp) { /* Use separated digest for all digest/hash operations, - * including GMAC */ - if (CSP_MODE_DIGEST == csp->csp_mode || CSP_MODE_ETA == csp->csp_mode) + * including GMAC. ETA and AEAD use separated digest + * due to FW limitation to specify offset to digest + * appended to pay-load buffer. */ + if (CSP_MODE_DIGEST == csp->csp_mode || CSP_MODE_ETA == csp->csp_mode || + CSP_MODE_AEAD == csp->csp_mode) return CPA_TRUE; return CPA_FALSE; diff --git a/sys/dev/qat/qat/qat_ocf.c b/sys/dev/qat/qat/qat_ocf.c index 2461f3134a77..7e5025b0fa28 100644 --- a/sys/dev/qat/qat/qat_ocf.c +++ b/sys/dev/qat/qat/qat_ocf.c @@ -29,6 +29,9 @@ #include "icp_adf_accel_mgr.h" #include "lac_sal_types.h" +/* To disable AEAD HW MAC verification */ +#include "icp_sal_user.h" + /* QAT OCF specific headers */ #include "qat_ocf_mem_pool.h" #include "qat_ocf_utils.h" @@ -423,24 +426,6 @@ qat_ocf_session_init(device_t dev, switch (csp->csp_mode) { case CSP_MODE_AEAD: - sessionSetupData.symOperation = - CPA_CY_SYM_OP_ALGORITHM_CHAINING; - /* Place the digest result in a buffer unrelated to srcBuffer */ - sessionSetupData.digestIsAppended = CPA_TRUE; - /* For GCM and CCM driver forces to verify digest on HW */ - sessionSetupData.verifyDigest = CPA_TRUE; - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { - sessionSetupData.cipherSetupData.cipherDirection = - CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; - sessionSetupData.algChainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; - } else { - sessionSetupData.cipherSetupData.cipherDirection = - CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; - sessionSetupData.algChainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; - } - break; case CSP_MODE_ETA: sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; @@ -1086,6 +1071,15 @@ qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev) goto fail; } + /* Disable forcing HW MAC validation for AEAD */ + status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE); + if (CPA_STATUS_SUCCESS != status) { + device_printf( + qat_softc->sc_dev, + "unable to disable AEAD HW MAC verification\n"); + goto fail; + } + qat_ocf_instance->driver_id = qat_softc->cryptodev_id; startedInstances++; @@ -1222,6 +1216,7 @@ MODULE_DEPEND(qat, qat_200xx, 1, 1, 1); MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1); MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1); MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1); +MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1); MODULE_DEPEND(qat, crypto, 1, 1, 1); MODULE_DEPEND(qat, qat_common, 1, 1, 1); MODULE_DEPEND(qat, qat_api, 1, 1, 1); diff --git a/sys/dev/qat/qat_api/common/compression/dc_buffers.c b/sys/dev/qat/qat_api/common/compression/dc_buffers.c index 1a5d9bc8973e..4f4e836ccf8f 100644 --- a/sys/dev/qat/qat_api/common/compression/dc_buffers.c +++ b/sys/dev/qat/qat_api/common/compression/dc_buffers.c @@ -26,9 +26,14 @@ #include "sal_types_compression.h" #include "icp_qat_fw_comp.h" +#include "sal_hw_gen.h" #define CPA_DC_CEIL_DIV(x, y) (((x) + (y)-1) / (y)) #define DC_DEST_BUFF_EXTRA_DEFLATE_GEN2 (55) +#define DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_STATIC (1029) +#define DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_DYN (512) +#define DC_DEST_BUFF_MIN_EXTRA_BYTES(x) ((x < 8) ? (8 - x) : 0) +#define DC_BUF_MAX_SIZE (0xFFFFFFFF) CpaStatus cpaDcBufferListGetMetaSize(const CpaInstanceHandle instanceHandle, @@ -72,13 +77,60 @@ cpaDcBnpBufferListGetMetaSize(const CpaInstanceHandle instanceHandle, static inline CpaStatus dcDeflateBoundGen2(CpaDcHuffType huffType, Cpa32U inputSize, Cpa32U *outputSize) { + Cpa64U inBufferSize = inputSize; + Cpa64U outBufferSize = 0; + /* Formula for GEN2 deflate: * ceil(9 * Total input bytes / 8) + 55 bytes. * 55 bytes is the skid pad value for GEN2 devices. + * Adding extra bytes = `DC_DEST_BUFF_MIN_EXTRA_BYTES(inputSize)` + * when calculated value from `CPA_DC_CEIL_DIV(9 * inputSize, 8) + + * DC_DEST_BUFF_EXTRA_DEFLATE_GEN2` is less than 64 bytes to + * achieve a safer output buffer size of 64 bytes. */ - *outputSize = - CPA_DC_CEIL_DIV(9 * inputSize, 8) + DC_DEST_BUFF_EXTRA_DEFLATE_GEN2; + outBufferSize = CPA_DC_CEIL_DIV(9 * inBufferSize, 8) + + DC_DEST_BUFF_EXTRA_DEFLATE_GEN2 + + DC_DEST_BUFF_MIN_EXTRA_BYTES(inputSize); + if (outBufferSize > DC_BUF_MAX_SIZE) + *outputSize = DC_BUF_MAX_SIZE; + else + *outputSize = (Cpa32U)outBufferSize; + + return CPA_STATUS_SUCCESS; +} + +static inline CpaStatus +dcDeflateBoundGen4(CpaDcHuffType huffType, Cpa32U inputSize, Cpa32U *outputSize) +{ + Cpa64U outputSizeLong; + Cpa64U inputSizeLong = (Cpa64U)inputSize; + + switch (huffType) { + case CPA_DC_HT_STATIC: + /* Formula for GEN4 static deflate: + * ceil((9*sourceLen)/8) + 5 + 1024. */ + outputSizeLong = CPA_DC_CEIL_DIV(9 * inputSizeLong, 8) + + DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_STATIC; + break; + case CPA_DC_HT_FULL_DYNAMIC: + /* Formula for GEN4 dynamic deflate: + * Ceil ((9*sourceLen)/8)ā–’| + + * ((((8/7) * sourceLen)/ 16KB) * (150+5)) + 512 + */ + outputSizeLong = DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_DYN; + outputSizeLong += CPA_DC_CEIL_DIV(9 * inputSizeLong, 8); + outputSizeLong += ((8 * inputSizeLong * 155) / 7) / (16 * 1024); + break; + default: + return CPA_STATUS_INVALID_PARAM; + } + + /* Avoid output size overflow */ + if (outputSizeLong & 0xffffffff00000000UL) + return CPA_STATUS_INVALID_PARAM; + + *outputSize = (Cpa32U)outputSizeLong; return CPA_STATUS_SUCCESS; } @@ -88,6 +140,7 @@ cpaDcDeflateCompressBound(const CpaInstanceHandle dcInstance, Cpa32U inputSize, Cpa32U *outputSize) { + sal_compression_service_t *pService = NULL; CpaInstanceHandle insHandle = NULL; if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) { @@ -112,5 +165,10 @@ cpaDcDeflateCompressBound(const CpaInstanceHandle dcInstance, return CPA_STATUS_INVALID_PARAM; } - return dcDeflateBoundGen2(huffType, inputSize, outputSize); + pService = (sal_compression_service_t *)insHandle; + if (isDcGen4x(pService)) { + return dcDeflateBoundGen4(huffType, inputSize, outputSize); + } else { + return dcDeflateBoundGen2(huffType, inputSize, outputSize); + } } diff --git a/sys/dev/qat/qat_api/common/compression/dc_datapath.c b/sys/dev/qat/qat_api/common/compression/dc_datapath.c index 0e2aa9f389e2..2e1f9ff96bd8 100644 --- a/sys/dev/qat/qat_api/common/compression/dc_datapath.c +++ b/sys/dev/qat/qat_api/common/compression/dc_datapath.c @@ -42,6 +42,7 @@ #include "lac_sync.h" #include "sal_service_state.h" #include "sal_qat_cmn_msg.h" +#include "sal_hw_gen.h" #include "dc_error_counter.h" #define DC_COMP_MAX_BUFF_SIZE (1024 * 64) @@ -71,6 +72,28 @@ getDcErrorCounter(CpaDcReqStatus dcError) return 0; } +static inline void +dcUpdateXltOverflowChecksumsGen4(const dc_compression_cookie_t *pCookie, + const icp_qat_fw_resp_comp_pars_t *pRespPars, + CpaDcRqResults *pDcResults) +{ + dc_session_desc_t *pSessionDesc = + DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle); + + /* Recompute CRC checksum when either the checksum type + * is CPA_DC_CRC32 or when the integrity CRCs are enabled. + */ + if (CPA_DC_CRC32 == pSessionDesc->checksumType) { + pDcResults->checksum = pRespPars->crc.legacy.curr_crc32; + + /* No need to recalculate the swCrc64I here as this will get + * handled later in dcHandleIntegrityChecksumsGen4. + */ + } else if (CPA_DC_ADLER32 == pSessionDesc->checksumType) { + pDcResults->checksum = pRespPars->crc.legacy.curr_adler_32; + } +} + void dcCompression_ProcessCallback(void *pRespMsg) { @@ -86,6 +109,8 @@ dcCompression_ProcessCallback(void *pRespMsg) dc_compression_cookie_t *pCookie = NULL; CpaDcOpData *pOpData = NULL; CpaBoolean cmpPass = CPA_TRUE, xlatPass = CPA_TRUE; + CpaBoolean isDcDp = CPA_FALSE; + CpaBoolean integrityCrcCheck = CPA_FALSE; CpaBoolean verifyHwIntegrityCrcs = CPA_FALSE; Cpa8U cmpErr = ERR_CODE_NO_ERROR, xlatErr = ERR_CODE_NO_ERROR; dc_request_dir_t compDecomp = DC_COMPRESSION_REQUEST; @@ -102,17 +127,20 @@ dcCompression_ProcessCallback(void *pRespMsg) pCookie = (dc_compression_cookie_t *)pReqData; if (!pCookie) return; - pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle); - if (CPA_TRUE == pSessionDesc->isDcDp) { + pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle); + pService = (sal_compression_service_t *)(pCookie->dcInstance); + + isDcDp = pSessionDesc->isDcDp; + if (CPA_TRUE == isDcDp) { pResponse = (CpaDcDpOpData *)pReqData; pResults = &(pResponse->results); if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) { compDecomp = DC_DECOMPRESSION_REQUEST; } + pCookie = NULL; } else { - pSessionDesc = pCookie->pSessionDesc; pResults = pCookie->pResults; callbackTag = pCookie->callbackTag; pCbFunc = pCookie->pSessionDesc->pCompressionCb; @@ -120,8 +148,6 @@ dcCompression_ProcessCallback(void *pRespMsg) pOpData = pCookie->pDcOpData; } - pService = (sal_compression_service_t *)(pCookie->dcInstance); - opStatus = pCompRespMsg->comn_resp.comn_status; if (NULL != pOpData) { @@ -142,15 +168,17 @@ dcCompression_ProcessCallback(void *pRespMsg) pResults->status = (Cpa8S)cmpErr; pResults->consumed = 0; pResults->produced = 0; - if (CPA_TRUE == pSessionDesc->isDcDp) { + if (CPA_TRUE == isDcDp) { if (pResponse) pResponse->responseStatus = CPA_STATUS_UNSUPPORTED; (pService->pDcDpCb)(pResponse); } else { /* Free the memory pool */ - Lac_MemPoolEntryFree(pCookie); - pCookie = NULL; + if (NULL != pCookie) { + Lac_MemPoolEntryFree(pCookie); + pCookie = NULL; + } if (NULL != pCbFunc) { pCbFunc(callbackTag, status); } @@ -169,9 +197,23 @@ dcCompression_ProcessCallback(void *pRespMsg) ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(opStatus)); } - if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) { - cmpPass = CPA_TRUE; - cmpErr = ERR_CODE_NO_ERROR; + if (isDcGen2x(pService)) { + /* QAT1.7 and QAT 1.8 hardware */ + if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) { + cmpPass = CPA_TRUE; + cmpErr = ERR_CODE_NO_ERROR; + } + } else { + /* QAT2.0 hardware cancels the incomplete file errors + * only for DEFLATE algorithm. + * Decompression direction is not tested in the callback as + * the request does not allow it. + */ + if ((pSessionDesc->compType == CPA_DC_DEFLATE) && + (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr)) { + cmpPass = CPA_TRUE; + cmpErr = ERR_CODE_NO_ERROR; + } } /* log the slice hang and endpoint push/pull error inside the response */ @@ -199,8 +241,7 @@ dcCompression_ProcessCallback(void *pRespMsg) xlatErr = pCompRespMsg->comn_resp.comn_error.s1.xlat_err_code; /* Return a fatal error or a potential error in the translator - * slice - * if the compression slice did not return any error */ + * slice if the compression slice did not return any error */ if ((CPA_DC_OK == pResults->status) || (CPA_DC_FATALERR == (Cpa8S)xlatErr)) { pResults->status = (Cpa8S)xlatErr; @@ -209,7 +250,7 @@ dcCompression_ProcessCallback(void *pRespMsg) /* Update dc error counter */ dcErrorLog(pResults->status); - if (CPA_FALSE == pSessionDesc->isDcDp) { + if (CPA_FALSE == isDcDp) { /* In case of any error for an end of packet request, we need to * update * the request type for the following request */ @@ -223,17 +264,27 @@ dcCompression_ProcessCallback(void *pRespMsg) ((CPA_DC_STATELESS == pSessionDesc->sessState) && (DC_COMPRESSION_REQUEST == compDecomp))) { /* Overflow is a valid use case for Traditional API - * only. - * Stateful Overflow is supported in both compression - * and - * decompression direction. - * Stateless Overflow is supported only in compression - * direction. + * only. Stateful Overflow is supported in both + * compression and decompression direction. Stateless + * Overflow is supported only in compression direction. */ if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr) cmpPass = CPA_TRUE; if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) { + if (isDcGen4x(pService) && + (CPA_TRUE == + pService->comp_device_data + .translatorOverflow)) { + pResults->consumed = + pCompRespMsg->comp_resp_pars + .input_byte_counter; + + dcUpdateXltOverflowChecksumsGen4( + pCookie, + &pCompRespMsg->comp_resp_pars, + pResults); + } xlatPass = CPA_TRUE; } } @@ -242,6 +293,7 @@ dcCompression_ProcessCallback(void *pRespMsg) cmpPass = CPA_FALSE; } if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) { + /* XLT overflow is not valid for Data Plane requests */ xlatPass = CPA_FALSE; } } @@ -254,7 +306,13 @@ dcCompression_ProcessCallback(void *pRespMsg) pCompRespMsg->comp_resp_pars.output_byte_counter; pSessionDesc->cumulativeConsumedBytes += pResults->consumed; - if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) { + /* Handle Checksum for end to end data integrity. */ + if (CPA_TRUE == + pService->generic_service_info.integrityCrcCheck && + CPA_TRUE == integrityCrcCheck) { + pSessionDesc->previousChecksum = + pSessionDesc->seedSwCrc.swCrc32I; + } else if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) { if (CPA_DC_CRC32 == pSessionDesc->checksumType) { pResults->checksum = pCompRespMsg->comp_resp_pars.crc.legacy @@ -279,7 +337,7 @@ dcCompression_ProcessCallback(void *pRespMsg) if ((CPA_DC_OVERFLOW != (Cpa8S)xlatErr) && (CPA_TRUE == verifyHwIntegrityCrcs)) { pSessionDesc->previousChecksum = - pSessionDesc->seedSwCrc.swCrcI; + pSessionDesc->seedSwCrc.swCrc32I; } /* Check if a CNV recovery happened and @@ -292,7 +350,7 @@ dcCompression_ProcessCallback(void *pRespMsg) pService); } - if (CPA_TRUE == pSessionDesc->isDcDp) { + if (CPA_TRUE == isDcDp) { if (pResponse) pResponse->responseStatus = CPA_STATUS_SUCCESS; } else { @@ -305,8 +363,26 @@ dcCompression_ProcessCallback(void *pRespMsg) } } } else { +#ifdef ICP_DC_RETURN_COUNTERS_ON_ERROR + /* Extract the response from the firmware */ + pResults->consumed = + pCompRespMsg->comp_resp_pars.input_byte_counter; + pResults->produced = + pCompRespMsg->comp_resp_pars.output_byte_counter; + + if (CPA_DC_STATEFUL == pSessionDesc->sessState) { + pSessionDesc->cumulativeConsumedBytes += + pResults->consumed; + } else { + /* In the stateless case all requests have both SOP and + * EOP set */ + pSessionDesc->cumulativeConsumedBytes = + pResults->consumed; + } +#else pResults->consumed = 0; pResults->produced = 0; +#endif if (CPA_DC_OVERFLOW == pResults->status && CPA_DC_STATELESS == pSessionDesc->sessState) { /* This error message will be returned by Data Plane API @@ -319,7 +395,7 @@ dcCompression_ProcessCallback(void *pRespMsg) "Unrecoverable error: stateless overflow. You may need to increase the size of your destination buffer.\n"); } - if (CPA_TRUE == pSessionDesc->isDcDp) { + if (CPA_TRUE == isDcDp) { if (pResponse) pResponse->responseStatus = CPA_STATUS_FAIL; } else { @@ -338,7 +414,7 @@ dcCompression_ProcessCallback(void *pRespMsg) } } - if (CPA_TRUE == pSessionDesc->isDcDp) { + if (CPA_TRUE == isDcDp) { /* Decrement number of stateless pending callbacks for session */ pSessionDesc->pendingDpStatelessCbCount--; @@ -383,7 +459,7 @@ dcCompression_ProcessCallback(void *pRespMsg) * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in * *****************************************************************************/ -static CpaStatus +CpaStatus dcCheckOpData(sal_compression_service_t *pService, CpaDcOpData *pOpData) { CpaDcSkipMode skipMode = 0; @@ -440,6 +516,14 @@ dcCheckOpData(sal_compression_service_t *pService, CpaDcOpData *pOpData) "supported on this device"); return CPA_STATUS_INVALID_PARAM; } + + if (CPA_TRUE == pOpData->integrityCrcCheck && + NULL == pOpData->pCrcData) { + LAC_INVALID_PARAM_LOG("Integrity CRC data structure " + "not intialized in CpaDcOpData"); + return CPA_STATUS_INVALID_PARAM; + } + return CPA_STATUS_SUCCESS; } @@ -583,8 +667,9 @@ dcCheckDestinationData(sal_compression_service_t *pService, if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { /* Check if intermediate buffers are supported */ - if ((0 == pService->pInterBuffPtrsArrayPhyAddr) || - (NULL == pService->pInterBuffPtrsArray)) { + if ((isDcGen2x(pService)) && + ((0 == pService->pInterBuffPtrsArrayPhyAddr) || + (NULL == pService->pInterBuffPtrsArray))) { LAC_LOG_ERROR( "No intermediate buffer defined for this instance " "- see cpaDcStartInstance"); @@ -702,6 +787,7 @@ dcCreateRequest(dc_compression_cookie_t *pCookie, Cpa8U crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY; Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; Cpa8U cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; + CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX; CpaBoolean integrityCrcCheck = CPA_FALSE; CpaStatus status = CPA_STATUS_SUCCESS; CpaDcFlush flush = CPA_DC_FLUSH_NONE; @@ -757,7 +843,7 @@ dcCreateRequest(dc_compression_cookie_t *pCookie, * bigger as allocated by the user. We ensure that this is not the case * in dcCheckSourceData and cast the values to Cpa32U here */ pCookie->srcTotalDataLenInBytes = (Cpa32U)srcTotalDataLenInBytes; - if ((DC_COMPRESSION_REQUEST == compDecomp) && + if ((isDcGen2x(pService)) && (DC_COMPRESSION_REQUEST == compDecomp) && (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) { if (pService->minInterBuffSizeInBytes < (Cpa32U)dstTotalDataLenInBytes) { @@ -813,9 +899,9 @@ dcCreateRequest(dc_compression_cookie_t *pCookie, initial_crc32 = 0; if (CPA_DC_ADLER32 == pSessionDesc->checksumType) { - pSessionDesc->previousChecksum = 1; + pSessionDesc->previousChecksum = initial_adler; } else { - pSessionDesc->previousChecksum = 0; + pSessionDesc->previousChecksum = initial_crc32; } } else if (CPA_DC_STATELESS == pSessionDesc->sessState) { pSessionDesc->previousChecksum = pResults->checksum; @@ -906,7 +992,6 @@ dcCreateRequest(dc_compression_cookie_t *pCookie, eop = ICP_QAT_FW_COMP_NOT_EOP; } } else { - if (DC_REQUEST_FIRST == pSessionDesc->requestType) { /* Reinitialise the cumulative amount of consumed bytes */ @@ -928,6 +1013,9 @@ dcCreateRequest(dc_compression_cookie_t *pCookie, * cnvDecompReq also needs to be set */ case DC_CNV: cnvDecompReq = ICP_QAT_FW_COMP_CNV; + if (isDcGen4x(pService)) { + cnvErrorInjection = pSessionDesc->cnvErrorInjection; + } break; case DC_NO_CNV: cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; @@ -936,8 +1024,14 @@ dcCreateRequest(dc_compression_cookie_t *pCookie, } /* LW 18 */ - rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( - sop, eop, bFinal, cnvDecompReq, cnvRecovery, crcMode); + rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop, + eop, + bFinal, + cnvDecompReq, + cnvRecovery, + cnvErrorInjection, + crcMode); + pMsg->comp_pars.req_par_flags = rpCmdFlags; /* Populates the QAT common request middle part of the message @@ -1244,10 +1338,7 @@ dcZeroLengthRequests(sal_compression_service_t *pService, COMPRESSION_STAT_INC(numDecompCompleted, pService); } - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot unlock session lock"); - } + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); if ((NULL != pCbFunc) && (LacSync_GenWakeupSyncCaller != pCbFunc)) { @@ -1371,7 +1462,7 @@ cpaDcCompressData(CpaInstanceHandle dcInstance, return dcCompDecompData(pService, pSessionDesc, - dcInstance, + insHandle, pSessionHandle, pSrcBuff, pDestBuff, @@ -1506,10 +1597,7 @@ cpaDcCompressData2(CpaInstanceHandle dcInstance, if (CPA_DC_STATEFUL == pSessionDesc->sessState) { /* Lock the session to check if there are in-flight stateful * requests */ - if (CPA_STATUS_SUCCESS != - LAC_SPINLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot unlock session lock"); - } + LAC_SPINLOCK(&(pSessionDesc->sessionLock)); /* Check if there is already one in-flight stateful request */ if (0 != @@ -1517,10 +1605,7 @@ cpaDcCompressData2(CpaInstanceHandle dcInstance, &(pSessionDesc->pendingStatefulCbCount))) { LAC_LOG_ERROR( "Only one in-flight stateful request supported"); - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot unlock session lock"); - } + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); return CPA_STATUS_RETRY; } @@ -1537,10 +1622,7 @@ cpaDcCompressData2(CpaInstanceHandle dcInstance, } qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount)); - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot unlock session lock"); - } + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); } if (CPA_TRUE == pOpData->compressAndVerify) { @@ -1549,7 +1631,7 @@ cpaDcCompressData2(CpaInstanceHandle dcInstance, return dcCompDecompData(pService, pSessionDesc, - dcInstance, + insHandle, pSessionHandle, pSrcBuff, pDestBuff, @@ -1659,16 +1741,50 @@ cpaDcDecompressData(CpaInstanceHandle dcInstance, pService = (sal_compression_service_t *)insHandle; + /* Check if SAL is initialised otherwise return an error */ + SAL_RUNNING_CHECK(insHandle); + + /* This check is outside the parameter checking as it is needed to + * manage zero length requests */ + if (CPA_STATUS_SUCCESS != + LacBuffDesc_BufferListVerifyNull(pSrcBuff, + &srcBuffSize, + LAC_NO_ALIGNMENT_SHIFT)) { + QAT_UTILS_LOG("Invalid source buffer list parameter"); + return CPA_STATUS_INVALID_PARAM; + } + + /* Ensure this is a compression instance */ + SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION); + + if (dcCheckSourceData(pSessionHandle, + pSrcBuff, + pDestBuff, + pResults, + flushFlag, + srcBuffSize, + NULL) != CPA_STATUS_SUCCESS) { + return CPA_STATUS_INVALID_PARAM; + } + if (dcCheckDestinationData(pService, + pSessionHandle, + pDestBuff, + DC_DECOMPRESSION_REQUEST) != + CPA_STATUS_SUCCESS) { + return CPA_STATUS_INVALID_PARAM; + } pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); + if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) { + QAT_UTILS_LOG("Invalid sessDirection value"); + return CPA_STATUS_INVALID_PARAM; + } + + if (CPA_DC_STATEFUL == pSessionDesc->sessState) { /* Lock the session to check if there are in-flight stateful * requests */ - if (CPA_STATUS_SUCCESS != - LAC_SPINLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot lock session lock"); - return CPA_STATUS_RESOURCE; - } + LAC_SPINLOCK(&(pSessionDesc->sessionLock)); /* Check if there is already one in-flight stateful request */ if (0 != @@ -1676,37 +1792,36 @@ cpaDcDecompressData(CpaInstanceHandle dcInstance, &(pSessionDesc->pendingStatefulCbCount))) { LAC_LOG_ERROR( "Only one in-flight stateful request supported"); - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot unlock session lock"); - } + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); return CPA_STATUS_RETRY; } - if ((0 == srcBuffSize) || - ((1 == srcBuffSize) && (CPA_DC_FLUSH_FINAL != flushFlag) && - (CPA_DC_FLUSH_FULL != flushFlag))) { - if (CPA_TRUE == - dcZeroLengthRequests(pService, - pSessionDesc, - pResults, - flushFlag, - callbackTag, - DC_DECOMPRESSION_REQUEST)) { - return CPA_STATUS_SUCCESS; + /* Gen 4 handle 0 len requests in FW */ + if (isDcGen2x(pService)) { + if ((0 == srcBuffSize) || + ((1 == srcBuffSize) && + (CPA_DC_FLUSH_FINAL != flushFlag) && + (CPA_DC_FLUSH_FULL != flushFlag))) { + if (CPA_TRUE == + dcZeroLengthRequests( + pService, + pSessionDesc, + pResults, + flushFlag, + callbackTag, + DC_DECOMPRESSION_REQUEST)) { + return CPA_STATUS_SUCCESS; + } } } qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount)); - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) { - LAC_LOG_ERROR("Cannot unlock session lock"); - } + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); } return dcCompDecompData(pService, pSessionDesc, - dcInstance, + insHandle, pSessionHandle, pSrcBuff, pDestBuff, @@ -1768,12 +1883,89 @@ cpaDcDecompressData2(CpaInstanceHandle dcInstance, pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); - if (CPA_DC_STATEFUL == pSessionDesc->sessState) { - LAC_INVALID_PARAM_LOG("Invalid session: Stateful session is " - "not supported"); + LAC_CHECK_NULL_PARAM(insHandle); + + /* Check if SAL is initialised otherwise return an error */ + SAL_RUNNING_CHECK(insHandle); + + /* This check is outside the parameter checking as it is needed to + * manage zero length requests */ + if (CPA_STATUS_SUCCESS != + LacBuffDesc_BufferListVerifyNull(pSrcBuff, + &srcBuffSize, + LAC_NO_ALIGNMENT_SHIFT)) { + QAT_UTILS_LOG("Invalid source buffer list parameter"); return CPA_STATUS_INVALID_PARAM; } + /* Ensure this is a compression instance */ + SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION); + + if (CPA_STATUS_SUCCESS != + dcCheckSourceData(pSessionHandle, + pSrcBuff, + pDestBuff, + pResults, + CPA_DC_FLUSH_NONE, + srcBuffSize, + NULL)) { + return CPA_STATUS_INVALID_PARAM; + } + if (CPA_STATUS_SUCCESS != + dcCheckDestinationData(pService, + pSessionHandle, + pDestBuff, + DC_DECOMPRESSION_REQUEST)) { + return CPA_STATUS_INVALID_PARAM; + } + + if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) { + return CPA_STATUS_INVALID_PARAM; + } + + if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) { + QAT_UTILS_LOG("Invalid sessDirection value"); + return CPA_STATUS_INVALID_PARAM; + } + + + if (CPA_DC_STATEFUL == pSessionDesc->sessState) { + /* Lock the session to check if there are in-flight stateful + * requests */ + LAC_SPINLOCK(&(pSessionDesc->sessionLock)); + + /* Check if there is already one in-flight stateful request */ + if (0 != + qatUtilsAtomicGet( + &(pSessionDesc->pendingStatefulCbCount))) { + LAC_LOG_ERROR( + "Only one in-flight stateful request supported"); + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); + return CPA_STATUS_RETRY; + } + + /* Gen 4 handle 0 len requests in FW */ + if (isDcGen2x(pService)) { + if ((0 == srcBuffSize) || + ((1 == srcBuffSize) && + (CPA_DC_FLUSH_FINAL != pOpData->flushFlag) && + (CPA_DC_FLUSH_FULL != pOpData->flushFlag))) { + if (CPA_TRUE == + dcZeroLengthRequests( + pService, + pSessionDesc, + pResults, + pOpData->flushFlag, + callbackTag, + DC_DECOMPRESSION_REQUEST)) { + return CPA_STATUS_SUCCESS; + } + } + } + qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount)); + LAC_SPINUNLOCK(&(pSessionDesc->sessionLock)); + } + return dcCompDecompData(pService, pSessionDesc, insHandle, diff --git a/sys/dev/qat/qat_api/common/compression/dc_dp.c b/sys/dev/qat/qat_api/common/compression/dc_dp.c index 4a24bf17dc32..9b00c5b09d7e 100644 --- a/sys/dev/qat/qat_api/common/compression/dc_dp.c +++ b/sys/dev/qat/qat_api/common/compression/dc_dp.c @@ -41,6 +41,7 @@ #include "sal_service_state.h" #include "sal_qat_cmn_msg.h" #include "icp_sal_poll.h" +#include "sal_hw_gen.h" /** ***************************************************************************** @@ -87,8 +88,8 @@ dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) /* Compressing zero byte is not supported */ if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) && (0 == pOpData->bufferLenToCompress)) { - QAT_UTILS_LOG( - "The source buffer length to compress needs to be greater than zero byte.\n"); + QAT_UTILS_LOG("The source buffer length to compress needs to " + "be greater than zero byte.\n"); return CPA_STATUS_INVALID_PARAM; } @@ -171,8 +172,7 @@ dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) } else { /* We are assuming that there is enough memory in the source and * destination buffer lists. We only receive physical addresses - * of the - * buffers so we are unable to test it here */ + * of the buffers so we are unable to test it here */ LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->srcBuffer); LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->destBuffer); } @@ -183,8 +183,9 @@ dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) (CPA_DC_DIR_COMBINED == pSessionDesc->sessDirection)) { if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { /* Check if Intermediate Buffer Array pointer is NULL */ - if ((0 == pService->pInterBuffPtrsArrayPhyAddr) || - (NULL == pService->pInterBuffPtrsArray)) { + if (isDcGen2x(pService) && + ((0 == pService->pInterBuffPtrsArrayPhyAddr) || + (NULL == pService->pInterBuffPtrsArray))) { QAT_UTILS_LOG( "No intermediate buffer defined for this instance - see cpaDcStartInstance.\n"); return CPA_STATUS_INVALID_PARAM; @@ -312,7 +313,10 @@ dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; Cpa8U cnvnrCompReq = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; + CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX; + sal_compression_service_t *pService = NULL; + pService = (sal_compression_service_t *)(pOpData->dcInstance); pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); if (CPA_DC_DIR_COMPRESS == pOpData->sessDirection) { @@ -320,6 +324,11 @@ dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) /* CNV check */ if (CPA_TRUE == pOpData->compressAndVerify) { cnvDecompReq = ICP_QAT_FW_COMP_CNV; + if (isDcGen4x(pService)) { + cnvErrorInjection = + pSessionDesc->cnvErrorInjection; + } + /* CNVNR check */ if (CPA_TRUE == pOpData->compressAndVerifyAndRecover) { cnvnrCompReq = ICP_QAT_FW_COMP_CNV_RECOVERY; @@ -343,7 +352,13 @@ dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) pCurrentQatMsg->comp_pars.req_par_flags |= ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( - 0, 0, 0, cnvDecompReq, cnvnrCompReq, 0); + ICP_QAT_FW_COMP_NOT_SOP, + ICP_QAT_FW_COMP_NOT_EOP, + ICP_QAT_FW_COMP_NOT_BFINAL, + cnvDecompReq, + cnvnrCompReq, + cnvErrorInjection, + ICP_QAT_FW_COMP_CRC_MODE_LEGACY); SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pCurrentQatMsg, pOpData, diff --git a/sys/dev/qat/qat_api/common/compression/dc_session.c b/sys/dev/qat/qat_api/common/compression/dc_session.c index 1d742e227a10..fbce72cb7bfb 100644 --- a/sys/dev/qat/qat_api/common/compression/dc_session.c +++ b/sys/dev/qat/qat_api/common/compression/dc_session.c @@ -23,6 +23,7 @@ #include "icp_qat_fw.h" #include "icp_qat_fw_comp.h" #include "icp_qat_hw.h" +#include "icp_qat_hw_20_comp.h" /* ******************************************************************************* @@ -36,6 +37,7 @@ #include "lac_buffer_desc.h" #include "sal_service_state.h" #include "sal_qat_cmn_msg.h" +#include "sal_hw_gen.h" /** ***************************************************************************** @@ -54,7 +56,7 @@ * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature * *****************************************************************************/ -static CpaStatus +CpaStatus dcCheckSessionData(const CpaDcSessionSetupData *pSessionData, CpaInstanceHandle dcInstance) { @@ -67,6 +69,7 @@ dcCheckSessionData(const CpaDcSessionSetupData *pSessionData, QAT_UTILS_LOG("Invalid compLevel value\n"); return CPA_STATUS_INVALID_PARAM; } + if ((pSessionData->autoSelectBestHuffmanTree < CPA_DC_ASB_DISABLED) || (pSessionData->autoSelectBestHuffmanTree > CPA_DC_ASB_UNCOMP_STATIC_DYNAMIC_WITH_NO_HDRS)) { @@ -122,10 +125,10 @@ dcCheckSessionData(const CpaDcSessionSetupData *pSessionData, * *****************************************************************************/ static void -dcCompHwBlockPopulate(dc_session_desc_t *pSessionDesc, +dcCompHwBlockPopulate(sal_compression_service_t *pService, + dc_session_desc_t *pSessionDesc, icp_qat_hw_compression_config_t *pCompConfig, - dc_request_dir_t compDecomp, - icp_qat_hw_compression_delayed_match_t enableDmm) + dc_request_dir_t compDecomp) { icp_qat_hw_compression_direction_t dir = ICP_QAT_HW_COMPRESSION_DIR_COMPRESS; @@ -134,6 +137,7 @@ dcCompHwBlockPopulate(dc_session_desc_t *pSessionDesc, icp_qat_hw_compression_depth_t depth = ICP_QAT_HW_COMPRESSION_DEPTH_1; icp_qat_hw_compression_file_type_t filetype = ICP_QAT_HW_COMPRESSION_FILE_TYPE_0; + icp_qat_hw_compression_delayed_match_t dmm; /* Set the direction */ if (DC_COMPRESSION_REQUEST == compDecomp) { @@ -148,6 +152,13 @@ dcCompHwBlockPopulate(dc_session_desc_t *pSessionDesc, QAT_UTILS_LOG("Algorithm not supported for Compression\n"); } + /* Set delay match mode */ + if (CPA_TRUE == pService->comp_device_data.enableDmm) { + dmm = ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED; + } else { + dmm = ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED; + } + /* Set the depth */ if (DC_DECOMPRESSION_REQUEST == compDecomp) { depth = ICP_QAT_HW_COMPRESSION_DEPTH_1; @@ -162,8 +173,13 @@ dcCompHwBlockPopulate(dc_session_desc_t *pSessionDesc, case CPA_DC_L3: depth = ICP_QAT_HW_COMPRESSION_DEPTH_8; break; - default: + case CPA_DC_L4: depth = ICP_QAT_HW_COMPRESSION_DEPTH_16; + break; + default: + depth = pService->comp_device_data + .highestHwCompressionDepth; + break; } } @@ -171,10 +187,138 @@ dcCompHwBlockPopulate(dc_session_desc_t *pSessionDesc, * modes will be used in the future for precompiled huffman trees */ filetype = ICP_QAT_HW_COMPRESSION_FILE_TYPE_0; - pCompConfig->val = ICP_QAT_HW_COMPRESSION_CONFIG_BUILD( - dir, enableDmm, algo, depth, filetype); + pCompConfig->lower_val = ICP_QAT_HW_COMPRESSION_CONFIG_BUILD( + dir, dmm, algo, depth, filetype); - pCompConfig->reserved = 0; + /* Upper 32-bits of the configuration word do not need to be + * configured with legacy devices. + */ + pCompConfig->upper_val = 0; +} + +static void +dcCompHwBlockPopulateGen4(sal_compression_service_t *pService, + dc_session_desc_t *pSessionDesc, + icp_qat_hw_compression_config_t *pCompConfig, + dc_request_dir_t compDecomp) +{ + /* Compression related */ + if (DC_COMPRESSION_REQUEST == compDecomp) { + icp_qat_hw_comp_20_config_csr_upper_t hw_comp_upper_csr; + icp_qat_hw_comp_20_config_csr_lower_t hw_comp_lower_csr; + + memset(&hw_comp_upper_csr, 0, sizeof hw_comp_upper_csr); + memset(&hw_comp_lower_csr, 0, sizeof hw_comp_lower_csr); + + /* Disable Literal + Length Limit Block Drop by default and + * enable it only for dynamic deflate compression. + */ + hw_comp_lower_csr.lllbd = + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED; + + switch (pSessionDesc->compType) { + case CPA_DC_DEFLATE: + /* DEFLATE algorithm settings */ + hw_comp_lower_csr.skip_ctrl = + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; + + if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { + hw_comp_lower_csr.algo = + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; + } else /* Static DEFLATE */ + { + hw_comp_lower_csr.algo = + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE; + hw_comp_upper_csr.scb_ctrl = + ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE; + } + + if (CPA_DC_STATEFUL == pSessionDesc->sessState) { + hw_comp_upper_csr.som_ctrl = + ICP_QAT_HW_COMP_20_SOM_CONTROL_REPLAY_MODE; + } + break; + default: + QAT_UTILS_LOG("Compression algorithm not supported\n"); + break; + } + /* Set the search depth */ + switch (pSessionDesc->compLevel) { + case CPA_DC_L1: + case CPA_DC_L2: + case CPA_DC_L3: + case CPA_DC_L4: + case CPA_DC_L5: + hw_comp_lower_csr.sd = + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; + hw_comp_lower_csr.hash_col = + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_DONT_ALLOW; + break; + case CPA_DC_L6: + case CPA_DC_L7: + case CPA_DC_L8: + hw_comp_lower_csr.sd = + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6; + break; + case CPA_DC_L9: + hw_comp_lower_csr.sd = + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9; + break; + default: + hw_comp_lower_csr.sd = pService->comp_device_data + .highestHwCompressionDepth; + if ((CPA_DC_HT_FULL_DYNAMIC == + pSessionDesc->huffType) && + (CPA_DC_DEFLATE == pSessionDesc->compType)) { + /* Enable Literal + Length Limit Block Drop + * with dynamic deflate compression when + * highest compression levels are selected. + */ + hw_comp_lower_csr.lllbd = + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; + } + break; + } + /* Same for all algorithms */ + hw_comp_lower_csr.abd = ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED; + hw_comp_lower_csr.hash_update = + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; + hw_comp_lower_csr.edmm = + (CPA_TRUE == pService->comp_device_data.enableDmm) ? + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED : + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED; + + /* Hard-coded HW-specific values */ + hw_comp_upper_csr.nice = + ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; + hw_comp_upper_csr.lazy = + ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; + + pCompConfig->upper_val = + ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); + + pCompConfig->lower_val = + ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + } else /* Decompress */ + { + icp_qat_hw_decomp_20_config_csr_lower_t hw_decomp_lower_csr; + + memset(&hw_decomp_lower_csr, 0, sizeof hw_decomp_lower_csr); + + /* Set the algorithm */ + if (CPA_DC_DEFLATE == pSessionDesc->compType) { + hw_decomp_lower_csr.algo = + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; + } else { + QAT_UTILS_LOG("Algorithm not supported for " + "Decompression\n"); + } + + pCompConfig->upper_val = 0; + pCompConfig->lower_val = + ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER( + hw_decomp_lower_csr); + } } /** @@ -268,10 +412,19 @@ dcCompContentDescPopulate(sal_compression_service_t *pService, pCompControlBlock->resrvd = 0; /* Populate Compression Hardware Setup Block */ - dcCompHwBlockPopulate(pSessionDesc, - pCompConfig, - compDecomp, - pService->comp_device_data.enableDmm); + if (isDcGen4x(pService)) { + dcCompHwBlockPopulateGen4(pService, + pSessionDesc, + pCompConfig, + compDecomp); + } else if (isDcGen2x(pService)) { + dcCompHwBlockPopulate(pService, + pSessionDesc, + pCompConfig, + compDecomp); + } else { + QAT_UTILS_LOG("Invalid QAT generation value\n"); + } } /** @@ -286,7 +439,7 @@ dcCompContentDescPopulate(sal_compression_service_t *pService, * @param[in] nextSlice Next slice * *****************************************************************************/ -static void +void dcTransContentDescPopulate(icp_qat_fw_comp_req_t *pMsg, icp_qat_fw_slice_t nextSlice) { @@ -333,14 +486,70 @@ dcGetContextSize(CpaInstanceHandle dcInstance, *pContextSize = 0; if ((CPA_DC_STATEFUL == pSessionData->sessState) && - (CPA_DC_DEFLATE == pSessionData->compType) && (CPA_DC_DIR_COMPRESS != pSessionData->sessDirection)) { - *pContextSize = - pCompService->comp_device_data.inflateContextSize; + switch (pSessionData->compType) { + case CPA_DC_DEFLATE: + *pContextSize = + pCompService->comp_device_data.inflateContextSize; + break; + default: + QAT_UTILS_LOG("Invalid compression algorithm."); + return CPA_STATUS_FAIL; + } } return CPA_STATUS_SUCCESS; } +CpaStatus +dcGetCompressCommandId(sal_compression_service_t *pService, + CpaDcSessionSetupData *pSessionData, + Cpa8U *pDcCmdId) +{ + CpaStatus status = CPA_STATUS_SUCCESS; + LAC_CHECK_NULL_PARAM(pService); + LAC_CHECK_NULL_PARAM(pSessionData); + LAC_CHECK_NULL_PARAM(pDcCmdId); + + switch (pSessionData->compType) { + case CPA_DC_DEFLATE: + *pDcCmdId = (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) ? + ICP_QAT_FW_COMP_CMD_DYNAMIC : + ICP_QAT_FW_COMP_CMD_STATIC; + break; + default: + QAT_UTILS_LOG("Algorithm not supported for " + "compression\n"); + status = CPA_STATUS_UNSUPPORTED; + break; + } + + return status; +} + +CpaStatus +dcGetDecompressCommandId(sal_compression_service_t *pService, + CpaDcSessionSetupData *pSessionData, + Cpa8U *pDcCmdId) +{ + CpaStatus status = CPA_STATUS_SUCCESS; + LAC_CHECK_NULL_PARAM(pService); + LAC_CHECK_NULL_PARAM(pSessionData); + LAC_CHECK_NULL_PARAM(pDcCmdId); + + switch (pSessionData->compType) { + case CPA_DC_DEFLATE: + *pDcCmdId = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + QAT_UTILS_LOG("Algorithm not supported for " + "decompression\n"); + status = CPA_STATUS_UNSUPPORTED; + break; + } + + return status; +} + CpaStatus dcInitSession(CpaInstanceHandle dcInstance, CpaDcSessionHandle pSessionHandle, @@ -394,7 +603,17 @@ dcInitSession(CpaInstanceHandle dcInstance, return CPA_STATUS_UNSUPPORTED; } - if (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) { + /* Check for Gen4 and stateful, return error if both exist */ + if ((isDcGen4x(pService)) && + (CPA_DC_STATEFUL == pSessionData->sessState && + CPA_DC_DIR_DECOMPRESS != pSessionData->sessDirection)) { + QAT_UTILS_LOG("Stateful sessions are not supported for " + "compression direction"); + return CPA_STATUS_UNSUPPORTED; + } + + if ((isDcGen2x(pService)) && + (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType)) { /* Test if DRAM is available for the intermediate buffers */ if ((NULL == pService->pInterBuffPtrsArray) && (0 == pService->pInterBuffPtrsArrayPhyAddr)) { @@ -404,7 +623,8 @@ dcInitSession(CpaInstanceHandle dcInstance, pSessionData->huffType = CPA_DC_HT_STATIC; } else { QAT_UTILS_LOG( - "No buffer defined for this instance - see cpaDcStartInstance.\n"); + "No buffer defined for this instance - " + "see cpaDcStartInstance.\n"); return CPA_STATUS_RESOURCE; } } @@ -541,7 +761,8 @@ dcInitSession(CpaInstanceHandle dcInstance, pSessionDesc->pendingDpStatelessCbCount = 0; if (CPA_DC_DIR_DECOMPRESS != pSessionData->sessDirection) { - if (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) { + if ((isDcGen2x(pService)) && + CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) { /* Populate the compression section of the content * descriptor */ dcCompContentDescPopulate(pService, @@ -607,17 +828,29 @@ dcInitSession(CpaInstanceHandle dcInstance, pDataIntegrityCrcs = &pSessionDesc->dataIntegrityCrcs; pDataIntegrityCrcs->crc32 = 0; pDataIntegrityCrcs->adler32 = 1; - pDataIntegrityCrcs->oCrc32Cpr = DC_INVALID_CRC; - pDataIntegrityCrcs->iCrc32Cpr = DC_INVALID_CRC; - pDataIntegrityCrcs->oCrc32Xlt = DC_INVALID_CRC; - pDataIntegrityCrcs->iCrc32Xlt = DC_INVALID_CRC; - pDataIntegrityCrcs->xorFlags = DC_XOR_FLAGS_DEFAULT; - pDataIntegrityCrcs->crcPoly = DC_CRC_POLY_DEFAULT; - pDataIntegrityCrcs->xorOut = DC_XOR_OUT_DEFAULT; - /* Initialise seed checksums */ - pSessionDesc->seedSwCrc.swCrcI = 0; - pSessionDesc->seedSwCrc.swCrcO = 0; + if (isDcGen2x(pService)) { + pDataIntegrityCrcs->oCrc32Cpr = DC_INVALID_CRC; + pDataIntegrityCrcs->iCrc32Cpr = DC_INVALID_CRC; + pDataIntegrityCrcs->oCrc32Xlt = DC_INVALID_CRC; + pDataIntegrityCrcs->iCrc32Xlt = DC_INVALID_CRC; + pDataIntegrityCrcs->xorFlags = DC_XOR_FLAGS_DEFAULT; + pDataIntegrityCrcs->crcPoly = DC_CRC_POLY_DEFAULT; + pDataIntegrityCrcs->xorOut = DC_XOR_OUT_DEFAULT; + } else { + pDataIntegrityCrcs->iCrc64Cpr = DC_INVALID_CRC; + pDataIntegrityCrcs->oCrc64Cpr = DC_INVALID_CRC; + pDataIntegrityCrcs->iCrc64Xlt = DC_INVALID_CRC; + pDataIntegrityCrcs->oCrc64Xlt = DC_INVALID_CRC; + pDataIntegrityCrcs->crc64Poly = DC_CRC64_POLY_DEFAULT; + pDataIntegrityCrcs->xor64Out = DC_XOR64_OUT_DEFAULT; + } + + /* Initialise seed checksums. + * It initializes swCrc32I, swCrc32O, too(union). + */ + pSessionDesc->seedSwCrc.swCrc64I = 0; + pSessionDesc->seedSwCrc.swCrc64O = 0; /* Populate the cmdFlags */ switch (pSessionDesc->autoSelectBestHuffmanTree) { @@ -646,6 +879,7 @@ dcInitSession(CpaInstanceHandle dcInstance, ICP_QAT_FW_COMP_BFINAL, ICP_QAT_FW_COMP_NO_CNV, ICP_QAT_FW_COMP_NO_CNV_RECOVERY, + ICP_QAT_FW_COMP_NO_CNV_DFX, ICP_QAT_FW_COMP_CRC_MODE_LEGACY); cmdFlags = @@ -656,11 +890,16 @@ dcInitSession(CpaInstanceHandle dcInstance, secureRam); if (CPA_DC_DIR_DECOMPRESS != pSessionData->sessDirection) { - if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { - dcCmdId = (icp_qat_fw_la_cmd_id_t)( - ICP_QAT_FW_COMP_CMD_DYNAMIC); - } + status = dcGetCompressCommandId(pService, + pSessionData, + (Cpa8U *)&dcCmdId); + if (CPA_STATUS_SUCCESS != status) { + QAT_UTILS_LOG( + "Couldn't get compress command ID for current " + "session data."); + return status; + } pReqCache = &(pSessionDesc->reqCacheComp); pReqCache->comp_pars.req_par_flags = rpCmdFlags; pReqCache->comp_pars.crc.legacy.initial_adler = 1; @@ -675,8 +914,16 @@ dcInitSession(CpaInstanceHandle dcInstance, } if (CPA_DC_DIR_COMPRESS != pSessionData->sessDirection) { - dcCmdId = - (icp_qat_fw_la_cmd_id_t)(ICP_QAT_FW_COMP_CMD_DECOMPRESS); + status = dcGetDecompressCommandId(pService, + pSessionData, + (Cpa8U *)&dcCmdId); + if (CPA_STATUS_SUCCESS != status) { + QAT_UTILS_LOG( + "Couldn't get decompress command ID for current " + "session data."); + + return status; + } pReqCache = &(pSessionDesc->reqCacheDecomp); pReqCache->comp_pars.req_par_flags = rpCmdFlags; pReqCache->comp_pars.crc.legacy.initial_adler = 1; @@ -730,10 +977,14 @@ cpaDcResetSession(const CpaInstanceHandle dcInstance, { CpaStatus status = CPA_STATUS_SUCCESS; CpaInstanceHandle insHandle = NULL; + sal_compression_service_t *pService = NULL; dc_session_desc_t *pSessionDesc = NULL; Cpa64U numPendingStateless = 0; Cpa64U numPendingStateful = 0; icp_comms_trans_handle trans_handle = NULL; + dc_integrity_crc_fw_t *pDataIntegrityCrcs = NULL; + dc_sw_checksums_t *pSwCrcs = NULL; + LAC_CHECK_NULL_PARAM(pSessionHandle); pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); LAC_CHECK_NULL_PARAM(pSessionDesc); @@ -752,7 +1003,7 @@ cpaDcResetSession(const CpaInstanceHandle dcInstance, /* Check if SAL is running otherwise return an error */ SAL_RUNNING_CHECK(insHandle); if (CPA_TRUE == pSessionDesc->isDcDp) { - trans_handle = ((sal_compression_service_t *)dcInstance) + trans_handle = ((sal_compression_service_t *)insHandle) ->trans_handle_compression_tx; if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { /* Process the remaining messages on the ring */ @@ -798,7 +1049,32 @@ cpaDcResetSession(const CpaInstanceHandle dcInstance, } else { pSessionDesc->previousChecksum = 0; } + pSessionDesc->cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX; + + /* Reset integrity CRCs to default parameters. */ + pDataIntegrityCrcs = &pSessionDesc->dataIntegrityCrcs; + memset(pDataIntegrityCrcs, 0, sizeof(dc_integrity_crc_fw_t)); + pDataIntegrityCrcs->adler32 = 1; + + pService = (sal_compression_service_t *)insHandle; + if (isDcGen2x(pService)) { + pDataIntegrityCrcs->xorFlags = DC_XOR_FLAGS_DEFAULT; + pDataIntegrityCrcs->crcPoly = DC_CRC_POLY_DEFAULT; + pDataIntegrityCrcs->xorOut = DC_XOR_OUT_DEFAULT; + } else { + pDataIntegrityCrcs->crc64Poly = DC_CRC64_POLY_DEFAULT; + pDataIntegrityCrcs->xor64Out = DC_XOR64_OUT_DEFAULT; + } + + /* Reset seed SW checksums. */ + pSwCrcs = &pSessionDesc->seedSwCrc; + memset(pSwCrcs, 0, sizeof(dc_sw_checksums_t)); + + /* Reset integrity SW checksums. */ + pSwCrcs = &pSessionDesc->integritySwCrc; + memset(pSwCrcs, 0, sizeof(dc_sw_checksums_t)); } + /* Reset the pending callback counters */ qatUtilsAtomicSet(0, &pSessionDesc->pendingStatelessCbCount); qatUtilsAtomicSet(0, &pSessionDesc->pendingStatefulCbCount); @@ -886,12 +1162,7 @@ cpaDcRemoveSession(const CpaInstanceHandle dcInstance, } if ((CPA_DC_STATEFUL == pSessionDesc->sessState) && (CPA_STATUS_SUCCESS == status)) { - if (CPA_STATUS_SUCCESS != - LAC_SPINLOCK_DESTROY( - &(pSessionDesc->sessionLock))) { - QAT_UTILS_LOG( - "Failed to destory session lock.\n"); - } + LAC_SPINLOCK_DESTROY(&(pSessionDesc->sessionLock)); } } @@ -955,3 +1226,33 @@ cpaDcGetSessionSize(CpaInstanceHandle dcInstance, pSessionSize, pContextSize); } + +CpaStatus +dcSetCnvError(CpaInstanceHandle dcInstance, CpaDcSessionHandle pSessionHandle) +{ + LAC_CHECK_NULL_PARAM(pSessionHandle); + + dc_session_desc_t *pSessionDesc = NULL; + CpaInstanceHandle insHandle = NULL; + sal_compression_service_t *pService = NULL; + + if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) { + insHandle = dcGetFirstHandle(); + } else { + insHandle = dcInstance; + } + + pService = (sal_compression_service_t *)insHandle; + + if (isDcGen2x(pService)) { + QAT_UTILS_LOG("Unsupported compression feature.\n"); + return CPA_STATUS_UNSUPPORTED; + } + pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); + + LAC_CHECK_NULL_PARAM(pSessionDesc); + + pSessionDesc->cnvErrorInjection = ICP_QAT_FW_COMP_CNV_DFX; + + return CPA_STATUS_SUCCESS; +} diff --git a/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h b/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h index 0a6ef7191704..72cb08e4e128 100644 --- a/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h +++ b/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h @@ -26,6 +26,8 @@ * the management of skid buffers in the firmware */ #define DC_DEST_BUFFER_DYN_MIN_SIZE (128) #define DC_DEST_BUFFER_STA_MIN_SIZE (64) +#define DC_DEST_BUFFER_DYN_MIN_SIZE_GEN4 (512) +#define DC_DEST_BUFFER_STA_MIN_SIZE_GEN4 (1024) /* C62x and C3xxx pcie rev0 devices require an additional 32bytes */ #define DC_DEST_BUFFER_STA_ADDITIONAL_SIZE (32) @@ -93,8 +95,10 @@ * those are used to inform hardware of specifying CRC parameters to be used * when calculating CRCs */ #define DC_CRC_POLY_DEFAULT 0x04c11db7 +#define DC_CRC64_POLY_DEFAULT 0x42f0e1eba9ea3693ULL #define DC_XOR_FLAGS_DEFAULT 0xe0000 #define DC_XOR_OUT_DEFAULT 0xffffffff +#define DC_XOR64_OUT_DEFAULT 0x0ULL #define DC_INVALID_CRC 0x0 /** @@ -147,6 +151,13 @@ typedef struct dc_compression_cookie_s { /**< virtual userspace ptr to source SGL */ CpaBufferList *pUserDestBuff; /**< virtual userspace ptr to destination SGL */ + CpaDcCallbackFn pCbFunc; + /**< Callback function defined for the traditional sessionless API */ + CpaDcChecksum checksumType; + /**< Type of checksum */ + dc_integrity_crc_fw_t dataIntegrityCrcs; + /**< Data integrity table */ + } dc_compression_cookie_t; /** @@ -165,6 +176,9 @@ typedef struct dc_compression_cookie_s { *****************************************************************************/ void dcCompression_ProcessCallback(void *pRespMsg); +CpaStatus dcCheckOpData(sal_compression_service_t *pService, + CpaDcOpData *pOpData); + /** ***************************************************************************** * @ingroup Dc_DataCompression diff --git a/sys/dev/qat/qat_api/common/compression/include/dc_session.h b/sys/dev/qat/qat_api/common/compression/include/dc_session.h index 5a4961fadd60..041c60e5845c 100644 --- a/sys/dev/qat/qat_api/common/compression/include/dc_session.h +++ b/sys/dev/qat/qat_api/common/compression/include/dc_session.h @@ -17,6 +17,7 @@ #include "cpa_dc_dp.h" #include "icp_qat_fw_comp.h" #include "sal_qat_cmn_msg.h" +#include "sal_types_compression.h" /* Maximum number of intermediate buffers SGLs for devices * with a maximum of 6 compression slices */ @@ -35,6 +36,7 @@ /* Size of the history window. * Base 2 logarithm of maximum window size minus 8 */ +#define DC_4K_WINDOW_SIZE (4) #define DC_8K_WINDOW_SIZE (5) #define DC_16K_WINDOW_SIZE (6) #define DC_32K_WINDOW_SIZE (7) @@ -95,35 +97,81 @@ typedef struct dc_integrity_crc_fw_s { /* CRC32 checksum returned for compressed data */ Cpa32U adler32; /* ADLER32 checksum returned for compressed data */ - Cpa32U oCrc32Cpr; - /* CRC32 checksum returned for data output by compression accelerator */ - Cpa32U iCrc32Cpr; - /* CRC32 checksum returned for input data to compression accelerator */ - Cpa32U oCrc32Xlt; - /* CRC32 checksum returned for data output by translator accelerator */ - Cpa32U iCrc32Xlt; - /* CRC32 checksum returned for input data to translator accelerator */ - Cpa32U xorFlags; - /* Initialise transactor pCRC controls in state register */ - Cpa32U crcPoly; - /* CRC32 polynomial used by hardware */ - Cpa32U xorOut; - /* CRC32 from XOR stage (Input CRC is xor'ed with value in the state) */ - Cpa32U deflateBlockType; - /* Bit 1 - Bit 0 - * 0 0 -> RAW DATA + Deflate header. - * This will not produced any CRC check because - * the output will not come from the slices. - * It will be a simple copy from input to output - * buffers list. - * 0 1 -> Static deflate block type - * 1 0 -> Dynamic deflate block type - * 1 1 -> Invalid type */ + + union { + struct { + Cpa32U oCrc32Cpr; + /* CRC32 checksum returned for data output by + * compression accelerator */ + Cpa32U iCrc32Cpr; + /* CRC32 checksum returned for input data to compression + * accelerator + */ + Cpa32U oCrc32Xlt; + /* CRC32 checksum returned for data output by translator + * accelerator + */ + Cpa32U iCrc32Xlt; + /* CRC32 checksum returned for input data to translator + * accelerator + */ + Cpa32U xorFlags; + /* Initialise transactor pCRC controls in state register + */ + Cpa32U crcPoly; + /* CRC32 polynomial used by hardware */ + Cpa32U xorOut; + /* CRC32 from XOR stage (Input CRC is xor'ed with value + * in the state) */ + Cpa32U deflateBlockType; + /* Bit 1 - Bit 0 + * 0 0 -> RAW DATA + Deflate header. + * This will not produced any CRC check + * because the output will not come + * from the slices. It will be a simple + * copy from input to output buffer + * list. 0 1 -> Static deflate block type 1 0 -> + * Dynamic deflate block type 1 1 -> Invalid type + */ + }; + + struct { + Cpa64U iCrc64Cpr; + /* CRC64 checksum returned for input data to compression + * accelerator + */ + Cpa64U oCrc64Cpr; + /* CRC64 checksum returned for data output by + * compression accelerator */ + Cpa64U iCrc64Xlt; + /* CRC64 checksum returned for input data to translator + * accelerator + */ + Cpa64U oCrc64Xlt; + /* CRC64 checksum returned for data output by translator + * accelerator + */ + Cpa64U crc64Poly; + /* CRC64 polynomial used by hardware */ + Cpa64U xor64Out; + /* CRC64 from XOR stage (Input CRC is xor'ed with value + * in the state) */ + }; + }; } dc_integrity_crc_fw_t; typedef struct dc_sw_checksums_s { - Cpa32U swCrcI; - Cpa32U swCrcO; + union { + struct { + Cpa32U swCrc32I; + Cpa32U swCrc32O; + }; + + struct { + Cpa64U swCrc64I; + Cpa64U swCrc64O; + }; + }; } dc_sw_checksums_t; /* Session descriptor structure for compression */ @@ -211,6 +259,8 @@ typedef struct dc_session_desc_s { dc_sw_checksums_t seedSwCrc; /* Driver calculated integrity software CRC */ dc_sw_checksums_t integritySwCrc; + /* Flag to disable or enable CnV Error Injection mechanism */ + CpaBoolean cnvErrorInjection; } dc_session_desc_t; /** @@ -275,4 +325,106 @@ CpaStatus dcGetSessionSize(CpaInstanceHandle dcInstance, Cpa32U *pSessionSize, Cpa32U *pContextSize); +/** + ***************************************************************************** + * @ingroup Dc_DataCompression + * Set the cnvErrorInjection flag in session descriptor + * + * @description + * This function enables the CnVError injection for the session + * passed in. All Compression requests sent within the session + * are injected with CnV errors. This error injection is for the + * duration of the session. Resetting the session results in + * setting being cleared. CnV error injection does not apply to + * Data Plane API. + * + * @param[in] dcInstance Instance Handle + * @param[in] pSessionHandle Pointer to a session handle + * + * @retval CPA_STATUS_SUCCESS Function executed successfully + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in + * @retval CPA_STATUS_UNSUPPORTED Unsupported feature + *****************************************************************************/ +CpaStatus dcSetCnvError(CpaInstanceHandle dcInstance, + CpaDcSessionHandle pSessionHandle); + +/** + ***************************************************************************** + * @ingroup Dc_DataCompression + * Check that pSessionData is valid + * + * @description + * Check that all the parameters defined in the pSessionData are valid + * + * @param[in] pSessionData Pointer to a user instantiated structure + * containing session data + * + * @retval CPA_STATUS_SUCCESS Function executed successfully + * @retval CPA_STATUS_FAIL Function failed to find device + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in + * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature + * + *****************************************************************************/ +CpaStatus dcCheckSessionData(const CpaDcSessionSetupData *pSessionData, + CpaInstanceHandle dcInstance); + +/** + ***************************************************************************** + * @ingroup Dc_DataCompression + * Get the compression command id for the given session setup data. + * + * @description + * This function will get the compression command id based on parameters + * passed in the given session setup data. + * + * @param[in] pService Pointer to the service + * @param[in] pSessionData Pointer to a user instantiated + * structure containing session data + * @param[out] pDcCmdId Pointer to the command id + * + * @retval CPA_STATUS_SUCCESS Function executed successfully + * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature + * + *****************************************************************************/ +CpaStatus dcGetCompressCommandId(sal_compression_service_t *pService, + CpaDcSessionSetupData *pSessionData, + Cpa8U *pDcCmdId); + +/** + ***************************************************************************** + * @ingroup Dc_DataCompression + * Get the decompression command id for the given session setup data. + * + * @description + * This function will get the decompression command id based on parameters + * passed in the given session setup data. + * + * @param[in] pService Pointer to the service + * @param[in] pSessionData Pointer to a user instantiated + * structure containing session data + * @param[out] pDcCmdId Pointer to the command id + * + * @retval CPA_STATUS_SUCCESS Function executed successfully + * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature + * + *****************************************************************************/ +CpaStatus dcGetDecompressCommandId(sal_compression_service_t *pService, + CpaDcSessionSetupData *pSessionData, + Cpa8U *pDcCmdId); + +/** + ***************************************************************************** + * @ingroup Dc_DataCompression + * Populate the translator content descriptor + * + * @description + * This function will populate the translator content descriptor + * + * @param[out] pMsg Pointer to the compression message + * @param[in] nextSlice Next slice + * + *****************************************************************************/ +void dcTransContentDescPopulate(icp_qat_fw_comp_req_t *pMsg, + icp_qat_fw_slice_t nextSlice); + #endif /* DC_SESSION_H */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h index 6ae3c51e7766..76d0f4f08bb5 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h @@ -107,6 +107,21 @@ * Include private header files ******************************************************************************* */ +/** +***************************************************************************** +* @ingroup LacSym +* Spc state +* +* @description +* This enum is used to indicate the Spc state. +* +*****************************************************************************/ +typedef enum lac_single_pass_state_e { + NON_SPC, /* Algorithms other than CHACHA-POLY and AES-GCM */ + LIKELY_SPC, /* AES-GCM - Likely to handle it as single pass */ + SPC /* CHACHA-POLY and AES-GCM */ +} lac_single_pass_state_t; + /** ******************************************************************************* * @ingroup LacSym_Session @@ -214,15 +229,17 @@ typedef struct lac_session_desc_s { /**< Flag indicating whether the SymConstantsTable can be used or not */ CpaBoolean useOptimisedContentDesc : 1; /**< Flag indicating whether to use the optimised CD or not */ + CpaBoolean isPartialSupported : 1; + /**< Flag indicating whether symOperation support partial packet */ + CpaBoolean useStatefulSha3ContentDesc : 1; + /**< Flag indicating whether to use the stateful SHA3 CD or not */ icp_qat_la_bulk_req_hdr_t shramReqCacheHdr; icp_qat_fw_la_key_gen_common_t shramReqCacheMid; icp_qat_la_bulk_req_ftr_t shramReqCacheFtr; /**< Alternative pre-built request (header, mid & footer) * for use with symConstantsTable. */ - CpaBoolean isPartialSupported : 1; - /**< Flag indicating whether symOperation support partial packet */ - CpaBoolean isSinglePass : 1; - /**< Flag indicating whether symOperation is single pass operation */ + lac_single_pass_state_t singlePassState; + /**< Flag indicating whether symOperation support single pass */ icp_qat_fw_serv_specif_flags laCmdFlags; /**< Common request - Service specific flags type */ icp_qat_fw_comn_flags cmnRequestFlags; @@ -236,6 +253,23 @@ typedef struct lac_session_desc_s { /**< Hash Mode for the qat slices. Not to be confused with QA-API * hashMode */ + Cpa32U cipherSliceType; + /**< Cipher slice type to be used, set at init session time */ + Cpa8U cipherAesXtsKey1Forward[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< Cached AES XTS Forward key + * For CPM2.0 AES XTS key convertion need to be done in SW. + * Because use can update session direction at any time, + * also forward key needs to be cached + */ + Cpa8U cipherAesXtsKey1Reverse[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< AES XTS Reverse key + * For CPM2.0 AES XTS key convertion need to be done in SW. + * Reverse key always will be calcilated at session setup time and + * cached to be used when needed */ + Cpa8U cipherAesXtsKey2[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< For AES XTS session need to store Key2 value in order to generate + * tweak + */ void *writeRingMsgFunc; /**< function which will be called to write ring message */ Cpa32U aadLenInBytes; @@ -325,8 +359,7 @@ typedef struct lac_session_desc_d1_s { * a decrypt operation. */ CpaCySymPacketType partialState; /**< state of the partial packet. This can be written to by the perform - * because the SpinLock pPartialInFlightSpinlock guarantees that that - * the + * because the SpinLock pPartialInFlightSpinlock guarantees that the * state is accessible in only one place at a time. */ icp_qat_la_bulk_req_hdr_t reqCacheHdr; icp_qat_fw_la_key_gen_common_t reqCacheMid; @@ -382,15 +415,17 @@ typedef struct lac_session_desc_d1_s { /**< Flag indicating whether the SymConstantsTable can be used or not */ CpaBoolean useOptimisedContentDesc : 1; /**< Flag indicating whether to use the optimised CD or not */ + CpaBoolean isPartialSupported : 1; + /**< Flag indicating whether symOperation support partial packet */ + CpaBoolean useStatefulSha3ContentDesc : 1; + /**< Flag indicating whether to use the stateful SHA3 CD or not */ icp_qat_la_bulk_req_hdr_t shramReqCacheHdr; icp_qat_fw_la_key_gen_common_t shramReqCacheMid; icp_qat_la_bulk_req_ftr_t shramReqCacheFtr; /**< Alternative pre-built request (header, mid & footer) * for use with symConstantsTable. */ - CpaBoolean isPartialSupported : 1; - /**< Flag indicating whether symOperation support partial packet */ - CpaBoolean isSinglePass : 1; - /**< Flag indicating whether symOperation is single pass operation */ + lac_single_pass_state_t singlePassState; + /**< Flag indicating whether symOperation support single pass */ icp_qat_fw_serv_specif_flags laCmdFlags; /**< Common request - Service specific flags type */ icp_qat_fw_comn_flags cmnRequestFlags; @@ -404,6 +439,23 @@ typedef struct lac_session_desc_d1_s { /**< Hash Mode for the qat slices. Not to be confused with QA-API * hashMode */ + Cpa32U cipherSliceType; + /**< Cipher slice type to be used, set at init session time */ + Cpa8U cipherAesXtsKey1Forward[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< Cached AES XTS Forward key + * For CPM2.0 AES XTS key convertion need to be done in SW. + * Because use can update session direction at any time, + * also forward key needs to be cached + */ + Cpa8U cipherAesXtsKey1Reverse[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< AES XTS Reverse key + * For CPM2.0 AES XTS key convertion need to be done in SW. + * Reverse key always will be calcilated at session setup time and + * cached to be used when needed */ + Cpa8U cipherAesXtsKey2[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< For AES XTS session need to store Key2 value in order to generate + * tweak + */ void *writeRingMsgFunc; /**< function which will be called to write ring message */ } lac_session_desc_d1_t; @@ -444,8 +496,8 @@ typedef struct lac_session_desc_d2_s { /**< info on the hash state prefix buffer */ CpaCySymHashAlgorithm hashAlgorithm; /**< hash algorithm */ - Cpa32U authKeyLenInBytes; - /**< Authentication key length in bytes */ + Cpa32U authKeyLenInBytes; + /**< Authentication key length in bytes */ CpaCySymHashMode hashMode; /**< Mode of the hash operation. plain, auth or nested */ Cpa32U hashResultSize; @@ -459,8 +511,7 @@ typedef struct lac_session_desc_d2_s { * a decrypt operation. */ CpaCySymPacketType partialState; /**< state of the partial packet. This can be written to by the perform - * because the SpinLock pPartialInFlightSpinlock guarantees that that - * the + * because the SpinLock pPartialInFlightSpinlock guarantees that the * state is accessible in only one place at a time. */ icp_qat_la_bulk_req_hdr_t reqCacheHdr; icp_qat_fw_la_key_gen_common_t reqCacheMid; @@ -516,15 +567,17 @@ typedef struct lac_session_desc_d2_s { /**< Flag indicating whether the SymConstantsTable can be used or not */ CpaBoolean useOptimisedContentDesc : 1; /**< Flag indicating whether to use the optimised CD or not */ + CpaBoolean isPartialSupported : 1; + /**< Flag indicating whether symOperation support partial packet */ + CpaBoolean useStatefulSha3ContentDesc : 1; + /**< Flag indicating whether to use the stateful SHA3 CD or not */ icp_qat_la_bulk_req_hdr_t shramReqCacheHdr; icp_qat_fw_la_key_gen_common_t shramReqCacheMid; icp_qat_la_bulk_req_ftr_t shramReqCacheFtr; /**< Alternative pre-built request (header. mid & footer) * for use with symConstantsTable. */ - CpaBoolean isPartialSupported : 1; - /**< Flag indicating whether symOperation support partial packet */ - CpaBoolean isSinglePass : 1; - /**< Flag indicating whether symOperation is single pass operation */ + lac_single_pass_state_t singlePassState; + /**< Flag indicating whether symOperation support single pass */ icp_qat_fw_serv_specif_flags laCmdFlags; /**< Common request - Service specific flags type */ icp_qat_fw_comn_flags cmnRequestFlags; @@ -538,6 +591,23 @@ typedef struct lac_session_desc_d2_s { /**< Hash Mode for the qat slices. Not to be confused with QA-API * hashMode */ + Cpa32U cipherSliceType; + /**< Cipher slice type to be used, set at init session time */ + Cpa8U cipherAesXtsKey1Forward[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< Cached AES XTS Forward key + * For CPM2.0 AES XTS key convertion need to be done in SW. + * Because use can update session direction at any time, + * also forward key needs to be cached + */ + Cpa8U cipherAesXtsKey1Reverse[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< AES XTS Reverse key + * For CPM2.0 AES XTS key convertion need to be done in SW. + * Reverse key always will be calcilated at session setup time and + * cached to be used when needed */ + Cpa8U cipherAesXtsKey2[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH]; + /**< For AES XTS session need to store Key2 value in order to generate + * tweak + */ void *writeRingMsgFunc; /**< function which will be called to write ring message */ Cpa32U aadLenInBytes; diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h index 822e9ce03f94..0ae34333ec08 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h @@ -228,6 +228,7 @@ #include "lac_session.h" #include "lac_sym.h" +#include "lac_sal_types_crypto.h" /* * WARNING: There are no checks done on the parameters of the functions in @@ -253,8 +254,9 @@ * @retval CPA_STATUS_INVALID_PARAM Invalid parameter. * *****************************************************************************/ -CpaStatus LacCipher_SessionSetupDataCheck( - const CpaCySymCipherSetupData *pCipherSetupData); +CpaStatus +LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData, + Cpa32U capabilitiesMask); /** ******************************************************************************* @@ -309,4 +311,22 @@ CpaStatus LacCipher_PerformIvCheck(sal_service_t *pService, Cpa32U qatPacketType, Cpa8U **ppIvBuffer); +/** + ***************************************************************************** + * @ingroup LacCipher + * Return cipher slice type for given algorithm + * + * @description + * This function will check what cipher slice type should be used for given + * algorithms and CPM generation combination. + * Since CPM2.0 there is new UCS cipher slice available. + * + * @param[in] pService Pointer to service struct + * @param[in] cipherAlgorithm cipher algorithm + * @param[in] hashAlgorithm hash algorithm + * + *****************************************************************************/ +Cpa32U LacCipher_GetCipherSliceType(sal_crypto_service_t *pService, + CpaCySymCipherAlgorithm algorithm, + CpaCySymHashAlgorithm hash); #endif /* LAC_SYM_CIPHER_H */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h index 4eddb70420da..a7dc286f5d35 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h @@ -45,6 +45,14 @@ /* ARC4 256 bytes for key matrix, 2 for i and j and 6 bytes for padding */ #define LAC_CIPHER_ARC4_STATE_LEN_BYTES 264 +/* + * Constant values for CCM AAD buffer + */ +#define LAC_CIPHER_CCM_B0_SIZE 16 +#define LAC_CIPHER_CCM_ENCODED_AAD_LEN_SIZE 2 +#define LAC_CIPHER_CCM_AAD_OFFSET \ + (LAC_CIPHER_CCM_B0_SIZE + LAC_CIPHER_CCM_ENCODED_AAD_LEN_SIZE) + #define LAC_SYM_SNOW3G_CIPHER_CONFIG_FOR_HASH_SZ 40 /* Snow3g cipher config required for performing a Snow3g hash operation. * It contains 8 Bytes of config for hardware, 16 Bytes of Key and requires @@ -64,6 +72,9 @@ /* The IV length for AES F8 is 16 bytes */ #define LAC_CIPHER_AES_F8_IV_LENGTH 16 +/* The max key length for AES XTS 32 is bytes*/ +#define LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH 32 + /* For Snow3G UEA2, need to make sure last 8 Bytes of IV buffer are * zero. */ #define LAC_CIPHER_SNOW3G_UEA2_IV_BUFFER_ZERO_LENGTH 8 @@ -171,12 +182,17 @@ /* Macro to check if the Algorithm Mode is XTS */ #define LAC_CIPHER_IS_XTS_MODE(algo) (algo == CPA_CY_SYM_CIPHER_AES_XTS) +/* Macro to check if the accelerator has AES V2 capability */ +#define LAC_CIPHER_AES_V2(mask) ((mask)&ICP_ACCEL_CAPABILITIES_AES_V2) + /* Macro to check if the Algorithm is single pass */ #define LAC_CIPHER_IS_SPC(cipher, hash, mask) \ - ((LAC_CIPHER_IS_CHACHA(cipher) && (CPA_CY_SYM_HASH_POLY == hash) && \ - ((mask)&ICP_ACCEL_CAPABILITIES_CHACHA_POLY)) || \ - (LAC_CIPHER_IS_GCM(cipher) && ((CPA_CY_SYM_HASH_AES_GCM == hash) || \ - (CPA_CY_SYM_HASH_AES_GMAC == hash)) && \ - ((mask)&ICP_ACCEL_CAPABILITIES_AESGCM_SPC))) + (((mask)&ICP_ACCEL_CAPABILITIES_CHACHA_POLY && \ + LAC_CIPHER_IS_CHACHA(cipher) && (CPA_CY_SYM_HASH_POLY == hash)) || \ + (((mask)&ICP_ACCEL_CAPABILITIES_AESGCM_SPC) && \ + LAC_CIPHER_IS_GCM(cipher) && \ + ((CPA_CY_SYM_HASH_AES_GCM == hash) || \ + (CPA_CY_SYM_HASH_AES_GMAC == hash))) || \ + (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(mask))) #endif /* LAC_CIPHER_DEFS_H */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h index e95b0efb5b0e..ef035a930fb8 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h @@ -108,51 +108,26 @@ /* Constants for SHA3_384 algorithm */ #define LAC_HASH_SHA3_384_BLOCK_SIZE 104 /**< @ingroup LacHashDefs - * * SHA3_384 block size in bytes */ + * SHA3_384 block size in bytes */ #define LAC_HASH_SHA3_384_DIGEST_SIZE 48 /**< @ingroup LacHashDefs - * * SHA3_384 digest length in bytes */ + * SHA3_384 digest length in bytes */ #define LAC_HASH_SHA3_384_STATE_SIZE 48 /**< @ingroup LacHashDefs - * * SHA3_384 state size */ + * SHA3_384 state size */ /* Constants for SHA3_512 algorithm */ #define LAC_HASH_SHA3_512_BLOCK_SIZE 72 /**< @ingroup LacHashDefs - * * * SHA3_512 block size in bytes */ + * SHA3_512 block size in bytes */ #define LAC_HASH_SHA3_512_DIGEST_SIZE 64 /**< @ingroup LacHashDefs - * * * SHA3_512 digest length in bytes */ + * SHA3_512 digest length in bytes */ #define LAC_HASH_SHA3_512_STATE_SIZE 64 /**< @ingroup LacHashDefs - * * * SHA3_512 state size */ + * SHA3_512 state size */ -/* Constants for SHAKE_128 algorithm */ -#define LAC_HASH_SHAKE_128_BLOCK_SIZE 168 -/**< @ingroup LacHashDefs - * * * SHAKE_128 block size in bytes */ -#define LAC_HASH_SHAKE_128_DIGEST_SIZE 0xFFFFFFFF -/**< @ingroup LacHashDefs - * * * SHAKE_128 digest length in bytes ((2^32)-1)*/ - -/* Constants for SHAKE_256 algorithm */ -#define LAC_HASH_SHAKE_256_BLOCK_SIZE 136 -/**< @ingroup LacHashDefs - * * * SHAKE_256 block size in bytes */ -#define LAC_HASH_SHAKE_256_DIGEST_SIZE 0xFFFFFFFF -/**< @ingroup LacHashDefs - * * * SHAKE_256 digest length in bytes ((2^ 32)-1)*/ - -/* Constants for POLY algorithm */ -#define LAC_HASH_POLY_BLOCK_SIZE 64 -/**< @ingroup LacHashDefs - * POLY block size in bytes */ -#define LAC_HASH_POLY_DIGEST_SIZE 16 -/**< @ingroup LacHashDefs - * POLY digest length */ -#define LAC_HASH_POLY_STATE_SIZE 0 -/**< @ingroup LacHashDefs - * POLY state size */ +#define LAC_HASH_SHA3_STATEFUL_STATE_SIZE 200 /* Constants for SM3 algorithm */ #define LAC_HASH_SM3_BLOCK_SIZE 64 @@ -165,6 +140,17 @@ /**< @ingroup LacHashDefs * SM3 state size */ +/* Constants for POLY algorithm */ +#define LAC_HASH_POLY_BLOCK_SIZE 64 +/**< @ingroup LacHashDefs + * POLY block size in bytes */ +#define LAC_HASH_POLY_DIGEST_SIZE 16 +/**< @ingroup LacHashDefs + * POLY digest length */ +#define LAC_HASH_POLY_STATE_SIZE 0 +/**< @ingroup LacHashDefs + * POLY state size */ + /* Constants for XCBC precompute algorithm */ #define LAC_HASH_XCBC_PRECOMP_KEY_NUM 3 /**< @ingroup LacHashDefs @@ -285,7 +271,7 @@ * the size in an 8bit field */ #define LAC_MAX_HASH_STATE_STORAGE_SIZE \ - (sizeof(icp_qat_hw_auth_counter_t) + LAC_HASH_SHA512_STATE_SIZE) + (sizeof(icp_qat_hw_auth_counter_t) + LAC_HASH_SHA3_STATEFUL_STATE_SIZE) /**< Maximum size of the hash state storage section of the hash state prefix * buffer */ @@ -307,11 +293,8 @@ (algorithm == CPA_CY_SYM_HASH_SHA256) || \ (algorithm == CPA_CY_SYM_HASH_SHA384) || \ (algorithm == CPA_CY_SYM_HASH_SHA512) || \ - (algorithm == CPA_CY_SYM_HASH_SHA3_224) || \ - (algorithm == CPA_CY_SYM_HASH_SHA3_256) || \ - (algorithm == CPA_CY_SYM_HASH_SHA3_384) || \ - (algorithm == CPA_CY_SYM_HASH_SHA3_512) || \ - (algorithm == CPA_CY_SYM_HASH_SM3)) + (algorithm == CPA_CY_SYM_HASH_SM3)) || \ + (LAC_HASH_IS_SHA3(algorithm)) /**< @ingroup LacSymQatHash * Macro to detect if the hash algorithm is a HMAC algorithm */ @@ -341,4 +324,12 @@ * Nested. This applies to TLS. This is used to differentiate between * TLS and HMAC */ +#define LAC_HASH_IS_SHA3(algo) \ + ((algo == CPA_CY_SYM_HASH_SHA3_224) || \ + (algo == CPA_CY_SYM_HASH_SHA3_256) || \ + (algo == CPA_CY_SYM_HASH_SHA3_384) || \ + (algo == CPA_CY_SYM_HASH_SHA3_512)) +/**< @ingroup LacSymQatHash + * Macro to check if the hash algorithm is SHA3 */ + #endif /* LAC_SYM_HASH_DEFS_H */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h index af49764b6498..e0dbb0574d31 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h @@ -206,4 +206,43 @@ void LacSymQat_LaPacketCommandFlagSet(Cpa32U qatPacketType, void LacSymQat_LaSetDefaultFlags(icp_qat_fw_serv_specif_flags *laCmdFlags, CpaCySymOp symOp); +/** + ****************************************************************************** + * @ingroup LacSymQat + * + * + * @description + * this function defines whether the shared constants table can be used + * for a particular cipher and hash algorithm + * + * @param[in] ptr to session + + * @param[in] ptr to return offset into table for cipher config + + * @param[in] ptr to return offset into table for hash config + * + * @return CPA_TRUE if Constants table is available for use, CPA_FALSE if it's + * not. + * + *****************************************************************************/ +CpaBoolean LacSymQat_UseSymConstantsTable(lac_session_desc_t *pSession, + Cpa8U *cipherOffset, + Cpa8U *hashOffset); + +/** + ****************************************************************************** + * @ingroup LacSymQat + * + * + * @description + * this function calculates whether the optimized content descriptor can + * be used for a particular chained cipher and hash algorithm + * + * @param[in] ptr to session + * + * @return CPA_TRUE if optimized CD can be used, CPA_FALSE if it's not. + * + *****************************************************************************/ +CpaBoolean LacSymQat_UseOptimisedContentDesc(lac_session_desc_t *pSession); + #endif /* LAC_SYM_QAT_H */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h index 2829bca12a72..e292c3d6b587 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h @@ -42,9 +42,11 @@ #define LAC_SYM_QAT_CIPHER_NEXT_ID_BIT_OFFSET 24 #define LAC_SYM_QAT_CIPHER_CURR_ID_BIT_OFFSET 16 #define LAC_SYM_QAT_CIPHER_STATE_SIZE_BIT_OFFSET 8 -#define LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC 9 -#define LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC 2 -#define LAC_SYM_QAT_CIPHER_STATE_SIZE_SPC 48 +#define LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM 9 +#define LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM 8 +#define LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM 2 +#define LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE 48 + /** ****************************************************************************** * @ingroup LacSymQat_Cipher @@ -111,11 +113,13 @@ Cpa32U LacSymQat_CipherIvSizeBytesGet(CpaCySymCipherAlgorithm cipherAlgorithm); * @retval void * *****************************************************************************/ -CpaStatus LacSymQat_CipherRequestParamsPopulate(icp_qat_fw_la_bulk_req_t *pReq, - Cpa32U cipherOffsetInBytes, - Cpa32U cipherLenInBytes, - Cpa64U ivBufferPhysAddr, - Cpa8U *pIvBufferVirt); +CpaStatus +LacSymQat_CipherRequestParamsPopulate(lac_session_desc_t *pSessionDesc, + icp_qat_fw_la_bulk_req_t *pReq, + Cpa32U cipherOffsetInBytes, + Cpa32U cipherLenInBytes, + Cpa64U ivBufferPhysAddr, + Cpa8U *pIvBufferVirt); /** ****************************************************************************** @@ -194,6 +198,8 @@ void LacSymQat_CipherCtrlBlockInitialize(icp_qat_fw_la_bulk_req_t *pMsg); * @param[in] targetKeyLenInBytes cipher key length in bytes of selected * algorithm * + * @param[in] sliceType Cipher slice type to be used + * * @param[out] nextSlice SliceID for next control block * entry. This value is known only by * the calling component @@ -206,6 +212,7 @@ void LacSymQat_CipherCtrlBlockInitialize(icp_qat_fw_la_bulk_req_t *pMsg); void LacSymQat_CipherCtrlBlockWrite(icp_qat_la_bulk_req_ftr_t *pMsg, Cpa32U cipherAlgorithm, Cpa32U targetKeyLenInBytes, + Cpa32U sliceType, icp_qat_fw_slice_t nextSlice, Cpa8U cipherCfgOffsetInQuadWord); @@ -274,6 +281,8 @@ void LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession, * key length MUST match the key length * in the cipher setup data. * + * @param[in] sliceType Cipher slice type to be used + * * @param[in] pCipherHwBlock Pointer to the cipher hardware block * * @param[out] pCipherHwBlockSizeBytes Size in bytes of cipher setup block @@ -283,8 +292,10 @@ void LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession, * *****************************************************************************/ void LacSymQat_CipherHwBlockPopulateKeySetup( + lac_session_desc_t *pSessionDesc, const CpaCySymCipherSetupData *pCipherSetupData, Cpa32U targetKeyLenInBytes, + Cpa32U sliceType, const void *pCipherHwBlock, Cpa32U *pCipherHwBlockSizeBytes); diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_constants_table.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_constants_table.h new file mode 100644 index 000000000000..7adf660ab9c4 --- /dev/null +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_constants_table.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2022 Intel Corporation */ +/* $FreeBSD$ */ + +/** + ***************************************************************************** + * @file lac_sym_qat_constants_table.h + * + * @ingroup LacSymQat + * + * API to be used for the CySym constants table. + * + *****************************************************************************/ + +#ifndef LAC_SYM_QAT_CONSTANTS_TABLE_H +#define LAC_SYM_QAT_CONSTANTS_TABLE_H + +#include "cpa.h" +#include "icp_qat_fw_la.h" + +typedef struct lac_sym_qat_constants_s { + /* Note these arrays must match the tables in lac_sym_qat_constants.c + * icp_qat_hw_cipher_lookup_tbl and icp_qat_hw_auth_lookup_tbl */ + uint8_t cipher_offset[ICP_QAT_HW_CIPHER_DELIMITER] + [ICP_QAT_HW_CIPHER_MODE_DELIMITER][2][2]; + uint8_t auth_offset[ICP_QAT_HW_AUTH_ALGO_DELIMITER] + [ICP_QAT_HW_AUTH_MODE_DELIMITER][2]; +} lac_sym_qat_constants_t; + +/** + ******************************************************************************* + * @ingroup LacSymQat + * LacSymQat_ConstantsInitLookupTables + * + * + * @description + * The SymCy constants table is 1K of static data which is passed down + * to the FW to be stored in SHRAM for use by the FW. + * This function populates the associated lookup tables which the IA + * driver uses. + * Where there is config data available in the constants table the lookup + * table stores the offset into the constants table. + * Where there's no suitable config data available in the constants table + * zero is stored in the lookup table. + * + * @return none + * + *****************************************************************************/ +void LacSymQat_ConstantsInitLookupTables(CpaInstanceHandle instanceHandle); + +/** +******************************************************************************* +* @ingroup LacSymQat +* LacSymQat_ConstantsGetCipherOffset +* +* @description +* This function looks up the cipher constants lookup array for +* a specific cipher algorithm, mode, direction and convert flag. +* If the lookup table value is zero then there's no suitable config data +* available in the constants table. +* If the value > zero, then there is config data available in the constants +* table which is stored in SHRAM for use by the FW. The value is the offset +* into the constants table, it is returned to the caller in poffset. +* +* +* @param[in] Cipher Algorithm +* @param[in] Cipher Mode +* @param[in] Direction - encrypt/decrypt +* @param[in] convert / no convert +* @param[out] offset into constants table +* +* @return none +* +*****************************************************************************/ +void LacSymQat_ConstantsGetCipherOffset(CpaInstanceHandle instanceHandle, + uint8_t algo, + uint8_t mode, + uint8_t direction, + uint8_t convert, + uint8_t *poffset); + +/** +******************************************************************************* +* @ingroup LacSymQat +* LacSymQat_ConstantsGetAuthOffset +* +* @description +* This function looks up the auth constants lookup array for +* a specific auth algorithm, mode, direction and convert flag. +* If the lookup table value is zero then there's no suitable config data +* available in the constants table. +* If the value > zero, then there is config data available in the constants +* table which is stored in SHRAM for use by the FW. The value is the offset +* into the constants table, it is returned to the caller in poffset. +* +* +* @param[in] auth Algorithm +* @param[in] auth Mode +* @param[in] nested / no nested +* @param[out] offset into constants table +* +* @return none +* +*****************************************************************************/ +void LacSymQat_ConstantsGetAuthOffset(CpaInstanceHandle instanceHandle, + uint8_t algo, + uint8_t mode, + uint8_t nested, + uint8_t *poffset); + +#endif /* LAC_SYM_QAT_SHRAM_CONSTANTS_TABLE_H */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h index 147e10f573f0..0dfb16c8338f 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h @@ -126,6 +126,10 @@ typedef struct lac_sym_qat_hash_state_buffer_info_s { * @param[in] useOptimisedContentDesc Indicate if optimised content desc * is used for this session. * + * @param[in] useStatefulSha3ContentDesc + * Indicate if stateful SHA3 content desc + * is used for this session. + * * @param[in] pPrecompute For auth mode, this is the pointer * to the precompute data. Otherwise this * should be set to NULL @@ -145,6 +149,7 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, icp_qat_hw_auth_mode_t qatHashMode, CpaBoolean useSymConstantsTable, CpaBoolean useOptimisedContentDesc, + CpaBoolean useStatefulSha3ContentDesc, lac_sym_qat_hash_precompute_info_t *pPrecompute, Cpa32U *pHashBlkSizeInBytes); diff --git a/sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c b/sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c index 2f27a1781876..604e5751fba9 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c @@ -798,8 +798,9 @@ LacSymKey_MgfCommon(const CpaInstanceHandle instanceHandle, ICP_QAT_FW_SLICE_DRAM_WR, ICP_QAT_HW_AUTH_MODE0, /* just a plain hash */ CPA_FALSE, /* Not using sym Constants Table in Shared SRAM - */ + */ CPA_FALSE, /* not using the optimised Content Desc */ + CPA_FALSE, /* Not using the stateful SHA3 Content Desc */ NULL, &hashBlkSizeInBytes); @@ -1550,9 +1551,11 @@ LacSymKey_KeyGenSslTls_GenCommon(CpaInstanceHandle instanceHandle, LAC_SYM_KEY_NO_HASH_BLK_OFFSET_QW, ICP_QAT_FW_SLICE_DRAM_WR, qatHashMode, - CPA_FALSE, /* Not using sym Constants Table in SRAM */ - CPA_FALSE, /* Not using the optimised content Desc */ - NULL, /* Precompute data */ + CPA_FALSE, /* Not using sym Constants Table in Shared SRAM + */ + CPA_FALSE, /* not using the optimised content Desc */ + CPA_FALSE, /* Not using the stateful SHA3 Content Desc */ + NULL, /* precompute data */ &hashBlkSizeInBytes); /* SSL3 */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c index da8b1ca4b91b..9234b649cf2f 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c @@ -52,72 +52,7 @@ #include "sal_string_parse.h" #include "lac_sym_auth_enc.h" #include "lac_sym_qat.h" - -/** - * @ingroup LacAlgChain - * Function which checks for support of partial packets for symmetric - * crypto operations - * - * @param[in] pService Pointer to service descriptor - * @param[in/out] pSessionDesc Pointer to session descriptor - * - */ -static void -LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask, - lac_session_desc_t *pSessionDesc) -{ - CpaBoolean isHashPartialSupported = CPA_FALSE; - CpaBoolean isCipherPartialSupported = CPA_FALSE; - CpaBoolean isPartialSupported = CPA_FALSE; - - switch (pSessionDesc->cipherAlgorithm) { - /* Following ciphers don't support partial */ - case CPA_CY_SYM_CIPHER_KASUMI_F8: - case CPA_CY_SYM_CIPHER_AES_F8: - case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: - case CPA_CY_SYM_CIPHER_CHACHA: - case CPA_CY_SYM_CIPHER_ZUC_EEA3: - break; - /* All others support partial */ - default: - isCipherPartialSupported = CPA_TRUE; - } - switch (pSessionDesc->hashAlgorithm) { - /* Following hash don't support partial */ - case CPA_CY_SYM_HASH_KASUMI_F9: - case CPA_CY_SYM_HASH_SNOW3G_UIA2: - case CPA_CY_SYM_HASH_POLY: - case CPA_CY_SYM_HASH_ZUC_EIA3: - case CPA_CY_SYM_HASH_SHAKE_128: - case CPA_CY_SYM_HASH_SHAKE_256: - break; - /* Following hash may support partial based on device capabilities */ - case CPA_CY_SYM_HASH_SHA3_256: - if (ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask) { - isHashPartialSupported = CPA_TRUE; - } - break; - /* All others support partial */ - default: - isHashPartialSupported = CPA_TRUE; - } - switch (pSessionDesc->symOperation) { - case CPA_CY_SYM_OP_CIPHER: - isPartialSupported = isCipherPartialSupported; - break; - case CPA_CY_SYM_OP_HASH: - isPartialSupported = isHashPartialSupported; - break; - case CPA_CY_SYM_OP_ALGORITHM_CHAINING: - if (isCipherPartialSupported && isHashPartialSupported) { - isPartialSupported = CPA_TRUE; - } - break; - case CPA_CY_SYM_OP_NONE: - break; - } - pSessionDesc->isPartialSupported = isPartialSupported; -} +#include "sal_hw_gen.h" /** * @ingroup LacAlgChain @@ -184,6 +119,148 @@ LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList, return CPA_STATUS_FAIL; } +/** + * @ingroup LacAlgChain + * Function which checks for support of partial packets for symmetric + * crypto operations + * + * @param[in] pService Pointer to service descriptor + * @param[in/out] pSessionDesc Pointer to session descriptor + * + */ +static void +LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask, + lac_session_desc_t *pSessionDesc) +{ + CpaBoolean isHashPartialSupported = CPA_FALSE; + CpaBoolean isCipherPartialSupported = CPA_FALSE; + CpaBoolean isPartialSupported = CPA_FALSE; + + switch (pSessionDesc->cipherAlgorithm) { + /* Following ciphers don't support partial */ + case CPA_CY_SYM_CIPHER_KASUMI_F8: + case CPA_CY_SYM_CIPHER_AES_F8: + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: + case CPA_CY_SYM_CIPHER_CHACHA: + case CPA_CY_SYM_CIPHER_ZUC_EEA3: + break; + /* All others support partial */ + default: + isCipherPartialSupported = CPA_TRUE; + break; + } + switch (pSessionDesc->hashAlgorithm) { + /* Following hash don't support partial */ + case CPA_CY_SYM_HASH_KASUMI_F9: + case CPA_CY_SYM_HASH_SNOW3G_UIA2: + case CPA_CY_SYM_HASH_POLY: + case CPA_CY_SYM_HASH_ZUC_EIA3: + break; + /* Following hash may support partial based on device capabilities */ + case CPA_CY_SYM_HASH_SHA3_256: + if (ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask) { + isHashPartialSupported = CPA_TRUE; + } + break; + /* All others support partial */ + default: + isHashPartialSupported = CPA_TRUE; + break; + } + switch (pSessionDesc->symOperation) { + case CPA_CY_SYM_OP_CIPHER: + isPartialSupported = isCipherPartialSupported; + break; + case CPA_CY_SYM_OP_HASH: + isPartialSupported = isHashPartialSupported; + break; + case CPA_CY_SYM_OP_ALGORITHM_CHAINING: + if (isCipherPartialSupported && isHashPartialSupported) { + isPartialSupported = CPA_TRUE; + } + break; + case CPA_CY_SYM_OP_NONE: + break; + default: + break; + } + + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) { + /* UCS slice has no support for state flush and + * because of that is not able to do partial processing */ + isPartialSupported = CPA_FALSE; + } + + pSessionDesc->isPartialSupported = isPartialSupported; +} + +static void +LacAlgChain_CipherCDBuild_ForOptimisedCD( + const CpaCySymCipherSetupData *pCipherData, + lac_session_desc_t *pSessionDesc, + icp_qat_fw_slice_t nextSlice, + Cpa8U cipherOffsetInConstantsTable, + Cpa8U *pOptimisedHwBlockBaseInDRAM, + Cpa32U *pOptimisedHwBlockOffsetInDRAM) +{ + Cpa8U *pCipherKeyField = NULL; + Cpa32U sizeInBytes = 0; + pCipherKeyField = pOptimisedHwBlockBaseInDRAM; + + /* Need to build up the alternative CD for SHRAM Constants Table use + * with an optimised content desc of 64 bytes for this case. Cipher key + * will be in the Content desc in DRAM, The cipher config data + * is now in the SHRAM constants table. */ + + LacSymQat_CipherHwBlockPopulateKeySetup( + pSessionDesc, + pCipherData, + pCipherData->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, + pCipherKeyField, + &sizeInBytes); + + LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr), + pSessionDesc->cipherAlgorithm, + pSessionDesc->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, + nextSlice, + cipherOffsetInConstantsTable); + + *pOptimisedHwBlockOffsetInDRAM += sizeInBytes; +} + +static void +LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData *pCipherData, + lac_session_desc_t *pSessionDesc, + icp_qat_fw_slice_t nextSlice, + Cpa8U cipherOffsetInConstantsTable) +{ + Cpa32U sizeInBytes = 0; + Cpa8U *pCipherKeyField = NULL; + /* Need to build up the alternative CD for SHRAM Constants Table use + * Cipher key will be in the Request, The cipher config data is now in + * the SHRAM constants table. And nothing is now stored in the content + * desc */ + pCipherKeyField = (Cpa8U *)&( + pSessionDesc->shramReqCacheHdr.cd_pars.s1.serv_specif_fields); + + LacSymQat_CipherHwBlockPopulateKeySetup( + pSessionDesc, + pCipherData, + pCipherData->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, + pCipherKeyField, + &sizeInBytes); + + LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr), + pSessionDesc->cipherAlgorithm, + pSessionDesc->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, + nextSlice, + cipherOffsetInConstantsTable); +} + static void LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData, lac_session_desc_t *pSessionDesc, @@ -192,11 +269,14 @@ LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData, icp_qat_fw_comn_flags *pCmnRequestFlags, icp_qat_fw_serv_specif_flags *pLaCmdFlags, Cpa8U *pHwBlockBaseInDRAM, - Cpa32U *pHwBlockOffsetInDRAM) + Cpa32U *pHwBlockOffsetInDRAM, + Cpa32U capabilitiesMask) { Cpa8U *pCipherKeyField = NULL; Cpa8U cipherOffsetInReqQW = 0; Cpa32U sizeInBytes = 0; + void *pCfgData = NULL; + Cpa32U cfgOffset = 0; /* Construct the ContentDescriptor in DRAM */ cipherOffsetInReqQW = (*pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES); @@ -204,11 +284,15 @@ LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData, *pLaCmdFlags, ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP); /* construct cipherConfig in CD in DRAM */ + cfgOffset = *pHwBlockOffsetInDRAM; + pCfgData = pHwBlockBaseInDRAM + cfgOffset; LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, - pHwBlockBaseInDRAM + - *pHwBlockOffsetInDRAM, + pCfgData, &sizeInBytes); + ICP_QAT_FW_LA_SLICE_TYPE_SET(*pLaCmdFlags, + pSessionDesc->cipherSliceType); + *pHwBlockOffsetInDRAM += sizeInBytes; /* Cipher key will be in CD in DRAM. @@ -220,8 +304,10 @@ LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData, QAT_COMN_CD_FLD_TYPE_64BIT_ADR); LacSymQat_CipherHwBlockPopulateKeySetup( + pSessionDesc, pCipherData, pCipherData->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, pCipherKeyField, &sizeInBytes); /* update offset */ @@ -230,14 +316,15 @@ LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData, LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->reqCacheFtr), pSessionDesc->cipherAlgorithm, pSessionDesc->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, nextSlice, cipherOffsetInReqQW); - if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_CHACHA(pSessionDesc->cipherAlgorithm)) { + if (NON_SPC != pSessionDesc->singlePassState) { LacSymQat_CipherCtrlBlockWrite( &(pSessionDesc->reqSpcCacheFtr), pSessionDesc->cipherAlgorithm, pSessionDesc->cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, ICP_QAT_FW_SLICE_DRAM_WR, cipherOffsetInReqQW); } @@ -275,6 +362,7 @@ LacAlgChain_HashCDBuild( pSessionDesc->qatHashMode, CPA_FALSE, CPA_FALSE, + pSessionDesc->useStatefulSha3ContentDesc, pPrecomputeData, &sizeInBytes); @@ -282,6 +370,69 @@ LacAlgChain_HashCDBuild( *pHwBlockOffsetInDRAM += sizeInBytes; sizeInBytes = 0; + + if (pSessionDesc->useOptimisedContentDesc) { + LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr), + instanceHandle, + pHashData, + pOptimisedHwBlockBaseInDRAM, + hashOffsetInConstantsTable, + nextSlice, + pSessionDesc->qatHashMode, + CPA_TRUE, + CPA_TRUE, + CPA_FALSE, + pPrecomputeDataOptimisedCd, + &sizeInBytes); + + *pOptimisedHwBlockOffsetInDRAM += sizeInBytes; + } else if (pSessionDesc->useSymConstantsTable) { + /* Need to build up the alternative CD for SHRAM Constants Table + * use */ + LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr), + instanceHandle, + pHashData, + pHwBlockBaseInDRAM, + hashOffsetInConstantsTable, + nextSlice, + pSessionDesc->qatHashMode, + CPA_TRUE, + CPA_FALSE, + CPA_FALSE, + pPrecomputeData, + &sizeInBytes); + } +} + +static Cpa16U +LacAlgChain_GetCipherConfigSize(lac_session_desc_t *pSessionDesc) +{ + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) { + return sizeof(icp_qat_hw_ucs_cipher_config_t); + } else { + return sizeof(icp_qat_hw_cipher_config_t); + } +} + +static Cpa16U +LacAlgChain_GetCipherConfigOffset(lac_session_desc_t *pSessionDesc) +{ + Cpa16U offset = 0; + + if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation || + SPC == pSessionDesc->singlePassState) { + icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = + (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc + ->reqCacheFtr.cd_ctrl; + offset = cd_ctrl->cipher_cfg_offset; + } else if (CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) { + icp_qat_fw_cipher_cd_ctrl_hdr_t *cd_ctrl = + (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&pSessionDesc + ->reqCacheFtr.cd_ctrl; + offset = cd_ctrl->cipher_cfg_offset; + } + + return offset * LAC_QUAD_WORD_IN_BYTES; } CpaStatus @@ -298,7 +449,7 @@ LacAlgChain_SessionAADUpdate(lac_session_desc_t *pSessionDesc, req_params->u2.aad_sz = LAC_ALIGN_POW2_ROUNDUP(newAADLength, LAC_HASH_AES_GCM_BLOCK_SIZE); - if (CPA_TRUE == pSessionDesc->isSinglePass) { + if (SPC == pSessionDesc->singlePassState) { Cpa8U *pHwBlockBaseInDRAM = NULL; Cpa32U hwBlockOffsetInDRAM = 0; Cpa32U pSizeInBytes = 0; @@ -310,10 +461,10 @@ LacAlgChain_SessionAADUpdate(lac_session_desc_t *pSessionDesc, CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) { if (LAC_CIPHER_IS_GCM(cipher)) { hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC); + LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); } else { hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC); + LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); } } LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, @@ -343,6 +494,8 @@ LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc, CpaCySymCipherSetupData cipherSetupData = { 0 }; Cpa32U sizeInBytes; Cpa8U *pCipherKeyField; + Cpa16U cipherConfigSize; + Cpa16U cipherConfigOffset; sal_qat_content_desc_info_t *pCdInfo = &(pSessionDesc->contentDescInfo); @@ -350,15 +503,23 @@ LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc, cipherSetupData.cipherKeyLenInBytes = pSessionDesc->cipherKeyLenInBytes; cipherSetupData.pCipherKey = pCipherKey; + cipherSetupData.cipherDirection = pSessionDesc->cipherDirection; + + cipherConfigSize = + LacAlgChain_GetCipherConfigSize(pSessionDesc); + cipherConfigOffset = + LacAlgChain_GetCipherConfigOffset(pSessionDesc); + + pCipherKeyField = (Cpa8U *)pCdInfo->pData + cipherConfigOffset + + cipherConfigSize; switch (pSessionDesc->symOperation) { case CPA_CY_SYM_OP_CIPHER: { - pCipherKeyField = (Cpa8U *)pCdInfo->pData + - sizeof(icp_qat_hw_cipher_config_t); - LacSymQat_CipherHwBlockPopulateKeySetup( + pSessionDesc, &(cipherSetupData), cipherSetupData.cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, pCipherKeyField, &sizeInBytes); @@ -368,26 +529,21 @@ LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc, .serv_specif_fields); LacSymQat_CipherHwBlockPopulateKeySetup( + pSessionDesc, &(cipherSetupData), cipherSetupData.cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, pCipherKeyField, &sizeInBytes); } } break; case CPA_CY_SYM_OP_ALGORITHM_CHAINING: { - icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = - (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t - *)&pSessionDesc->reqCacheFtr.cd_ctrl; - - pCipherKeyField = (Cpa8U *)pCdInfo->pData + - cd_ctrl->cipher_cfg_offset * - LAC_QUAD_WORD_IN_BYTES + - sizeof(icp_qat_hw_cipher_config_t); - LacSymQat_CipherHwBlockPopulateKeySetup( + pSessionDesc, &(cipherSetupData), cipherSetupData.cipherKeyLenInBytes, + pSessionDesc->cipherSliceType, pCipherKeyField, &sizeInBytes); } break; @@ -411,10 +567,13 @@ LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc, Cpa8U *pInnerState1 = NULL; Cpa8U *pInnerState2 = NULL; CpaCySymSessionSetupData sessionSetup = { 0 }; + Cpa16U cipherConfigSize; if (pSessionDesc == NULL || pAuthKey == NULL) return CPA_STATUS_FAIL; + cipherConfigSize = LacAlgChain_GetCipherConfigSize(pSessionDesc); + icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr .cd_ctrl; @@ -448,30 +607,28 @@ LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc, /* Calculate offset of cipher key */ if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { sessionSetup.cipherSetupData.pCipherKey = - (Cpa8U *)pHwBlockBaseInDRAM + - sizeof(icp_qat_hw_cipher_config_t); + (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize; } else if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { sessionSetup.cipherSetupData.pCipherKey = - pOutHashSetup + sizeof(icp_qat_hw_cipher_config_t); - } else if (CPA_TRUE == pSessionDesc->isSinglePass) { + pOutHashSetup + cipherConfigSize; + } else if (SPC == pSessionDesc->singlePassState) { CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm; Cpa32U hwBlockOffsetInDRAM = 0; if (pSessionDesc->cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) { sessionSetup.cipherSetupData.pCipherKey = - (Cpa8U *)pHwBlockBaseInDRAM + - sizeof(icp_qat_hw_cipher_config_t); + (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize; } else { if (LAC_CIPHER_IS_GCM(cipher)) hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC); + LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); else hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC); + LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); sessionSetup.cipherSetupData.pCipherKey = (Cpa8U *)pHwBlockBaseInDRAM + hwBlockOffsetInDRAM + - sizeof(icp_qat_hw_cipher_config_t); + cipherConfigSize; } } @@ -497,8 +654,7 @@ LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc, /* SHRAM Constants Table not used for Auth-Enc */ } } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) { - Cpa8U *authKey = - (Cpa8U *)pOutHashSetup + sizeof(icp_qat_hw_cipher_config_t); + Cpa8U *authKey = (Cpa8U *)pOutHashSetup + cipherConfigSize; memcpy(authKey, pAuthKey, pSessionDesc->authKeyLenInBytes); } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm || CPA_CY_SYM_HASH_AES_CBC_MAC == pSessionDesc->hashAlgorithm) { @@ -526,6 +682,247 @@ LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc, return status; } +static void +buildCmdData(sal_crypto_service_t *pService, + lac_session_desc_t *pSessionDesc, + CpaCySymAlgChainOrder *chainOrder, + Cpa16U *proto, + icp_qat_fw_serv_specif_flags *laCmdFlags, + icp_qat_fw_comn_flags *cmnRequestFlags) +{ + /* LW 28 is used to set hash flags for AlgChaining. */ + icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = + (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr + .cd_ctrl; + + /* proto refers to Protocol Flags, which is legacy FW <=> IA interface + * for ZUC and Snow3G. Use extended protocol flags for AlgChaining. + */ + *proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */ + + switch (pSessionDesc->symOperation) { + case CPA_CY_SYM_OP_CIPHER: + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; + + if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == + pSessionDesc->cipherAlgorithm) { + *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; + } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 == + pSessionDesc->cipherAlgorithm) { + *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; + } + if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) { + *proto = ICP_QAT_FW_LA_CCM_PROTO; + } + break; + + case CPA_CY_SYM_OP_HASH: + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH; + if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == + pSessionDesc->hashAlgorithm) { + *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; + } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == + pSessionDesc->hashAlgorithm) { + *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; + } + break; + + case CPA_CY_SYM_OP_ALGORITHM_CHAINING: + if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) { + *proto = ICP_QAT_FW_LA_CCM_PROTO; + + /* Derive chainOrder from direction for isAuthEncryptOp + * cases */ + /* For CCM & GCM modes: force digest verify flag _TRUE + for decrypt and _FALSE for encrypt. For all other + cases use user defined value */ + + if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == + pSessionDesc->cipherDirection) { + *chainOrder = + CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; + pSessionDesc->digestVerify = CPA_FALSE; + } else { + *chainOrder = + CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; + if (CPA_TRUE == pService->forceAEADMacVerify) { + pSessionDesc->digestVerify = CPA_TRUE; + } + } + } else if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm)) { + *proto = ICP_QAT_FW_LA_GCM_PROTO; + + /* Derive chainOrder from direction for isAuthEncryptOp + * cases */ + /* For CCM & GCM modes: force digest verify flag _TRUE + for decrypt and _FALSE for encrypt. For all other + cases use user defined value */ + + if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == + pSessionDesc->cipherDirection) { + *chainOrder = + CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; + pSessionDesc->digestVerify = CPA_FALSE; + } else { + *chainOrder = + CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; + if (CPA_TRUE == pService->forceAEADMacVerify) { + pSessionDesc->digestVerify = CPA_TRUE; + } + } + } else if (LAC_CIPHER_IS_CHACHA( + pSessionDesc->cipherAlgorithm)) { + if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == + pSessionDesc->cipherDirection) { + *chainOrder = + CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; + } else { + *chainOrder = + CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; + } + } else { + pSessionDesc->isAuthEncryptOp = CPA_FALSE; + + if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == + pSessionDesc->cipherAlgorithm) { + *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; + } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 == + pSessionDesc->cipherAlgorithm) { + *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; + } + + if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == + pSessionDesc->hashAlgorithm) { + /* Need to set LW 28 hash flags as well. */ + ICP_QAT_FW_HASH_FLAG_SNOW3G_UIA2_SET( + cd_ctrl->hash_flags, QAT_FW_LA_SNOW3G_UIA2); + } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == + pSessionDesc->hashAlgorithm) { + /* Need to set LW 28 hash flags as well. */ + ICP_QAT_FW_HASH_FLAG_ZUC_EIA3_SET( + cd_ctrl->hash_flags, QAT_FW_LA_ZUC_EIA3); + } + } + + if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH == + *chainOrder) { + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + } else if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER == + *chainOrder) { + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + } + break; + + default: + break; + } + + /* + * Build the header flags with the default settings for this session. + */ + if (pSessionDesc->isDPSession == CPA_TRUE) { + *cmnRequestFlags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + LAC_SYM_DP_QAT_PTR_TYPE); + } else { + *cmnRequestFlags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + LAC_SYM_DEFAULT_QAT_PTR_TYPE); + } + + LacSymQat_LaSetDefaultFlags(laCmdFlags, pSessionDesc->symOperation); + + return; +} + +static void +updateLaCmdFlags(lac_session_desc_t *pSessionDesc, + Cpa16U proto, + icp_qat_fw_serv_specif_flags *laCmdFlags) +{ + if (pSessionDesc->isAuth) { + if (pSessionDesc->digestVerify) { + ICP_QAT_FW_LA_CMP_AUTH_SET(*laCmdFlags, + ICP_QAT_FW_LA_CMP_AUTH_RES); + ICP_QAT_FW_LA_RET_AUTH_SET( + *laCmdFlags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); + } else { + ICP_QAT_FW_LA_RET_AUTH_SET(*laCmdFlags, + ICP_QAT_FW_LA_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET( + *laCmdFlags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + } + } + + if ((CPA_CY_SYM_CIPHER_ZUC_EEA3 == pSessionDesc->cipherAlgorithm) || + (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) { + /* New bit position (12) for ZUC. The FW provides a specific + * macro to use to set the ZUC proto flag. With the new FW I/F + * this needs to be set for both Cipher and Auth */ + ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(*laCmdFlags, proto); + } else { + /* Configure the common header */ + ICP_QAT_FW_LA_PROTO_SET(*laCmdFlags, proto); + } + + /* set Append flag, if digest is appended */ + if (pSessionDesc->digestIsAppended) { + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( + *laCmdFlags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); + } else { + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( + *laCmdFlags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); + } +} + +static lac_single_pass_state_t +LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher, + CpaCySymHashAlgorithm hash, + Cpa32U capabilitiesMask) +{ + lac_single_pass_state_t state = NON_SPC; + if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CHACHA_POLY) { + switch (cipher) { + case CPA_CY_SYM_CIPHER_CHACHA: { + if (CPA_CY_SYM_HASH_POLY == hash) + state = SPC; + break; + } + case CPA_CY_SYM_CIPHER_AES_GCM: { + if ((CPA_CY_SYM_HASH_AES_GCM == hash) || + (CPA_CY_SYM_HASH_AES_GMAC == hash)) + state = LIKELY_SPC; + break; + } + case CPA_CY_SYM_CIPHER_AES_CCM: { + if (LAC_CIPHER_AES_V2(capabilitiesMask)) + state = SPC; + } + default: + /* Do Nothing as it is NON_SPC */ + break; + } + } + return state; +} + +static CpaBoolean +LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired, + Cpa32U capabilitiesMask, + lac_session_desc_t *pSessionDesc) +{ + CpaBoolean hasSha3Ext = + ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask; + CpaBoolean useStatefulSha3DescFlag = CPA_FALSE; + + if (hasSha3Ext && !partialsNotRequired && + (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH) && + LAC_HASH_IS_SHA3(pSessionDesc->hashAlgorithm)) { + useStatefulSha3DescFlag = CPA_TRUE; + } + return useStatefulSha3DescFlag; +} + /** @ingroup LacAlgChain */ CpaStatus LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, @@ -534,6 +931,7 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, { CpaStatus stat, status = CPA_STATUS_SUCCESS; sal_qat_content_desc_info_t *pCdInfo = NULL; + sal_qat_content_desc_info_t *pCdInfoOptimised = NULL; sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; Cpa32U capabilitiesMask = pService->generic_service_info.capabilitiesMask; @@ -543,7 +941,9 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, Cpa32U optimisedHwBlockOffsetInDRAM = 0; Cpa8U cipherOffsetInConstantsTable = 0; Cpa8U hashOffsetInConstantsTable = 0; + icp_qat_fw_comn_flags cmnRequestFlags = 0; icp_qat_fw_comn_req_t *pMsg = NULL; + icp_qat_fw_comn_req_t *pMsgS = NULL; const CpaCySymCipherSetupData *pCipherData; const CpaCySymHashSetupData *pHashData; Cpa16U proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */ @@ -584,147 +984,58 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, pSessionDesc->symOperation = pSessionSetupData->symOperation; switch (pSessionDesc->symOperation) { case CPA_CY_SYM_OP_CIPHER: - pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; - pSessionDesc->isCipher = TRUE; - pSessionDesc->isAuth = FALSE; + pSessionDesc->isCipher = CPA_TRUE; + pSessionDesc->isAuth = CPA_FALSE; pSessionDesc->isAuthEncryptOp = CPA_FALSE; - - if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == - pSessionSetupData->cipherSetupData.cipherAlgorithm) { - proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; - } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 == - pSessionSetupData->cipherSetupData.cipherAlgorithm) { - proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; - } + pSessionDesc->singlePassState = NON_SPC; break; case CPA_CY_SYM_OP_HASH: - pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH; - pSessionDesc->isCipher = FALSE; - pSessionDesc->isAuth = TRUE; + pSessionDesc->isCipher = CPA_FALSE; + pSessionDesc->isAuth = CPA_TRUE; pSessionDesc->isAuthEncryptOp = CPA_FALSE; - - if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == - pSessionSetupData->hashSetupData.hashAlgorithm) { - proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; - } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == - pSessionSetupData->hashSetupData.hashAlgorithm) { - proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; - } - + pSessionDesc->singlePassState = NON_SPC; break; - case CPA_CY_SYM_OP_ALGORITHM_CHAINING: - pSessionDesc->isCipher = TRUE; - pSessionDesc->isAuth = TRUE; + case CPA_CY_SYM_OP_ALGORITHM_CHAINING: { + pSessionDesc->isCipher = CPA_TRUE; + pSessionDesc->isAuth = CPA_TRUE; + pSessionDesc->singlePassState = + LacSymAlgChain_GetSpcState(pCipherData->cipherAlgorithm, + pHashData->hashAlgorithm, + capabilitiesMask); - { - /* set up some useful shortcuts */ - CpaCySymCipherAlgorithm cipherAlgorithm = - pSessionSetupData->cipherSetupData.cipherAlgorithm; - CpaCySymCipherDirection cipherDir = - pSessionSetupData->cipherSetupData.cipherDirection; - - if (LAC_CIPHER_IS_CCM(cipherAlgorithm)) { - pSessionDesc->isAuthEncryptOp = CPA_TRUE; - pSessionDesc->digestIsAppended = CPA_TRUE; - proto = ICP_QAT_FW_LA_CCM_PROTO; - - /* Derive chainOrder from direction for - * isAuthEncryptOp - * cases */ - /* For CCM & GCM modes: force digest verify flag - _TRUE - for decrypt and _FALSE for encrypt. For all - other cases - use user defined value */ - - if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == - cipherDir) { - chainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; - pSessionDesc->digestVerify = CPA_FALSE; - } else { - chainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; - pSessionDesc->digestVerify = CPA_TRUE; - } - } else if (LAC_CIPHER_IS_GCM(cipherAlgorithm)) { - pSessionDesc->isAuthEncryptOp = CPA_TRUE; - proto = ICP_QAT_FW_LA_GCM_PROTO; - - /* Derive chainOrder from direction for - * isAuthEncryptOp - * cases */ - /* For CCM & GCM modes: force digest verify flag - _TRUE - for decrypt and _FALSE for encrypt. For all - other cases - use user defined value */ - - if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == - cipherDir) { - chainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; - pSessionDesc->digestVerify = CPA_FALSE; - } else { - chainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; - pSessionDesc->digestVerify = CPA_TRUE; - } - } else if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) { - pSessionDesc->isAuthEncryptOp = CPA_TRUE; - proto = ICP_QAT_FW_LA_SINGLE_PASS_PROTO; - - if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == - cipherDir) { - chainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; - } else { - chainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; - } - } else { - pSessionDesc->isAuthEncryptOp = CPA_FALSE; - /* Use the chainOrder passed in */ - chainOrder = pSessionSetupData->algChainOrder; - if ((chainOrder != - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) && - (chainOrder != - CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) { - LAC_INVALID_PARAM_LOG("algChainOrder"); - return CPA_STATUS_INVALID_PARAM; - } - - if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == - pSessionSetupData->hashSetupData - .hashAlgorithm) { - proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; - } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == - pSessionSetupData->hashSetupData - .hashAlgorithm) { - proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; - } - } - - if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH == - chainOrder) { - pSessionDesc->laCmdId = - ICP_QAT_FW_LA_CMD_CIPHER_HASH; - } else if ( - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER == - chainOrder) { - pSessionDesc->laCmdId = - ICP_QAT_FW_LA_CMD_HASH_CIPHER; + switch (pSessionSetupData->cipherSetupData.cipherAlgorithm) { + case CPA_CY_SYM_CIPHER_AES_CCM: { + pSessionDesc->isAuthEncryptOp = CPA_TRUE; + pSessionDesc->digestIsAppended = CPA_TRUE; + } break; + case CPA_CY_SYM_CIPHER_AES_GCM: + case CPA_CY_SYM_CIPHER_CHACHA: + pSessionDesc->isAuthEncryptOp = CPA_TRUE; + break; + default: { + pSessionDesc->isAuthEncryptOp = CPA_FALSE; + /* Use the chainOrder passed in */ + chainOrder = pSessionSetupData->algChainOrder; + if ((chainOrder != + CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) && + (chainOrder != + CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) { + LAC_INVALID_PARAM_LOG("algChainOrder"); + return CPA_STATUS_INVALID_PARAM; } + } break; } - break; + } break; default: + pSessionDesc->singlePassState = NON_SPC; break; } if (pSessionDesc->isCipher) { -/* Populate cipher specific session data */ + /* Populate cipher specific session data */ - status = LacCipher_SessionSetupDataCheck(pCipherData); + status = LacCipher_SessionSetupDataCheck(pCipherData, + capabilitiesMask); if (CPA_STATUS_SUCCESS == status) { pSessionDesc->cipherAlgorithm = @@ -735,16 +1046,12 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, pCipherData->cipherDirection; /* ARC4 base key isn't added to the content descriptor, - * because - * we don't need to pass it directly to the QAT engine. - * Instead - * an initial cipher state & key matrix is derived from - * the - * base key and provided to the QAT through the state - * pointer - * in the request params. We'll store this initial state - * in - * the session descriptor. */ + * because we don't need to pass it directly to the QAT + * engine. Instead an initial cipher state & key matrix + * is derived from the base key and provided to the QAT + * through the state pointer in the request params. + * We'll store this initial state in the session + * descriptor. */ if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) { LacSymQat_CipherArc4StateInit( @@ -809,27 +1116,20 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, ICP_QAT_HW_AUTH_MODE0; } else /* CPA_CY_SYM_HASH_MODE_AUTH && anything except CPA_CY_SYM_HASH_AES_CBC_MAC - */ + */ { if (IS_HMAC_ALG(pHashData->hashAlgorithm)) { - /* SHA3 and SM3 HMAC do not support - * precompute, force MODE2 - * for AUTH */ - if ((CPA_CY_SYM_HASH_SHA3_224 == - pHashData->hashAlgorithm) || - (CPA_CY_SYM_HASH_SHA3_256 == - pHashData->hashAlgorithm) || - (CPA_CY_SYM_HASH_SHA3_384 == - pHashData->hashAlgorithm) || - (CPA_CY_SYM_HASH_SHA3_512 == - pHashData->hashAlgorithm) || + /* SHA3 HMAC and SM3 do not support + * precompute, force MODE2 for AUTH */ + if (LAC_HASH_IS_SHA3( + pHashData->hashAlgorithm) || (CPA_CY_SYM_HASH_SM3 == pHashData->hashAlgorithm)) { pSessionDesc->qatHashMode = ICP_QAT_HW_AUTH_MODE2; } else { pSessionDesc->qatHashMode = - ICP_QAT_HW_AUTH_MODE1; + pService->qatHmacMode; } } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pHashData->hashAlgorithm) { @@ -847,39 +1147,71 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, * build the message templates * create two content descriptors in the case we can support using SHRAM * constants and an optimised content descriptor. we have to do this in - *case - * of partials. - * 64 byte content descriptor is used in the SHRAM case for - *AES-128-HMAC-SHA1 + *case of partials. 64 byte content descriptor is used in the SHRAM case + *for AES-128-HMAC-SHA1 *-----------------------------------------------------------------------*/ if (CPA_STATUS_SUCCESS == status) { + pSessionDesc->cipherSliceType = + LacCipher_GetCipherSliceType(pService, + pSessionDesc->cipherAlgorithm, + pSessionDesc->hashAlgorithm); LacSymCheck_IsPartialSupported(capabilitiesMask, pSessionDesc); + pSessionDesc->useOptimisedContentDesc = CPA_FALSE; + pSessionDesc->useStatefulSha3ContentDesc = CPA_FALSE; + + /* Build configuration data */ + buildCmdData(pService, + pSessionDesc, + &chainOrder, + &proto, + &pSessionDesc->laCmdFlags, + &cmnRequestFlags); + + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == + pSessionDesc->cipherSliceType) + pSessionDesc->useSymConstantsTable = CPA_FALSE; + else + pSessionDesc->useSymConstantsTable = + LacSymQat_UseSymConstantsTable( + pSessionDesc, + &cipherOffsetInConstantsTable, + &hashOffsetInConstantsTable); + + /* for a certain combination of Algorthm Chaining we want to + use an optimised cd block */ + + if (pSessionDesc->symOperation == + CPA_CY_SYM_OP_ALGORITHM_CHAINING && + pSessionDesc->useSymConstantsTable == CPA_TRUE) { + pSessionDesc->useOptimisedContentDesc = + LacSymQat_UseOptimisedContentDesc(pSessionDesc); + } + + /* check whether we need to construct content desc for stateful + * SHA3 */ + pSessionDesc->useStatefulSha3ContentDesc = + LacAlgChain_UseStatefulSha3ContentDesc( + pSessionSetupData->partialsNotRequired, + capabilitiesMask, + pSessionDesc); /* setup some convenience pointers */ pCdInfo = &(pSessionDesc->contentDescInfo); pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; hwBlockOffsetInDRAM = 0; - /* - * Build the header flags with the default settings for this - * session. - */ - if (pSessionDesc->isDPSession == CPA_TRUE) { - pSessionDesc->cmnRequestFlags = - ICP_QAT_FW_COMN_FLAGS_BUILD( - QAT_COMN_CD_FLD_TYPE_64BIT_ADR, - LAC_SYM_DP_QAT_PTR_TYPE); - } else { - pSessionDesc->cmnRequestFlags = - ICP_QAT_FW_COMN_FLAGS_BUILD( - QAT_COMN_CD_FLD_TYPE_64BIT_ADR, - LAC_SYM_DEFAULT_QAT_PTR_TYPE); + /* set up the pointer for the optimised content desc if this is + * possible we still have to support both cd types in case of + * partials so we construct both */ + if (pSessionDesc->useOptimisedContentDesc == CPA_TRUE) { + pCdInfoOptimised = + &(pSessionDesc->contentDescOptimisedInfo); + pOptimisedHwBlockBaseInDRAM = + (Cpa8U *)pCdInfoOptimised->pData; + optimisedHwBlockOffsetInDRAM = 0; } - LacSymQat_LaSetDefaultFlags(&pSessionDesc->laCmdFlags, - pSessionDesc->symOperation); - switch (pSessionDesc->symOperation) { case CPA_CY_SYM_OP_CIPHER: { LacAlgChain_CipherCDBuild( @@ -890,13 +1222,22 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, &pSessionDesc->cmnRequestFlags, &pSessionDesc->laCmdFlags, pHwBlockBaseInDRAM, - &hwBlockOffsetInDRAM); + &hwBlockOffsetInDRAM, + capabilitiesMask); + + if (pSessionDesc->useSymConstantsTable) { + LacAlgChain_CipherCDBuild_ForSHRAM( + pCipherData, + pSessionDesc, + ICP_QAT_FW_SLICE_DRAM_WR, + cipherOffsetInConstantsTable); + } } break; case CPA_CY_SYM_OP_HASH: LacAlgChain_HashCDBuild(pHashData, instanceHandle, pSessionDesc, - ICP_QAT_FW_SLICE_DRAM_WR, + ICP_QAT_FW_SLICE_NULL, hashOffsetInConstantsTable, &pSessionDesc->cmnRequestFlags, &pSessionDesc->laCmdFlags, @@ -909,14 +1250,10 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, break; case CPA_CY_SYM_OP_ALGORITHM_CHAINING: /* For CCM/GCM, CPM firmware currently expects the - * cipher and - * hash h/w setup blocks to be arranged according to the - * chain - * order (Except for GCM/CCM, order doesn't actually - * matter as - * long as the config offsets are set correctly in CD - * control - * blocks + * cipher and hash h/w setup blocks to be arranged + * according to the chain order (Except for GCM/CCM, + * order doesn't actually matter as long as the config + * offsets are set correctly in CD control blocks */ if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER == chainOrder) { @@ -943,11 +1280,20 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, &pSessionDesc->cmnRequestFlags, &pSessionDesc->laCmdFlags, pHwBlockBaseInDRAM, - &hwBlockOffsetInDRAM); - if (LAC_CIPHER_IS_SPC( - pCipherData->cipherAlgorithm, - pHashData->hashAlgorithm, - capabilitiesMask)) { + &hwBlockOffsetInDRAM, + capabilitiesMask); + + if (pSessionDesc->useOptimisedContentDesc) { + LacAlgChain_CipherCDBuild_ForOptimisedCD( + pCipherData, + pSessionDesc, + ICP_QAT_FW_SLICE_DRAM_WR, + cipherOffsetInConstantsTable, + pOptimisedHwBlockBaseInDRAM, + &optimisedHwBlockOffsetInDRAM); + } + + if (NON_SPC != pSessionDesc->singlePassState) { pCdInfo->hwBlkSzQuadWords = (LAC_BYTES_TO_QUADWORDS( hwBlockOffsetInDRAM)); @@ -957,7 +1303,8 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, (icp_qat_fw_comn_req_t *)pMsg, pCdInfo); } - } else { + } else /* CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH */ + { LacAlgChain_CipherCDBuild( pCipherData, pSessionDesc, @@ -966,12 +1313,20 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, &pSessionDesc->cmnRequestFlags, &pSessionDesc->laCmdFlags, pHwBlockBaseInDRAM, - &hwBlockOffsetInDRAM); + &hwBlockOffsetInDRAM, + capabilitiesMask); - if (LAC_CIPHER_IS_SPC( - pCipherData->cipherAlgorithm, - pHashData->hashAlgorithm, - capabilitiesMask)) { + if (pSessionDesc->useOptimisedContentDesc) { + LacAlgChain_CipherCDBuild_ForOptimisedCD( + pCipherData, + pSessionDesc, + ICP_QAT_FW_SLICE_AUTH, + cipherOffsetInConstantsTable, + pOptimisedHwBlockBaseInDRAM, + &optimisedHwBlockOffsetInDRAM); + } + + if (NON_SPC != pSessionDesc->singlePassState) { pCdInfo->hwBlkSzQuadWords = LAC_BYTES_TO_QUADWORDS( hwBlockOffsetInDRAM); @@ -1009,8 +1364,7 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, CpaBoolean hashStateBuffer = CPA_TRUE; /* set up fields in both the cd_ctrl and reqParams which - * describe - * the ReqParams block */ + * describe the ReqParams block */ LacSymQat_HashSetupReqParamsMetaData( &(pSessionDesc->reqCacheFtr), instanceHandle, @@ -1019,6 +1373,17 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, pSessionDesc->qatHashMode, pSessionDesc->digestVerify); + if (pSessionDesc->useSymConstantsTable) { + /* Need to set up for SHRAM Constants Table use also */ + LacSymQat_HashSetupReqParamsMetaData( + &(pSessionDesc->shramReqCacheFtr), + instanceHandle, + pHashData, + hashStateBuffer, + pSessionDesc->qatHashMode, + pSessionDesc->digestVerify); + } + /* populate the hash state prefix buffer info structure * (part of user allocated session memory & the * buffer itself. For CCM/GCM the buffer is stored in the @@ -1056,86 +1421,122 @@ LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, pSessionDesc->hashStatePrefixBuffer, precomputeData.pState1, precomputeData.pState2); + if (pSessionDesc->useOptimisedContentDesc) { + status = LacHash_PrecomputeDataCreate( + instanceHandle, + (CpaCySymSessionSetupData *) + pSessionSetupData, + LacSymAlgChain_HashPrecomputeDoneCb, + pSessionDesc, + pSessionDesc->hashStatePrefixBuffer, + precomputeDataOptimisedCd.pState1, + precomputeDataOptimisedCd.pState2); + } } else if (pHashData->hashAlgorithm == CPA_CY_SYM_HASH_AES_CBC_MAC) { - LAC_OS_BZERO(precomputeData.pState2, - precomputeData.state2Size); - memcpy(precomputeData.pState2, - pHashData->authModeSetupData.authKey, - pHashData->authModeSetupData - .authKeyLenInBytes); - } - } - if (CPA_STATUS_SUCCESS == status) { - - if (pSessionDesc->digestVerify) { - - ICP_QAT_FW_LA_CMP_AUTH_SET( - pSessionDesc->laCmdFlags, - ICP_QAT_FW_LA_CMP_AUTH_RES); - ICP_QAT_FW_LA_RET_AUTH_SET( - pSessionDesc->laCmdFlags, - ICP_QAT_FW_LA_NO_RET_AUTH_RES); - } else { - - ICP_QAT_FW_LA_RET_AUTH_SET( - pSessionDesc->laCmdFlags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET( - pSessionDesc->laCmdFlags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + if (NULL != precomputeData.pState2) { + LAC_OS_BZERO(precomputeData.pState2, + precomputeData.state2Size); + memcpy(precomputeData.pState2, + pHashData->authModeSetupData + .authKey, + pHashData->authModeSetupData + .authKeyLenInBytes); + } } } } if (CPA_STATUS_SUCCESS == status) { + /* Configure the ContentDescriptor field + in the request if not done already */ pCdInfo->hwBlkSzQuadWords = LAC_BYTES_TO_QUADWORDS(hwBlockOffsetInDRAM); pMsg = (icp_qat_fw_comn_req_t *)&(pSessionDesc->reqCacheHdr); - - /* Configure the ContentDescriptor field - * in the request if not done already */ SalQatMsg_ContentDescHdrWrite((icp_qat_fw_comn_req_t *)pMsg, pCdInfo); - if (CPA_CY_SYM_CIPHER_ZUC_EEA3 == - pSessionSetupData->cipherSetupData.cipherAlgorithm || - pHashData->hashAlgorithm == CPA_CY_SYM_HASH_ZUC_EIA3) { - /* New bit position (12) for ZUC. The FW provides a - * specific macro - * to use to set the ZUC proto flag. With the new FW I/F - * this needs - * to be set for both Cipher and Auth */ - ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( - pSessionDesc->laCmdFlags, proto); - } else { - /* Configure the common header */ - ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, - proto); + pMsgS = + (icp_qat_fw_comn_req_t *)&(pSessionDesc->shramReqCacheHdr); + /*If we are using the optimised CD then + we have to set this up correctly in the SHARM reqCache*/ + if (pSessionDesc->useOptimisedContentDesc) { + pCdInfoOptimised->hwBlkSzQuadWords = + LAC_BYTES_TO_QUADWORDS( + optimisedHwBlockOffsetInDRAM); + SalQatMsg_ContentDescHdrWrite( + (icp_qat_fw_comn_req_t *)pMsgS, pCdInfoOptimised); } - /* set Append flag, if digest is appended */ - if (pSessionDesc->digestIsAppended) { - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( - pSessionDesc->laCmdFlags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); - } else { - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( - pSessionDesc->laCmdFlags, - ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); - } + /* Updates command flags basing on configured alg */ + updateLaCmdFlags(pSessionDesc, + proto, + &pSessionDesc->laCmdFlags); SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsg, ICP_QAT_FW_COMN_REQ_CPM_FW_LA, pSessionDesc->laCmdId, pSessionDesc->cmnRequestFlags, pSessionDesc->laCmdFlags); + + /* Need to duplicate if SHRAM Constants Table used */ + if (pSessionDesc->useSymConstantsTable) { + ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET( + pSessionDesc->laCmdFlags, + ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP); + + if (pSessionDesc->isCipher && + !pSessionDesc->useOptimisedContentDesc) { + ICP_QAT_FW_COMN_CD_FLD_TYPE_SET( + cmnRequestFlags, + QAT_COMN_CD_FLD_TYPE_16BYTE_DATA); + } + + SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsgS, + ICP_QAT_FW_COMN_REQ_CPM_FW_LA, + pSessionDesc->laCmdId, + cmnRequestFlags, + pSessionDesc->laCmdFlags); + } } return status; } +static void +LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t *pMsg, + Cpa32U packetType, + icp_qat_hw_auth_mode_t qatHashMode) +{ + icp_qat_fw_auth_cd_ctrl_hdr_t *pAuthCdCtrlHdr = NULL; + + pAuthCdCtrlHdr = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); + + if (IS_HASH_MODE_2(qatHashMode)) { + if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) || + (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) { + ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( + pAuthCdCtrlHdr->hash_flags, + QAT_FW_LA_SKIP_INNER_STATE1_LOAD); + ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( + pAuthCdCtrlHdr->hash_flags, + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); + } else if (ICP_QAT_FW_LA_PARTIAL_END == packetType) { + ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( + pAuthCdCtrlHdr->hash_flags, + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); + } + } else { + if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) || + (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) { + ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( + pAuthCdCtrlHdr->hash_flags, + QAT_FW_LA_SKIP_INNER_STATE1_LOAD); + } + } +} + /** @ingroup LacAlgChain */ CpaStatus LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, @@ -1171,7 +1572,10 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, CpaCySymHashAlgorithm hash; Cpa8U paddingLen = 0; Cpa8U blockLen = 0; + CpaBoolean digestIsAppended = CPA_FALSE; + Cpa32U aadLenInBytes = 0; Cpa64U srcPktSize = 0; + Cpa64U dstPktSize = 0; /* Set the command id */ laCmdId = pSessionDesc->laCmdId; @@ -1179,35 +1583,51 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, cipher = pSessionDesc->cipherAlgorithm; hash = pSessionDesc->hashAlgorithm; + CpaBoolean isSpCcm = + (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(capabilitiesMask)); + + if (CPA_CY_SYM_HASH_AES_GMAC == hash) { + pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes; + if (pOpData->messageLenToHashInBytes == 0 || + pOpData->pAdditionalAuthData != NULL) { + LAC_INVALID_PARAM_LOG( + "For AES_GMAC, AAD Length " + "(messageLenToHashInBytes) must " + "be non zero and pAdditionalAuthData " + "must be NULL"); + return CPA_STATUS_INVALID_PARAM; + } + } + + aadLenInBytes = pSessionDesc->aadLenInBytes; + /* Convert Alg Chain Request to Cipher Request for CCP and * AES_GCM single pass */ - if (!pSessionDesc->isSinglePass && - LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) && - (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes)) { + if ((NON_SPC != pSessionDesc->singlePassState) && + (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes))) { pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; laCmdId = pSessionDesc->laCmdId; pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER; - pSessionDesc->isSinglePass = CPA_TRUE; + pSessionDesc->singlePassState = SPC; pSessionDesc->isCipher = CPA_TRUE; pSessionDesc->isAuthEncryptOp = CPA_FALSE; pSessionDesc->isAuth = CPA_FALSE; - if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { - pSessionDesc->aadLenInBytes = - pOpData->messageLenToHashInBytes; - if (ICP_QAT_FW_SPC_AAD_SZ_MAX < - pSessionDesc->aadLenInBytes) { + + if (CPA_CY_SYM_HASH_AES_GMAC == hash) { + if (ICP_QAT_FW_SPC_AAD_SZ_MAX < aadLenInBytes) { LAC_INVALID_PARAM_LOG( "aadLenInBytes for AES_GMAC"); return CPA_STATUS_INVALID_PARAM; } } - /* New bit position (13) for SINGLE PASS. * The FW provides a specific macro to use to set the proto flag */ ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO); - ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); + if (isCyGen2x(pService)) { + ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); + } pCdInfo = &(pSessionDesc->contentDescInfo); pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; @@ -1215,11 +1635,24 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, pSessionDesc->cipherDirection) { if (LAC_CIPHER_IS_GCM(cipher)) hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC); - else + LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); + else if (LAC_CIPHER_IS_CHACHA(cipher)) hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC); + LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); + } else if (isSpCcm) { + hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( + LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM); } + + /* Update cipher slice type */ + pSessionDesc->cipherSliceType = + LacCipher_GetCipherSliceType(pService, + pSessionDesc->cipherAlgorithm, + pSessionDesc->hashAlgorithm); + + ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags, + pSessionDesc->cipherSliceType); + /* construct cipherConfig in CD in DRAM */ LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, pHwBlockBaseInDRAM + @@ -1228,31 +1661,53 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( pSessionDesc->reqSpcCacheHdr), ICP_QAT_FW_COMN_REQ_CPM_FW_LA, - pSessionDesc->laCmdId, + laCmdId, + pSessionDesc->cmnRequestFlags, + pSessionDesc->laCmdFlags); + } else if ((SPC == pSessionDesc->singlePassState) && + (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) { + pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; + pSessionDesc->singlePassState = LIKELY_SPC; + pSessionDesc->isCipher = CPA_TRUE; + pSessionDesc->isAuthEncryptOp = CPA_TRUE; + pSessionDesc->isAuth = CPA_TRUE; + pCdInfo = &(pSessionDesc->contentDescInfo); + pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; + + if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == + pSessionDesc->cipherDirection) { + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + } else { + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + } + + laCmdId = pSessionDesc->laCmdId; + ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( + pSessionDesc->laCmdFlags, 0); + ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, + ICP_QAT_FW_LA_GCM_PROTO); + + LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, + pHwBlockBaseInDRAM + + hwBlockOffsetInDRAM, + &sizeInBytes); + + SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( + pSessionDesc->reqCacheHdr), + ICP_QAT_FW_COMN_REQ_CPM_FW_LA, + laCmdId, pSessionDesc->cmnRequestFlags, pSessionDesc->laCmdFlags); - } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { - pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes; } - if (LAC_CIPHER_IS_CHACHA(cipher) && - (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) { + else if (LAC_CIPHER_IS_CHACHA(cipher) && + (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) { LAC_INVALID_PARAM_LOG("IV for CHACHA"); return CPA_STATUS_INVALID_PARAM; - } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { - if (pOpData->messageLenToHashInBytes == 0 || - pOpData->pAdditionalAuthData != NULL) { - LAC_INVALID_PARAM_LOG( - "For AES_GMAC, AAD Length " - "(messageLenToHashInBytes) must " - "be non zero and pAdditionalAuthData " - "must be NULL"); - status = CPA_STATUS_INVALID_PARAM; - } } - if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { - if (CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) { + if ((CPA_TRUE == pSessionDesc->isAuthEncryptOp) || isSpCcm) { + if (CPA_CY_SYM_HASH_AES_CCM == hash) { status = LacSymAlgChain_CheckCCMData( pOpData->pAdditionalAuthData, pOpData->pIv, @@ -1266,9 +1721,8 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, pOpData->messageLenToCipherInBytes, pOpData->ivLenInBytes); } - } else if (CPA_CY_SYM_HASH_AES_GCM == - pSessionDesc->hashAlgorithm) { - if (pSessionDesc->aadLenInBytes != 0 && + } else if (CPA_CY_SYM_HASH_AES_GCM == hash) { + if (aadLenInBytes != 0 && pOpData->pAdditionalAuthData == NULL) { LAC_INVALID_PARAM_LOG("pAdditionalAuthData"); status = CPA_STATUS_INVALID_PARAM; @@ -1304,37 +1758,48 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, &srcAddrPhys, CPA_FALSE, &(pService->generic_service_info)); - } else { - status = LacBuffDesc_BufferListDescWrite( - (CpaBufferList *)pSrcBuffer, - &srcAddrPhys, - CPA_FALSE, - &(pService->generic_service_info)); - } - if (CPA_STATUS_SUCCESS != status) { - LAC_LOG_ERROR("Unable to write src buffer descriptors"); - } - - /* For out of place operations */ - if ((pSrcBuffer != pDstBuffer) && - (CPA_STATUS_SUCCESS == status)) { - if (IS_ZERO_LENGTH_BUFFER_SUPPORTED(cipher, hash)) { + if (CPA_STATUS_SUCCESS != status) { + LAC_LOG_ERROR( + "Unable to write src buffer descriptors"); + } + /* For out of place operations */ + if ((pSrcBuffer != pDstBuffer) && + (CPA_STATUS_SUCCESS == status)) { status = LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer( pDstBuffer, &dstAddrPhys, CPA_FALSE, &(pService->generic_service_info)); - } else { + if (CPA_STATUS_SUCCESS != status) { + LAC_LOG_ERROR( + "Unable to write dest buffer descriptors"); + } + } + } else { + status = LacBuffDesc_BufferListDescWrite( + (CpaBufferList *)pSrcBuffer, + &srcAddrPhys, + CPA_FALSE, + &(pService->generic_service_info)); + if (CPA_STATUS_SUCCESS != status) { + LAC_LOG_ERROR( + "Unable to write src buffer descriptors in " + "LacBuffDesc_BufferListDescWrite"); + } + /* For out of place operations */ + if ((pSrcBuffer != pDstBuffer) && + (CPA_STATUS_SUCCESS == status)) { status = LacBuffDesc_BufferListDescWrite( pDstBuffer, &dstAddrPhys, CPA_FALSE, &(pService->generic_service_info)); - } - if (CPA_STATUS_SUCCESS != status) { - LAC_LOG_ERROR( - "Unable to write dest buffer descriptors"); + if (CPA_STATUS_SUCCESS != status) { + LAC_LOG_ERROR( + "Unable to write dest buffer descriptors in " + "LacBuffDesc_BufferListDescWrite"); + } } } } @@ -1359,9 +1824,9 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, * the first partial has been sent. Set a flag here so the * response knows to do this. */ - if ((laCmdId != ICP_QAT_FW_LA_CMD_AUTH) && + if (LAC_CIPHER_IS_XTS_MODE(cipher) && + (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) && (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) && - (LAC_CIPHER_IS_XTS_MODE(pSessionDesc->cipherAlgorithm)) && (qatPacketType == ICP_QAT_FW_LA_PARTIAL_START)) { pCookie->updateKeySizeOnRecieve = CPA_TRUE; } @@ -1374,26 +1839,21 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, pMsg = &(pCookie->qatMsg); pMsgDummy = (Cpa8U *)pMsg; - if (pSessionDesc->isSinglePass) { + if (SPC == pSessionDesc->singlePassState) { pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr); pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr); } else { /* Normally, we want to use the SHRAM Constants Table if - * possible - * for best performance (less DRAM accesses incurred by - * CPM). But - * we can't use it for partial-packet hash operations. - * This is why - * we build 2 versions of the message template at - * sessionInit, + * possible for best performance (less DRAM accesses + * incurred by CPM). But we can't use it for + * partial-packet hash operations. This is why we build + * 2 versions of the message template at sessionInit, * one for SHRAM Constants Table usage and the other - * (default) for - * Content Descriptor h/w setup data in DRAM. And we - * chose between - * them here on a per-request basis, when we know the - * packetType + * (default) for Content Descriptor h/w setup data in + * DRAM. And we chose between them here on a + * per-request basis, when we know the packetType */ if ((!pSessionDesc->useSymConstantsTable) || (pSessionDesc->isAuth && @@ -1418,8 +1878,9 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, 0, (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW)); - memcpy(pMsgDummy + (LAC_LONG_WORD_IN_BYTES * - LAC_START_OF_CACHE_FTR_IN_LW), + memcpy(pMsgDummy + + (LAC_LONG_WORD_IN_BYTES * + LAC_START_OF_CACHE_FTR_IN_LW), pCacheDummyFtr, (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW)); /* @@ -1442,11 +1903,11 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, LacSymQat_LaPacketCommandFlagSet( qatPacketType, laCmdId, - pSessionDesc->cipherAlgorithm, + cipher, &pMsg->comn_hdr.serv_specif_flags, pOpData->ivLenInBytes); - if (pSessionDesc->isSinglePass) { + if (SPC == pSessionDesc->singlePassState) { ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( pMsg->comn_hdr.serv_specif_flags, ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS); @@ -1463,7 +1924,11 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, } } + ICP_QAT_FW_LA_SLICE_TYPE_SET(pMsg->comn_hdr.serv_specif_flags, + pSessionDesc->cipherSliceType); + LacBuffDesc_BufferListTotalSizeGet(pSrcBuffer, &srcPktSize); + LacBuffDesc_BufferListTotalSizeGet(pDstBuffer, &dstPktSize); /* * Populate the CipherRequestParams section of the Request @@ -1472,8 +1937,9 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, Cpa8U *pIvBuffer = NULL; - status = LacCipher_PerformParamCheck( - pSessionDesc->cipherAlgorithm, pOpData, srcPktSize); + status = LacCipher_PerformParamCheck(cipher, + pOpData, + srcPktSize); if (CPA_STATUS_SUCCESS != status) { /* free the cookie */ Lac_MemPoolEntryFree(pCookie); @@ -1488,19 +1954,18 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, qatPacketType, &pIvBuffer); } - if (pSessionDesc->isSinglePass && + if ((SPC == pSessionDesc->singlePassState) && ((ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType) || (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType))) { /* For SPC stateful cipher state size for mid - * and - * end partial packet is 48 bytes + * and end partial packet is 48 bytes */ pSpcCdCtrlHdr = (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&( pMsg->cd_ctrl); pSpcCdCtrlHdr->cipher_state_sz = LAC_BYTES_TO_QUADWORDS( - LAC_SYM_QAT_CIPHER_STATE_SIZE_SPC); + LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE); } /*populate the cipher request parameters */ if (CPA_STATUS_SUCCESS == status) { @@ -1525,6 +1990,7 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, if (status == CPA_STATUS_SUCCESS) { status = LacSymQat_CipherRequestParamsPopulate( + pSessionDesc, pMsg, pOpData ->cryptoStartSrcOffsetInBytes, @@ -1535,39 +2001,45 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, } } - if (CPA_STATUS_SUCCESS == status && - pSessionDesc->isSinglePass) { + if ((SPC == pSessionDesc->singlePassState) && + CPA_STATUS_SUCCESS == status) { Cpa64U aadBufferPhysAddr = 0; /* For CHACHA and AES-GCM there is an AAD buffer - * if - * aadLenInBytes is nonzero In case of AES-GMAC, - * AAD buffer - * passed in the src buffer. + * if aadLenInBytes is nonzero In case of + * AES-GMAC, AAD buffer passed in the src + * buffer. */ - if (0 != pSessionDesc->aadLenInBytes && - CPA_CY_SYM_HASH_AES_GMAC != - pSessionDesc->hashAlgorithm) { + if ((0 != aadLenInBytes && + CPA_CY_SYM_HASH_AES_GMAC != hash) || + isSpCcm) { LAC_CHECK_NULL_PARAM( pOpData->pAdditionalAuthData); + Cpa32U aadDataLen = + pSessionDesc->aadLenInBytes; + + /* In case of AES_CCM, B0 block size and + * 2 bytes of AAD len encoding need to + * be added to total AAD data len */ + if (isSpCcm) + aadDataLen += + LAC_CIPHER_CCM_AAD_OFFSET; + blockLen = LacSymQat_CipherBlockSizeBytesGet( - pSessionDesc->cipherAlgorithm); - if ((pSessionDesc->aadLenInBytes % - blockLen) != 0) { + cipher); + if ((aadDataLen % blockLen) != 0) { paddingLen = blockLen - - (pSessionDesc - ->aadLenInBytes % - blockLen); + (aadDataLen % blockLen); memset( - &pOpData->pAdditionalAuthData - [pSessionDesc - ->aadLenInBytes], + &pOpData + ->pAdditionalAuthData + [aadDataLen], 0, paddingLen); } - /* User OpData memory being used for aad + /* User OpData memory being used for AAD * buffer */ /* get the physical address */ aadBufferPhysAddr = @@ -1588,10 +2060,33 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, *)((Cpa8U *)&( pMsg->serv_specif_rqpars) + ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); - pCipherReqParams->spc_aad_addr = - aadBufferPhysAddr; - pCipherReqParams->spc_aad_sz = - pSessionDesc->aadLenInBytes; + + icp_qat_fw_la_cipher_20_req_params_t + *pCipher20ReqParams = + (void + *)((Cpa8U *)&( + pMsg->serv_specif_rqpars) + + ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); + + if (isCyGen4x(pService)) { + pCipher20ReqParams + ->spc_aad_addr = + aadBufferPhysAddr; + pCipher20ReqParams->spc_aad_sz = + pSessionDesc->aadLenInBytes; + pCipher20ReqParams + ->spc_aad_offset = 0; + if (isSpCcm) + pCipher20ReqParams + ->spc_aad_sz += + LAC_CIPHER_CCM_AAD_OFFSET; + } else { + pCipherReqParams->spc_aad_addr = + aadBufferPhysAddr; + pCipherReqParams->spc_aad_sz = + (Cpa16U)pSessionDesc + ->aadLenInBytes; + } if (CPA_TRUE != pSessionDesc->digestIsAppended) { @@ -1605,13 +2100,24 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, ->generic_service_info, pOpData->pDigestResult); if (0 != digestBufferPhysAddr) { - pCipherReqParams - ->spc_auth_res_addr = - digestBufferPhysAddr; - pCipherReqParams - ->spc_auth_res_sz = - pSessionDesc - ->hashResultSize; + if (isCyGen4x( + pService)) { + pCipher20ReqParams + ->spc_auth_res_addr = + digestBufferPhysAddr; + pCipher20ReqParams + ->spc_auth_res_sz = + (Cpa8U)pSessionDesc + ->hashResultSize; + } else { + pCipherReqParams + ->spc_auth_res_addr = + digestBufferPhysAddr; + pCipherReqParams + ->spc_auth_res_sz = + (Cpa8U)pSessionDesc + ->hashResultSize; + } } else { LAC_LOG_ERROR( "Unable to get the physical address " @@ -1619,6 +2125,35 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, status = CPA_STATUS_FAIL; } + } else { + /* Check if the dest buffer can + * handle the digest, only for + * last packet */ + if (((ICP_QAT_FW_LA_PARTIAL_NONE == + qatPacketType) || + (ICP_QAT_FW_LA_PARTIAL_END == + qatPacketType))) { + if (dstPktSize < + (pOpData + ->cryptoStartSrcOffsetInBytes + + pOpData + ->messageLenToCipherInBytes + + pSessionDesc + ->hashResultSize)) + status = + CPA_STATUS_INVALID_PARAM; + } + if (isCyGen4x(pService)) { + pCipher20ReqParams + ->spc_auth_res_sz = + (Cpa8U)pSessionDesc + ->hashResultSize; + } else { + pCipherReqParams + ->spc_auth_res_sz = + (Cpa8U)pSessionDesc + ->hashResultSize; + } } } } @@ -1683,8 +2218,8 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, pHashReqParams->u2.aad_sz); /* Overwrite hash state buffer info - * structure pointer - * with the one created for CCM/GCM */ + * structure pointer with the one + * created for CCM/GCM */ pHashStateBufferInfo = &hashStateBufferInfo; @@ -1692,10 +2227,8 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, * case */ if (0 == hashStateBufferInfo.pDataPhys && - CPA_CY_SYM_HASH_AES_GCM != - pSessionDesc->hashAlgorithm && - CPA_CY_SYM_HASH_AES_GMAC != - pSessionDesc->hashAlgorithm) { + CPA_CY_SYM_HASH_AES_GCM != hash && + CPA_CY_SYM_HASH_AES_GMAC != hash) { LAC_LOG_ERROR( "Unable to get the physical address" "of the AAD\n"); @@ -1703,31 +2236,26 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, } /* for CCM/GCM the hash and cipher data - * regions - * are equal */ + * regions are equal */ authOffsetInBytes = pOpData ->cryptoStartSrcOffsetInBytes; /* For authenticated encryption, - * authentication length is - * determined by - * messageLenToCipherInBytes for AES-GCM - * and - * AES-CCM, and by + * authentication length is determined + * by messageLenToCipherInBytes for + * AES-GCM and AES-CCM, and by * messageLenToHashInBytes for AES-GMAC. * You don't see the latter here, as - * that is the initial - * value of authLenInBytes. */ - if (pSessionDesc->hashAlgorithm != - CPA_CY_SYM_HASH_AES_GMAC) + * that is the initial value of + * authLenInBytes. */ + if (hash != CPA_CY_SYM_HASH_AES_GMAC) authLenInBytes = pOpData ->messageLenToCipherInBytes; } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == - pSessionDesc->hashAlgorithm || - CPA_CY_SYM_HASH_ZUC_EIA3 == - pSessionDesc->hashAlgorithm) { + hash || + CPA_CY_SYM_HASH_ZUC_EIA3 == hash) { hashStateBufferInfo.pData = pOpData->pAdditionalAuthData; hashStateBufferInfo.pDataPhys = @@ -1739,7 +2267,7 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, hashStateBufferInfo .prefixAadSzQuadWords = LAC_BYTES_TO_QUADWORDS( - pSessionDesc->aadLenInBytes); + aadLenInBytes); pHashStateBufferInfo = &hashStateBufferInfo; @@ -1752,23 +2280,22 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, status = CPA_STATUS_FAIL; } } - if (CPA_CY_SYM_HASH_AES_CCM == - pSessionDesc->hashAlgorithm) { + if (CPA_CY_SYM_HASH_AES_CCM == hash) { if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT == pSessionDesc->cipherDirection) { /* On a decrypt path pSrcBuffer - * is used as this is - * where encrypted digest is - * located. Firmware - * uses encrypted digest for + * is used as this is where + * encrypted digest is located. + * Firmware uses encrypted + * digest for * compare/verification*/ pBufferList = (CpaBufferList *)pSrcBuffer; } else { /* On an encrypt path pDstBuffer - * is used as this is - * where encrypted digest will - * be written */ + * is used as this is where + * encrypted digest will be + * written */ pBufferList = (CpaBufferList *)pDstBuffer; } @@ -1787,21 +2314,29 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, pDigestResult = pOpData->pDigestResult; } + if (CPA_TRUE == + pSessionDesc->useStatefulSha3ContentDesc) { + LacAlgChain_StatefulSha3_SkipStateLoadFlags( + pMsg, + qatPacketType, + pSessionDesc->qatHashMode); + } + if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation) { /* In alg chaining mode, packets are not - * seen as partials - * for hash operations. Override to - * NONE. + * seen as partials for hash operations. + * Override to NONE. */ qatPacketType = ICP_QAT_FW_LA_PARTIAL_NONE; } - if (CPA_TRUE == - pSessionDesc->digestIsAppended) { + digestIsAppended = + pSessionDesc->digestIsAppended; + if (CPA_TRUE == digestIsAppended) { /*Check if the destination buffer can - * handle the digest - * if digestIsAppend is true*/ + * handle the digest if digestIsAppend + * is true*/ if (srcPktSize < (authOffsetInBytes + authLenInBytes + @@ -1824,10 +2359,10 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, qatPacketType, pSessionDesc->hashResultSize, pSessionDesc->digestVerify, - pSessionDesc->digestIsAppended ? + digestIsAppended ? NULL : pDigestResult, - pSessionDesc->hashAlgorithm, + hash, NULL); } } diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c index 70446ef988b0..050a237208e9 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c @@ -55,8 +55,9 @@ #include "lac_sal_types_crypto.h" #include "sal_service_state.h" -#define IS_EXT_ALG_CHAIN_UNSUPPORTED( \ - cipherAlgorithm, hashAlgorithm, extAlgchainSupported) \ +#define IS_EXT_ALG_CHAIN_UNSUPPORTED(cipherAlgorithm, \ + hashAlgorithm, \ + extAlgchainSupported) \ ((((CPA_CY_SYM_CIPHER_ZUC_EEA3 == cipherAlgorithm || \ CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == cipherAlgorithm) && \ CPA_CY_SYM_HASH_AES_CMAC == hashAlgorithm) || \ @@ -77,6 +78,10 @@ LacSymPerform_BufferParamCheck(const CpaBufferList *const pSrcBuffer, const lac_session_desc_t *const pSessionDesc, const CpaCySymOpData *const pOpData); +void LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, + icp_qat_fw_la_bulk_req_t *pCurrentQatMsg); +void LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest, + icp_qat_fw_la_bulk_req_t *pCurrentQatMsg); void getCtxSize(const CpaCySymSessionSetupData *pSessionSetupData, Cpa32U *pSessionCtxSizeInBytes); @@ -151,45 +156,40 @@ LacSymSession_ParamCheck(const CpaInstanceHandle instanceHandle, /* Protect against value of cipher outside the bitmap * and check if cipher algorithm is correct */ - if ((pCipherSetupData->cipherAlgorithm >= - CPA_CY_SYM_CIPHER_CAP_BITMAP_SIZE) || - (!CPA_BITMAP_BIT_TEST(capInfo.ciphers, - pCipherSetupData->cipherAlgorithm))) { + if (pCipherSetupData->cipherAlgorithm >= + CPA_CY_SYM_CIPHER_CAP_BITMAP_SIZE) { LAC_INVALID_PARAM_LOG("cipherAlgorithm"); return CPA_STATUS_INVALID_PARAM; } + if (!CPA_BITMAP_BIT_TEST(capInfo.ciphers, + pCipherSetupData->cipherAlgorithm)) { + LAC_UNSUPPORTED_PARAM_LOG( + "UnSupported cipherAlgorithm"); + return CPA_STATUS_UNSUPPORTED; + } } /* Ensure hash algorithm is correct and supported */ if ((CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionSetupData->symOperation) || (CPA_CY_SYM_OP_HASH == pSessionSetupData->symOperation)) { - /* Ensure SHAKE algorithms are not supported */ - if ((CPA_CY_SYM_HASH_SHAKE_128 == - pHashSetupData->hashAlgorithm) || - (CPA_CY_SYM_HASH_SHAKE_256 == - pHashSetupData->hashAlgorithm)) { - LAC_INVALID_PARAM_LOG( - "Hash algorithms SHAKE-128 and SHAKE-256 " - "are not supported."); - return CPA_STATUS_UNSUPPORTED; - } - /* Protect against value of hash outside the bitmap * and check if hash algorithm is correct */ - if ((pHashSetupData->hashAlgorithm >= - CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) || - (!CPA_BITMAP_BIT_TEST(capInfo.hashes, - pHashSetupData->hashAlgorithm))) { + if (pHashSetupData->hashAlgorithm >= + CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) { LAC_INVALID_PARAM_LOG("hashAlgorithm"); return CPA_STATUS_INVALID_PARAM; } + if (!CPA_BITMAP_BIT_TEST(capInfo.hashes, + pHashSetupData->hashAlgorithm)) { + LAC_UNSUPPORTED_PARAM_LOG("UnSupported hashAlgorithm"); + return CPA_STATUS_UNSUPPORTED; + } } /* ensure CCM, GCM, Kasumi, Snow3G and ZUC cipher and hash algorithms - * are - * selected together for Algorithm Chaining */ + * are selected together for Algorithm Chaining */ if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionSetupData->symOperation) { /* ensure both hash and cipher algorithms are POLY and CHACHA */ @@ -435,7 +435,11 @@ LacSymPerform_BufferParamCheck(const CpaBufferList *const pSrcBuffer, } } /* Check that src Buffer and dst Buffer Lengths are equal */ - if (srcBufferLen != dstBufferLen) { + /* CCM output needs to be longer than input buffer for appending + * tag*/ + if (srcBufferLen != dstBufferLen && + pSessionDesc->cipherAlgorithm != + CPA_CY_SYM_CIPHER_AES_CCM) { LAC_INVALID_PARAM_LOG( "Source and Dest buffer lengths need to be equal "); return CPA_STATUS_INVALID_PARAM; @@ -451,8 +455,7 @@ LacSymPerform_BufferParamCheck(const CpaBufferList *const pSrcBuffer, return CPA_STATUS_INVALID_PARAM; } else { /* This function checks to see if the partial packet - * sequence - * is correct */ + * sequence is correct */ if (CPA_STATUS_SUCCESS != LacSym_PartialPacketStateCheck( pOpData->packetType, @@ -551,7 +554,7 @@ LacSym_InitSession(const CpaInstanceHandle instanceHandle, const CpaCySymCipherSetupData *pCipherSetupData = NULL; const CpaCySymHashSetupData *pHashSetupData = NULL; -/* Instance param checking done by calling function */ + /* Instance param checking done by calling function */ LAC_CHECK_NULL_PARAM(pSessionSetupData); LAC_CHECK_NULL_PARAM(pSessionCtx); @@ -580,7 +583,7 @@ LacSym_InitSession(const CpaInstanceHandle instanceHandle, if (0 == physAddress) { LAC_LOG_ERROR( - "Unable to get the physical address of the session"); + "Unable to get the physical address of the session\n"); return CPA_STATUS_FAIL; } @@ -631,8 +634,8 @@ LacSym_InitSession(const CpaInstanceHandle instanceHandle, /* For asynchronous - use the user supplied callback * for synchronous - use the internal synchronous callback */ pSessionDesc->pSymCb = ((void *)NULL != (void *)pSymCb) ? - pSymCb : - LacSync_GenBufListVerifyCb; + pSymCb : + LacSync_GenBufListVerifyCb; } pSessionDesc->isDPSession = isDPSession; @@ -649,10 +652,8 @@ LacSym_InitSession(const CpaInstanceHandle instanceHandle, if (CPA_STATUS_SUCCESS == status) { /* Session set up via API call (not internal one) */ /* Services such as DRBG call the crypto api as part of their - * service - * hence the need to for the flag, it is needed to distinguish - * between - * an internal and external session. + * service hence the need to for the flag, it is needed to + * distinguish between an internal and external session. */ pSessionDesc->internalSession = CPA_FALSE; @@ -697,14 +698,11 @@ cpaCySymRemoveSession(const CpaInstanceHandle instanceHandle_in, /* * Based on one instance, we can initialize multiple sessions. * For example, we can initialize the session "X" and session - * "Y" with - * the same instance "A". If there is no operation pending for - * session - * "X", we can remove the session "X". + * "Y" with the same instance "A". If there is no operation + * pending for session "X", we can remove the session "X". * * Now we only check the @pSessionDesc->pendingDpCbCount, if it - * becomes - * zero, we can remove the session. + * becomes zero, we can remove the session. * * Why? * (1) We increase it in the cpaCySymDpEnqueueOp/ @@ -713,12 +711,10 @@ cpaCySymRemoveSession(const CpaInstanceHandle instanceHandle_in, * * If the @pSessionDesc->pendingDpCbCount becomes zero, it means * there is no operation pending for the session "X" anymore, so - * we can - * remove this session. Maybe there is still some requests left - * in the - * instance's ring (icp_adf_queueDataToSend() returns true), but - * the - * request does not belong to "X", it belongs to session "Y". + * we can remove this session. Maybe there is still some + * requests left in the instance's ring + * (icp_adf_queueDataToSend() returns true), but the request + * does not belong to "X", it belongs to session "Y". */ numPendingRequests = qatUtilsAtomicGet(&(pSessionDesc->u.pendingDpCbCount)); @@ -734,8 +730,7 @@ cpaCySymRemoveSession(const CpaInstanceHandle instanceHandle_in, status = CPA_STATUS_RETRY; if (CPA_TRUE == pSessionDesc->isDPSession) { /* Need to update tail if messages queue on tx hi ring - for - data plane api */ + for data plane api */ icp_comms_trans_handle trans_handle = ((sal_crypto_service_t *)instanceHandle) ->trans_handle_sym_tx; @@ -752,10 +747,7 @@ cpaCySymRemoveSession(const CpaInstanceHandle instanceHandle_in, } } if (CPA_STATUS_SUCCESS == status) { - if (CPA_STATUS_SUCCESS != - LAC_SPINLOCK_DESTROY(&pSessionDesc->requestQueueLock)) { - LAC_LOG_ERROR("Failed to destroy request queue lock"); - } + LAC_SPINLOCK_DESTROY(&pSessionDesc->requestQueueLock); if (CPA_FALSE == pSessionDesc->isDPSession) { LAC_SYM_STAT_INC(numSessionsRemoved, instanceHandle); } diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c index 01e9eca1f10a..2b95dd8cf6fe 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c @@ -69,7 +69,7 @@ LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc, const CpaCySymOpData *pOpData, CpaBoolean isCCM) { - Cpa8U authTagLen = 0; + Cpa32U authTagLen = 0; /* Retrieve authTagLen */ authTagLen = pSessionDesc->hashResultSize; @@ -138,10 +138,10 @@ LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, /* For a digest verify operation - for full packet and final partial * only, perform a comparison with the digest generated and with the one - * supplied in the packet. */ + * supplied in the packet. In case of AES_GCM in SPC mode, destination + * buffer needs to be cleared if digest verify operation fails */ - if (((pSessionDesc->isSinglePass && - (CPA_CY_SYM_CIPHER_AES_GCM == pSessionDesc->cipherAlgorithm)) || + if (((SPC == pSessionDesc->singlePassState) || (CPA_CY_SYM_OP_CIPHER != operationType)) && (CPA_TRUE == pSessionDesc->digestVerify) && ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) || @@ -151,14 +151,10 @@ LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, instanceHandle); /* The comparison has failed at this point (status is - * fail), - * need to clean any sensitive calculated data up to - * this point. - * The data calculated is no longer useful to the end - * result and - * does not need to be returned to the user so setting - * buffers to - * zero. + * fail), need to clean any sensitive calculated data up + * to this point. The data calculated is no longer + * useful to the end result and does not need to be + * returned to the user so setting buffers to zero. */ if (pSessionDesc->cipherAlgorithm == CPA_CY_SYM_CIPHER_AES_CCM) { @@ -200,8 +196,7 @@ LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, /* Update the user's IV buffer * Very important to do this BEFORE dequeuing * subsequent partial requests, as the state - * buffer - * may get overwritten + * buffer may get overwritten */ memcpy(pCookie->pOpData->pIv, pSessionDesc->cipherPartialOpState, @@ -218,8 +213,9 @@ LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, } else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) { if ((CPA_CY_SYM_OP_CIPHER == operationType) || (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { - if (CPA_TRUE == LAC_CIPHER_IS_XTS_MODE( - pSessionDesc->cipherAlgorithm)) { + if (CPA_TRUE == + LAC_CIPHER_IS_XTS_MODE( + pSessionDesc->cipherAlgorithm)) { /* * For XTS mode, we replace the updated key with * the original key - for subsequent partial @@ -294,51 +290,46 @@ LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, static void LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse, CpaBoolean qatRespStatusOkFlag, + CpaStatus status, lac_session_desc_t *pSessionDesc) { - CpaStatus status = CPA_STATUS_SUCCESS; + CpaCySymDpCbFunc pSymDpCb = NULL; /* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be * cleaned as stated in RFC 3610; in DP mode, it is the user - * responsability - * to do so */ + * responsability to do so */ - if (CPA_FALSE == pSessionDesc->isSinglePass) { - if ((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) || - (CPA_FALSE == pSessionDesc->digestVerify)) { - /* If not doing digest compare and qatRespStatusOkFlag - != - CPA_TRUE - then there is something very wrong */ - if (CPA_FALSE == qatRespStatusOkFlag) { - LAC_LOG_ERROR( - "Response status value not as expected"); - status = CPA_STATUS_FAIL; - } + if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) && + SPC != pSessionDesc->singlePassState) || + (CPA_FALSE == pSessionDesc->digestVerify)) { + /* If not doing digest compare and qatRespStatusOkFlag != + CPA_TRUE then there is something very wrong */ + if ((CPA_FALSE == qatRespStatusOkFlag) && + (status != CPA_STATUS_UNSUPPORTED)) { + LAC_LOG_ERROR("Response status value not as expected"); + status = CPA_STATUS_FAIL; } } - ((sal_crypto_service_t *)pResponse->instanceHandle) - ->pSymDpCb(pResponse, status, qatRespStatusOkFlag); + pSymDpCb = + ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb; + + pSymDpCb(pResponse, status, qatRespStatusOkFlag); + /* * Decrement the number of pending CB. * * If the @pendingDpCbCount becomes zero, we may remove the session, - * please - * read more information in the cpaCySymRemoveSession(). + * please read more information in the cpaCySymRemoveSession(). * * But there is a field in the @pResponse to store the session, * the "sessionCtx". In another word, in the above @->pSymDpCb() - * callback, - * it may use the session again. If we decrease the @pendingDpCbCount - * before - * the @->pSymDpCb(), there is a _risk_ the @->pSymDpCb() may reference - * to - * a deleted session. + * callback, it may use the session again. If we decrease the + * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the + * @->pSymDpCb() may reference to a deleted session. * * So in order to avoid the risk, we decrease the @pendingDpCbCount - * after - * the @->pSymDpCb() callback. + * after the @->pSymDpCb() callback. */ qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount); } @@ -367,6 +358,7 @@ LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId, void *pOpaqueData, icp_qat_fw_comn_flags cmnRespFlags) { + CpaStatus status = CPA_STATUS_SUCCESS; CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData; lac_session_desc_t *pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx); @@ -376,8 +368,13 @@ LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId, if (CPA_TRUE == pSessionDesc->isDPSession) { /* DP session */ + if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET( + cmnRespFlags)) { + status = CPA_STATUS_UNSUPPORTED; + } LacSymCb_ProcessDpCallback(pDpOpData, qatRespStatusOkFlag, + status, pSessionDesc); } else { /* Trad session */ @@ -414,11 +411,7 @@ LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) * be accessed by multiple contexts simultaneously for enqueue and * dequeue operations */ - if (CPA_STATUS_SUCCESS != - LAC_SPINLOCK(&pSessionDesc->requestQueueLock)) { - LAC_LOG_ERROR("Failed to lock request queue"); - return CPA_STATUS_RESOURCE; - } + LAC_SPINLOCK(&pSessionDesc->requestQueueLock); /* Clear the blocking flag in the session descriptor */ pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE; @@ -427,10 +420,9 @@ LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) { /* If we send a partial packet request, set the - * blockingOpsInProgress - * flag for the session to indicate that subsequent requests - * must be - * queued up until this request completes + * blockingOpsInProgress flag for the session to indicate that + * subsequent requests must be queued up until this request + * completes */ if (CPA_CY_SYM_PACKET_TYPE_FULL != pSessionDesc->pRequestQueueHead->pOpData->packetType) { @@ -438,14 +430,10 @@ LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) } /* At this point, we're clear to send the request. For cipher - * requests, - * we need to check if the session IV needs to be updated. This - * can - * only be done when no other partials are in flight for this - * session, - * to ensure the cipherPartialOpState buffer in the session - * descriptor - * is not currently in use + * requests, we need to check if the session IV needs to be + * updated. This can only be done when no other partials are in + * flight for this session, to ensure the cipherPartialOpState + * buffer in the session descriptor is not currently in use */ if (CPA_TRUE == pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) { @@ -464,13 +452,11 @@ LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) /* * Now we'll attempt to send the message directly to QAT. We'll - * keep - * looing until it succeeds (or at least a very high number of - * retries), - * as the failure only happens when the ring is full, and this - * is only - * a temporary situation. After a few retries, space will become - * availble, allowing the putMsg to succeed. + * keep looing until it succeeds (or at least a very high number + * of retries), as the failure only happens when the ring is + * full, and this is only a temporary situation. After a few + * retries, space will become availble, allowing the putMsg to + * succeed. */ retries = 0; do { @@ -483,8 +469,7 @@ LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) retries++; /* * Yield to allow other threads that may be on this - * session to poll - * and make some space on the ring + * session to poll and make some space on the ring */ if (CPA_STATUS_SUCCESS != status) { qatUtilsYield(); @@ -509,10 +494,7 @@ LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) } cleanup: - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock)) { - LAC_LOG_ERROR("Failed to unlock request queue"); - } + LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock); return status; } diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c index 0039c1f66884..2a6cdd7aa748 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c @@ -49,6 +49,7 @@ #include "lac_sym_qat_cipher.h" #include "lac_log.h" #include "lac_buffer_desc.h" +#include "sal_hw_gen.h" /* ******************************************************************************* @@ -66,14 +67,23 @@ LacCipher_PerformIvCheck(sal_service_t *pService, lac_session_desc_t *pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pOpData->sessionCtx); CpaCySymCipherAlgorithm algorithm = pSessionDesc->cipherAlgorithm; + unsigned ivLenInBytes = 0; - /* Perform IV check. */ - if (LAC_CIPHER_IS_CTR_MODE(algorithm) || - LAC_CIPHER_IS_CBC_MODE(algorithm) || - LAC_CIPHER_IS_AES_F8(algorithm) || - LAC_CIPHER_IS_XTS_MODE(algorithm)) { - unsigned ivLenInBytes = - LacSymQat_CipherIvSizeBytesGet(algorithm); + switch (algorithm) { + /* Perform IV check for CTR, CBC, XTS, F8 MODE. */ + case CPA_CY_SYM_CIPHER_AES_CTR: + case CPA_CY_SYM_CIPHER_3DES_CTR: + case CPA_CY_SYM_CIPHER_SM4_CTR: + case CPA_CY_SYM_CIPHER_AES_CCM: + case CPA_CY_SYM_CIPHER_AES_GCM: + case CPA_CY_SYM_CIPHER_CHACHA: + case CPA_CY_SYM_CIPHER_AES_CBC: + case CPA_CY_SYM_CIPHER_DES_CBC: + case CPA_CY_SYM_CIPHER_3DES_CBC: + case CPA_CY_SYM_CIPHER_SM4_CBC: + case CPA_CY_SYM_CIPHER_AES_F8: + case CPA_CY_SYM_CIPHER_AES_XTS: { + ivLenInBytes = LacSymQat_CipherIvSizeBytesGet(algorithm); LAC_CHECK_NULL_PARAM(pOpData->pIv); if (pOpData->ivLenInBytes != ivLenInBytes) { if (!(/* GCM with 12 byte IV is OK */ @@ -96,26 +106,21 @@ LacCipher_PerformIvCheck(sal_service_t *pService, /* Set the value of the ppIvBuffer to that supplied * by the user. * NOTE: There is no guarantee that this address is - * aligned on - * an 8 or 64 Byte address. */ + * aligned on an 8 or 64 Byte address. */ *ppIvBuffer = pOpData->pIv; } else { /* For partial packets, we use a per-session buffer to - * maintain - * the IV. This allows us to easily pass the updated IV - * forward - * to the next partial in the sequence. This makes - * internal - * buffering of partials easier to implement. + * maintain the IV. This allows us to easily pass the + * updated IV forward to the next partial in the + * sequence. This makes internal buffering of partials + * easier to implement. */ *ppIvBuffer = pSessionDesc->cipherPartialOpState; /* Ensure that the user's IV buffer gets updated between - * partial - * requests so that they may also see the residue from - * the - * previous partial. Not needed for final partials - * though. + * partial requests so that they may also see the + * residue from the previous partial. Not needed for + * final partials though. */ if ((ICP_QAT_FW_LA_PARTIAL_START == qatPacketType) || (ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType)) { @@ -124,97 +129,87 @@ LacCipher_PerformIvCheck(sal_service_t *pService, if (ICP_QAT_FW_LA_PARTIAL_START == qatPacketType) { /* if the previous partial state was - * full, then this is - * the first partial in the sequence so - * we need to copy - * in the user's IV. But, we have to be - * very careful - * here not to overwrite the - * cipherPartialOpState just - * yet in case there's a previous - * partial sequence in - * flight, so we defer the copy for now. - * This will be - * completed in the - * LacSymQueue_RequestSend() function. + * full, then this is the first partial + * in the sequence so we need to copy in + * the user's IV. But, we have to be + * very careful here not to overwrite + * the cipherPartialOpState just yet in + * case there's a previous partial + * sequence in flight, so we defer the + * copy for now. This will be completed + * in the LacSymQueue_RequestSend() + * function. */ pCbCookie->updateSessionIvOnSend = CPA_TRUE; } /* For subsequent partials in a sequence, we'll - * re-use the - * IV that was written back by the QAT, using - * internal - * request queueing if necessary to ensure that - * the next - * partial request isn't issued to the QAT until - * the + * re-use the IV that was written back by the + * QAT, using internal request queueing if + * necessary to ensure that the next partial + * request isn't issued to the QAT until the * previous one completes */ } } - } else if (LAC_CIPHER_IS_KASUMI(algorithm)) { + } break; + case CPA_CY_SYM_CIPHER_KASUMI_F8: { LAC_CHECK_NULL_PARAM(pOpData->pIv); - if (LAC_CIPHER_IS_KASUMI(algorithm) && - (pOpData->ivLenInBytes != LAC_CIPHER_KASUMI_F8_IV_LENGTH)) { + if (pOpData->ivLenInBytes != LAC_CIPHER_KASUMI_F8_IV_LENGTH) { LAC_INVALID_PARAM_LOG("invalid cipher IV size"); return CPA_STATUS_INVALID_PARAM; } - *ppIvBuffer = pOpData->pIv; - } else if (LAC_CIPHER_IS_SNOW3G_UEA2(algorithm)) { + } break; + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: { LAC_CHECK_NULL_PARAM(pOpData->pIv); - if (LAC_CIPHER_IS_SNOW3G_UEA2(algorithm) && - (pOpData->ivLenInBytes != ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ)) { + if (pOpData->ivLenInBytes != ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ) { LAC_INVALID_PARAM_LOG("invalid cipher IV size"); return CPA_STATUS_INVALID_PARAM; } *ppIvBuffer = pOpData->pIv; - } else if (LAC_CIPHER_IS_ARC4(algorithm)) { + } break; + case CPA_CY_SYM_CIPHER_ARC4: { if (ICP_QAT_FW_LA_PARTIAL_NONE == qatPacketType) { /* For full packets, the initial ARC4 state is stored in - * the - * session descriptor. Use it directly. + * the session descriptor. Use it directly. */ *ppIvBuffer = pSessionDesc->cipherARC4InitialState; } else { /* For partial packets, we maintain the running ARC4 - * state in - * dedicated buffer in the session descriptor + * state in dedicated buffer in the session descriptor */ *ppIvBuffer = pSessionDesc->cipherPartialOpState; if (ICP_QAT_FW_LA_PARTIAL_START == qatPacketType) { /* if the previous partial state was full, then - * this is the - * first partial in the sequence so we need to - * (re-)initialise - * the contents of the state buffer using the - * initial state - * that is stored in the session descriptor. - * But, we have to be - * very careful here not to overwrite the - * cipherPartialOpState - * just yet in case there's a previous partial - * sequence in + * this is the first partial in the sequence so + * we need to (re-)initialise the contents of + * the state buffer using the initial state that + * is stored in the session descriptor. But, we + * have to be very careful here not to overwrite + * the cipherPartialOpState just yet in case + * there's a previous partial sequence in * flight, so we defer the copy for now. This - * will be completed - * in the LacSymQueue_RequestSend() function - * when clear to send. + * will be completed in the + * LacSymQueue_RequestSend() function when clear + * to send. */ pCbCookie->updateSessionIvOnSend = CPA_TRUE; } } - } else if (LAC_CIPHER_IS_ZUC_EEA3(algorithm)) { + } break; + case CPA_CY_SYM_CIPHER_ZUC_EEA3: { LAC_CHECK_NULL_PARAM(pOpData->pIv); if (pOpData->ivLenInBytes != ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ) { LAC_INVALID_PARAM_LOG("invalid cipher IV size"); return CPA_STATUS_INVALID_PARAM; } *ppIvBuffer = pOpData->pIv; - } else { + } break; + default: *ppIvBuffer = NULL; } @@ -223,40 +218,50 @@ LacCipher_PerformIvCheck(sal_service_t *pService, CpaStatus -LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData) +LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData, + Cpa32U capabilitiesMask) { /* No key required for NULL algorithm */ if (!LAC_CIPHER_IS_NULL(pCipherSetupData->cipherAlgorithm)) { LAC_CHECK_NULL_PARAM(pCipherSetupData->pCipherKey); /* Check that algorithm and keys passed in are correct size */ - if (LAC_CIPHER_IS_ARC4(pCipherSetupData->cipherAlgorithm)) { + switch (pCipherSetupData->cipherAlgorithm) { + case CPA_CY_SYM_CIPHER_ARC4: if (pCipherSetupData->cipherKeyLenInBytes > ICP_QAT_HW_ARC4_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid ARC4 cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_CCM( - pCipherSetupData->cipherAlgorithm)) { - if (pCipherSetupData->cipherKeyLenInBytes != - ICP_QAT_HW_AES_128_KEY_SZ) { + break; + case CPA_CY_SYM_CIPHER_AES_CCM: + if (!LAC_CIPHER_AES_V2(capabilitiesMask) && + pCipherSetupData->cipherKeyLenInBytes != + ICP_QAT_HW_AES_128_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid AES CCM cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_XTS_MODE( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_AES_XTS: if ((pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_AES_128_XTS_KEY_SZ) && (pCipherSetupData->cipherKeyLenInBytes != - ICP_QAT_HW_AES_256_XTS_KEY_SZ)) { + ICP_QAT_HW_AES_256_XTS_KEY_SZ) && + (pCipherSetupData->cipherKeyLenInBytes != + ICP_QAT_HW_UCS_AES_128_XTS_KEY_SZ) && + (pCipherSetupData->cipherKeyLenInBytes != + ICP_QAT_HW_UCS_AES_256_XTS_KEY_SZ)) { LAC_INVALID_PARAM_LOG( "Invalid AES XTS cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_AES( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_AES_ECB: + case CPA_CY_SYM_CIPHER_AES_CBC: + case CPA_CY_SYM_CIPHER_AES_CTR: + case CPA_CY_SYM_CIPHER_AES_GCM: if ((pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_AES_128_KEY_SZ) && (pCipherSetupData->cipherKeyLenInBytes != @@ -267,8 +272,8 @@ LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData) "Invalid AES cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_AES_F8( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_AES_F8: if ((pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_AES_128_F8_KEY_SZ) && (pCipherSetupData->cipherKeyLenInBytes != @@ -279,35 +284,37 @@ LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData) "Invalid AES cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_DES( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_DES_ECB: + case CPA_CY_SYM_CIPHER_DES_CBC: if (pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_DES_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid DES cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_TRIPLE_DES( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_3DES_ECB: + case CPA_CY_SYM_CIPHER_3DES_CBC: + case CPA_CY_SYM_CIPHER_3DES_CTR: if (pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_3DES_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid Triple-DES cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_KASUMI( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_KASUMI_F8: /* QAT-FW only supports 128 bits Cipher Key size for - * Kasumi F8 - * Ref: 3GPP TS 55.216 V6.2.0 */ + * Kasumi F8 Ref: 3GPP TS 55.216 V6.2.0 */ if (pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_KASUMI_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid Kasumi cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_SNOW3G_UEA2( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: /* QAT-FW only supports 256 bits Cipher Key size for * Snow_3G */ if (pCipherSetupData->cipherKeyLenInBytes != @@ -316,8 +323,8 @@ LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData) "Invalid Snow_3G cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_ZUC_EEA3( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_ZUC_EEA3: /* ZUC EEA3 */ if (pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ) { @@ -325,23 +332,26 @@ LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData) "Invalid ZUC cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_CHACHA( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_CHACHA: if (pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_CHACHAPOLY_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid CHACHAPOLY cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_SM4( - pCipherSetupData->cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_SM4_ECB: + case CPA_CY_SYM_CIPHER_SM4_CBC: + case CPA_CY_SYM_CIPHER_SM4_CTR: if (pCipherSetupData->cipherKeyLenInBytes != ICP_QAT_HW_SM4_KEY_SZ) { LAC_INVALID_PARAM_LOG( "Invalid SM4 cipher key length"); return CPA_STATUS_INVALID_PARAM; } - } else { + break; + default: LAC_INVALID_PARAM_LOG("Invalid cipher algorithm"); return CPA_STATUS_INVALID_PARAM; } @@ -365,52 +375,86 @@ LacCipher_PerformParamCheck(CpaCySymCipherAlgorithm algorithm, LAC_INVALID_PARAM_LOG("cipher len + offset greater than " "srcBuffer packet len"); status = CPA_STATUS_INVALID_PARAM; - } - - if (CPA_STATUS_SUCCESS == status) { + } else { + /* Perform algorithm-specific checks */ + switch (algorithm) { + case CPA_CY_SYM_CIPHER_ARC4: + case CPA_CY_SYM_CIPHER_AES_CTR: + case CPA_CY_SYM_CIPHER_3DES_CTR: + case CPA_CY_SYM_CIPHER_SM4_CTR: + case CPA_CY_SYM_CIPHER_AES_CCM: + case CPA_CY_SYM_CIPHER_AES_GCM: + case CPA_CY_SYM_CIPHER_CHACHA: + case CPA_CY_SYM_CIPHER_KASUMI_F8: + case CPA_CY_SYM_CIPHER_AES_F8: + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: + case CPA_CY_SYM_CIPHER_ZUC_EEA3: + /* No action needed */ + break; /* * XTS Mode allow for ciphers which are not multiples of * the block size. */ - /* Perform algorithm-specific checks */ - if (LAC_CIPHER_IS_XTS_MODE(algorithm) && - ((pOpData->packetType == CPA_CY_SYM_PACKET_TYPE_FULL) || - (pOpData->packetType == - CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL))) { - /* - * If this is the last of a partial request - */ - if (pOpData->messageLenToCipherInBytes < - ICP_QAT_HW_AES_BLK_SZ) { - LAC_INVALID_PARAM_LOG( - "data size must be greater than block " - "size for last XTS partial or XTS " - "full packet"); - status = CPA_STATUS_INVALID_PARAM; + case CPA_CY_SYM_CIPHER_AES_XTS: + if ((pOpData->packetType == + CPA_CY_SYM_PACKET_TYPE_FULL) || + (pOpData->packetType == + CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL)) { + /* + * If this is the last of a partial request + */ + if (pOpData->messageLenToCipherInBytes < + ICP_QAT_HW_AES_BLK_SZ) { + LAC_INVALID_PARAM_LOG( + "data size must be greater than block" + " size for last XTS partial or XTS " + "full packet"); + status = CPA_STATUS_INVALID_PARAM; + } } - } else if (!(LAC_CIPHER_IS_ARC4(algorithm) || - LAC_CIPHER_IS_CTR_MODE(algorithm) || - LAC_CIPHER_IS_F8_MODE(algorithm) || - LAC_CIPHER_IS_SNOW3G_UEA2(algorithm) || - LAC_CIPHER_IS_XTS_MODE(algorithm) || - LAC_CIPHER_IS_CHACHA(algorithm) || - LAC_CIPHER_IS_ZUC_EEA3(algorithm))) { + break; + default: /* Mask & check below is based on assumption that block - * size is - * a power of 2. If data size is not a multiple of the - * block size, - * the "remainder" bits selected by the mask be non-zero + * size is a power of 2. If data size is not a multiple + * of the block size, the "remainder" bits selected by + * the mask be non-zero */ if (pOpData->messageLenToCipherInBytes & (LacSymQat_CipherBlockSizeBytesGet(algorithm) - 1)) { LAC_INVALID_PARAM_LOG( - "data size must be block size multiple"); + "data size must be block size" + " multiple"); status = CPA_STATUS_INVALID_PARAM; } } } - return status; } +Cpa32U +LacCipher_GetCipherSliceType(sal_crypto_service_t *pService, + CpaCySymCipherAlgorithm cipherAlgorithm, + CpaCySymHashAlgorithm hashAlgorithm) +{ + Cpa32U sliceType = ICP_QAT_FW_LA_USE_LEGACY_SLICE_TYPE; + Cpa32U capabilitiesMask = + pService->generic_service_info.capabilitiesMask; + + /* UCS Slice is supproted only in Gen4 */ + if (isCyGen4x(pService)) { + if (LAC_CIPHER_IS_XTS_MODE(cipherAlgorithm) || + LAC_CIPHER_IS_CHACHA(cipherAlgorithm) || + LAC_CIPHER_IS_GCM(cipherAlgorithm)) { + sliceType = ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE; + } else if (LAC_CIPHER_IS_CCM(cipherAlgorithm) && + LAC_CIPHER_AES_V2(capabilitiesMask)) { + sliceType = ICP_QAT_FW_LA_USE_LEGACY_SLICE_TYPE; + } else if (LAC_CIPHER_IS_AES(cipherAlgorithm) && + LAC_CIPHER_IS_CTR_MODE(cipherAlgorithm)) { + sliceType = ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE; + } + } + + return sliceType; +} diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c index 4ccf1f7f82d5..82add3a5dc08 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c @@ -35,14 +35,16 @@ #include "qat_utils.h" -#include "lac_mem.h" -#include "lac_log.h" -#include "lac_sym.h" -#include "lac_sym_qat_cipher.h" #include "lac_list.h" +#include "lac_log.h" +#include "lac_mem.h" #include "lac_sal_types_crypto.h" -#include "sal_service_state.h" +#include "lac_sym.h" +#include "lac_sym_cipher.h" #include "lac_sym_auth_enc.h" +#include "lac_sym_qat_cipher.h" +#include "sal_service_state.h" +#include "sal_hw_gen.h" typedef void (*write_ringMsgFunc_t)(CpaCySymDpOpData *pRequest, icp_qat_fw_la_bulk_req_t *pCurrentQatMsg); @@ -133,7 +135,7 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) /* digestVerify and digestIsAppended on Hash-Only operation not * supported */ if (pSessionDesc->digestIsAppended && pSessionDesc->digestVerify && - (CPA_CY_SYM_OP_HASH == pSessionDesc->symOperation)) { + (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH)) { LAC_INVALID_PARAM_LOG( "digestVerify and digestIsAppended set " "on Hash-Only operation is not supported"); @@ -143,10 +145,17 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) /* Cipher specific tests */ if (CPA_CY_SYM_OP_HASH != pSessionDesc->symOperation) { /* Perform IV check */ - if ((LAC_CIPHER_IS_CTR_MODE(pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_CBC_MODE(pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_AES_F8(pSessionDesc->cipherAlgorithm)) && - (!(LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)))) { + switch (pSessionDesc->cipherAlgorithm) { + case CPA_CY_SYM_CIPHER_AES_CTR: + case CPA_CY_SYM_CIPHER_3DES_CTR: + case CPA_CY_SYM_CIPHER_SM4_CTR: + case CPA_CY_SYM_CIPHER_AES_GCM: + case CPA_CY_SYM_CIPHER_CHACHA: + case CPA_CY_SYM_CIPHER_AES_CBC: + case CPA_CY_SYM_CIPHER_DES_CBC: + case CPA_CY_SYM_CIPHER_3DES_CBC: + case CPA_CY_SYM_CIPHER_SM4_CBC: + case CPA_CY_SYM_CIPHER_AES_F8: { Cpa32U ivLenInBytes = LacSymQat_CipherIvSizeBytesGet( pSessionDesc->cipherAlgorithm); if (pRequest->ivLenInBytes != ivLenInBytes) { @@ -164,11 +173,10 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) LAC_INVALID_PARAM_LOG("invalid iv of 0"); return CPA_STATUS_INVALID_PARAM; } - /* pRequest->pIv is only used for CCM so is not checked * here */ - } else if (LAC_CIPHER_IS_KASUMI( - pSessionDesc->cipherAlgorithm)) { + } break; + case CPA_CY_SYM_CIPHER_KASUMI_F8: { if (LAC_CIPHER_KASUMI_F8_IV_LENGTH != pRequest->ivLenInBytes) { LAC_INVALID_PARAM_LOG("invalid cipher IV size"); @@ -178,8 +186,8 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) LAC_INVALID_PARAM_LOG("invalid iv of 0"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_SNOW3G_UEA2( - pSessionDesc->cipherAlgorithm)) { + } break; + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: { if (ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ != pRequest->ivLenInBytes) { LAC_INVALID_PARAM_LOG("invalid cipher IV size"); @@ -189,8 +197,8 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) LAC_INVALID_PARAM_LOG("invalid iv of 0"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_ZUC_EEA3( - pSessionDesc->cipherAlgorithm)) { + } break; + case CPA_CY_SYM_CIPHER_ZUC_EEA3: { if (ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ != pRequest->ivLenInBytes) { LAC_INVALID_PARAM_LOG("invalid cipher IV size"); @@ -200,7 +208,8 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) LAC_INVALID_PARAM_LOG("invalid iv of 0"); return CPA_STATUS_INVALID_PARAM; } - } else if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) { + } break; + case CPA_CY_SYM_CIPHER_AES_CCM: { if (CPA_STATUS_SUCCESS != LacSymAlgChain_CheckCCMData( pRequest->pAdditionalAuthData, @@ -209,30 +218,42 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) pRequest->ivLenInBytes)) { return CPA_STATUS_INVALID_PARAM; } + } break; + default: + break; } - /* Perform algorithm-specific checks */ - if (!(LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_CTR_MODE(pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_F8_MODE(pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_SNOW3G_UEA2( - pSessionDesc->cipherAlgorithm) || - LAC_CIPHER_IS_ZUC_EEA3(pSessionDesc->cipherAlgorithm))) { + switch (pSessionDesc->cipherAlgorithm) { + case CPA_CY_SYM_CIPHER_ARC4: + case CPA_CY_SYM_CIPHER_AES_CTR: + case CPA_CY_SYM_CIPHER_3DES_CTR: + case CPA_CY_SYM_CIPHER_SM4_CTR: + case CPA_CY_SYM_CIPHER_AES_CCM: + case CPA_CY_SYM_CIPHER_AES_GCM: + case CPA_CY_SYM_CIPHER_CHACHA: + case CPA_CY_SYM_CIPHER_KASUMI_F8: + case CPA_CY_SYM_CIPHER_AES_F8: + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: + case CPA_CY_SYM_CIPHER_ZUC_EEA3: + /* No action needed */ + break; + default: { /* Mask & check below is based on assumption that block - * size is - * a power of 2. If data size is not a multiple of the - * block size, - * the "remainder" bits selected by the mask be non-zero + * size is a power of 2. If data size is not a multiple + * of the block size, the "remainder" bits selected by + * the mask be non-zero */ if (pRequest->messageLenToCipherInBytes & (LacSymQat_CipherBlockSizeBytesGet( pSessionDesc->cipherAlgorithm) - 1)) { LAC_INVALID_PARAM_LOG( - "Data size must be block size multiple"); + "Data size must be block size" + " multiple"); return CPA_STATUS_INVALID_PARAM; } } + } cipher = pSessionDesc->cipherAlgorithm; hash = pSessionDesc->hashAlgorithm; @@ -242,11 +263,9 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) if (LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) && (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) { /* For CHACHA and AES_GCM single pass there is an AAD - * buffer - * if aadLenInBytes is nonzero. AES_GMAC AAD is stored - * in - * source buffer, therefore there is no separate AAD - * buffer. */ + * buffer if aadLenInBytes is nonzero. AES_GMAC AAD is + * stored in source buffer, therefore there is no + * separate AAD buffer. */ if ((0 != pSessionDesc->aadLenInBytes) && (CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm)) { @@ -397,7 +416,7 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) * @ingroup cpaCySymDp * Write Message on the ring and write request params * This is the optimized version, which should not be used for - * algorithm of CCM, GCM and RC4 + * algorithm of CCM, GCM, CHACHA and RC4 * * @description * Write Message on the ring and write request params @@ -425,12 +444,15 @@ LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest, /* Write Request */ /* * Fill in the header and footer bytes of the ET ring message - cached - * from - * the session descriptor. + * from the session descriptor. */ - pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr); - pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr); - + if (!pSessionDesc->useSymConstantsTable) { + pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr); + pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr); + } else { + pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->shramReqCacheHdr); + pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->shramReqCacheFtr); + } memcpy(pMsgDummy, pCacheDummyHdr, (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)); @@ -457,6 +479,7 @@ LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest, if (pSessionDesc->isCipher) { LacSymQat_CipherRequestParamsPopulate( + pSessionDesc, pCurrentQatMsg, pRequest->cryptoStartSrcOffsetInBytes, pRequest->messageLenToCipherInBytes, @@ -493,14 +516,27 @@ LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest, * copied directly from the op request data because they share a * corresponding layout. The remaining 4 bytes are taken * from the session message template and use values - * preconfigured at - * sessionInit (updated per request for some specific cases - * below) + * preconfigured at sessionInit (updated per request for some + * specific cases below) */ - memcpy(pAuthReqPars, - (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), - ((unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) - - (unsigned long)pAuthReqPars)); + + /* We force a specific compiler optimisation here. The length + * to be copied turns out to be always 16, and by coding a + * memcpy with a literal value the compiler will compile inline + * code (in fact, only two vector instructions) to effect the + * copy. This gives us a huge performance increase. + */ + unsigned long cplen = + (unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) - + (unsigned long)pAuthReqPars; + if (cplen == 16) + memcpy(pAuthReqPars, + (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), + 16); + else + memcpy(pAuthReqPars, + (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), + cplen); if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { pAuthReqPars->hash_state_sz = @@ -548,25 +584,29 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, Cpa32U sizeInBytes = 0; CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm; CpaCySymHashAlgorithm hash = pSessionDesc->hashAlgorithm; + sal_crypto_service_t *pService = + (sal_crypto_service_t *)pRequest->instanceHandle; Cpa32U capabilitiesMask = ((sal_crypto_service_t *)pRequest->instanceHandle) ->generic_service_info.capabilitiesMask; + CpaBoolean isSpCcm = LAC_CIPHER_IS_CCM(cipher) && + LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask); + Cpa8U paddingLen = 0; Cpa8U blockLen = 0; + Cpa32U aadDataLen = 0; pMsgDummy = (Cpa8U *)pCurrentQatMsg; /* Write Request */ /* * Fill in the header and footer bytes of the ET ring message - cached - * from - * the session descriptor. + * from the session descriptor. */ - if (!pSessionDesc->isSinglePass && - LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) && - (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) { - pSessionDesc->isSinglePass = CPA_TRUE; + if ((NON_SPC != pSessionDesc->singlePassState) && + (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes))) { + pSessionDesc->singlePassState = SPC; pSessionDesc->isCipher = CPA_TRUE; pSessionDesc->isAuthEncryptOp = CPA_FALSE; pSessionDesc->isAuth = CPA_FALSE; @@ -581,7 +621,8 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, */ ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO); - ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); + if (isCyGen2x(pService)) + ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); pCdInfo = &(pSessionDesc->contentDescInfo); pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; @@ -589,11 +630,22 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, pSessionDesc->cipherDirection) { if (LAC_CIPHER_IS_GCM(cipher)) hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC); - else + LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); + else if (LAC_CIPHER_IS_CHACHA(cipher)) hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( - LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC); + LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); + } else if (isSpCcm) { + hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( + LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM); } + + /* Update slice type, as used algos changed */ + pSessionDesc->cipherSliceType = + LacCipher_GetCipherSliceType(pService, cipher, hash); + + ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags, + pSessionDesc->cipherSliceType); + /* construct cipherConfig in CD in DRAM */ LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, pHwBlockBaseInDRAM + @@ -605,10 +657,42 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, pSessionDesc->laCmdId, pSessionDesc->cmnRequestFlags, pSessionDesc->laCmdFlags); + } else if ((SPC == pSessionDesc->singlePassState) && + (LAC_CIPHER_SPC_IV_SIZE != pRequest->ivLenInBytes)) { + pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; + pSessionDesc->singlePassState = LIKELY_SPC; + pSessionDesc->isCipher = CPA_TRUE; + pSessionDesc->isAuthEncryptOp = CPA_TRUE; + pSessionDesc->isAuth = CPA_TRUE; + pCdInfo = &(pSessionDesc->contentDescInfo); + pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; + + if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == + pSessionDesc->cipherDirection) { + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + } else { + pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + } + + ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( + pSessionDesc->laCmdFlags, 0); + ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, + ICP_QAT_FW_LA_GCM_PROTO); + + LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, + pHwBlockBaseInDRAM + + hwBlockOffsetInDRAM, + &sizeInBytes); + SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( + pSessionDesc->reqCacheHdr), + ICP_QAT_FW_COMN_REQ_CPM_FW_LA, + pSessionDesc->laCmdId, + pSessionDesc->cmnRequestFlags, + pSessionDesc->laCmdFlags); } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { pSessionDesc->aadLenInBytes = pRequest->messageLenToHashInBytes; } - if (pSessionDesc->isSinglePass) { + if (SPC == pSessionDesc->singlePassState) { pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr); pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr); } else { @@ -644,8 +728,9 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, pRequest->srcBufferLen, pRequest->dstBufferLen); - if (CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm && - pSessionDesc->isAuth == CPA_TRUE) { + if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm && + pSessionDesc->isAuth == CPA_TRUE) || + isSpCcm) { /* prepare IV and AAD for CCM */ LacSymAlgChain_PrepareCCMData( pSessionDesc, @@ -655,15 +740,14 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, pRequest->ivLenInBytes); /* According to the API, for CCM and GCM, - * messageLenToHashInBytes - * and hashStartSrcOffsetInBytes are not initialized by the - * user and must be set by the driver + * messageLenToHashInBytes and hashStartSrcOffsetInBytes are not + * initialized by the user and must be set by the driver */ pRequest->hashStartSrcOffsetInBytes = pRequest->cryptoStartSrcOffsetInBytes; pRequest->messageLenToHashInBytes = pRequest->messageLenToCipherInBytes; - } else if (!pSessionDesc->isSinglePass && + } else if ((SPC != pSessionDesc->singlePassState) && (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm || CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm)) { /* GCM case */ @@ -693,56 +777,87 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, if (pSessionDesc->isCipher) { if (CPA_CY_SYM_CIPHER_ARC4 == pSessionDesc->cipherAlgorithm) { /* ARC4 does not have an IV but the field is used to - * store the - * initial state */ + * store the initial state */ pRequest->iv = pSessionDesc->cipherARC4InitialStatePhysAddr; } + ICP_QAT_FW_LA_SLICE_TYPE_SET( + pCurrentQatMsg->comn_hdr.serv_specif_flags, + pSessionDesc->cipherSliceType); + LacSymQat_CipherRequestParamsPopulate( + pSessionDesc, pCurrentQatMsg, pRequest->cryptoStartSrcOffsetInBytes, pRequest->messageLenToCipherInBytes, pRequest->iv, pRequest->pIv); - if (pSessionDesc->isSinglePass) { + if (SPC == pSessionDesc->singlePassState) { icp_qat_fw_la_cipher_req_params_t *pCipherReqParams = (icp_qat_fw_la_cipher_req_params_t *)((Cpa8U *)&( pCurrentQatMsg->serv_specif_rqpars) + ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); - pCipherReqParams->spc_aad_addr = - (uint64_t)pRequest->additionalAuthData; - pCipherReqParams->spc_aad_sz = - pSessionDesc->aadLenInBytes; + icp_qat_fw_la_cipher_20_req_params_t *pCipher20ReqParams = + (void + *)((Cpa8U *)&( + pCurrentQatMsg->serv_specif_rqpars) + + ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); - pCipherReqParams->spc_auth_res_addr = - (uint64_t)pRequest->digestResult; - pCipherReqParams->spc_auth_res_sz = - pSessionDesc->hashResultSize; + if (isCyGen4x(pService)) { + pCipher20ReqParams->spc_aad_addr = + (uint64_t)pRequest->additionalAuthData; + pCipher20ReqParams->spc_aad_sz = + pSessionDesc->aadLenInBytes; + pCipher20ReqParams->spc_aad_offset = 0; + + if (isSpCcm) + pCipher20ReqParams->spc_aad_sz += + LAC_CIPHER_CCM_AAD_OFFSET; + + pCipher20ReqParams->spc_auth_res_addr = + (uint64_t)pRequest->digestResult; + pCipher20ReqParams->spc_auth_res_sz = + (Cpa8U)pSessionDesc->hashResultSize; + } else { + pCipherReqParams->spc_aad_addr = + (uint64_t)pRequest->additionalAuthData; + pCipherReqParams->spc_aad_sz = + (Cpa16U)pSessionDesc->aadLenInBytes; + + pCipherReqParams->spc_auth_res_addr = + (uint64_t)pRequest->digestResult; + pCipherReqParams->spc_auth_res_sz = + (Cpa8U)pSessionDesc->hashResultSize; + } /* For CHACHA and AES_GCM single pass AAD buffer needs - * alignment - * if aadLenInBytes is nonzero. - * In case of AES-GMAC, AAD buffer passed in the src - * buffer. + * alignment if aadLenInBytes is nonzero. In case of + * AES-GMAC, AAD buffer passed in the src buffer. */ if (0 != pSessionDesc->aadLenInBytes && CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm) { blockLen = LacSymQat_CipherBlockSizeBytesGet( pSessionDesc->cipherAlgorithm); - if ((pSessionDesc->aadLenInBytes % blockLen) != - 0) { - paddingLen = blockLen - - (pSessionDesc->aadLenInBytes % - blockLen); - memset( - &pRequest->pAdditionalAuthData - [pSessionDesc->aadLenInBytes], - 0, - paddingLen); + aadDataLen = pSessionDesc->aadLenInBytes; + + /* In case of AES_CCM, B0 block size and 2 bytes + * of AAD len + * encoding need to be added to total AAD data + * len */ + if (isSpCcm) + aadDataLen += LAC_CIPHER_CCM_AAD_OFFSET; + + if ((aadDataLen % blockLen) != 0) { + paddingLen = + blockLen - (aadDataLen % blockLen); + memset(&pRequest->pAdditionalAuthData + [aadDataLen], + 0, + paddingLen); } } } @@ -777,32 +892,14 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, * copied directly from the op request data because they share a * corresponding layout. The remaining 4 bytes are taken * from the session message template and use values - * preconfigured at - * sessionInit (updated per request for some specific cases - * below) + * preconfigured at sessionInit (updated per request for some + * specific cases below) */ - - /* We force a specific compiler optimisation here. The length - * to - * be copied turns out to be always 16, and by coding a memcpy - * with - * a literal value the compiler will compile inline code (in - * fact, - * only two vector instructions) to effect the copy. This gives - * us - * a huge performance increase. - */ - unsigned long cplen = - (unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) - - (unsigned long)pAuthReqPars; - if (cplen == 16) - memcpy(pAuthReqPars, - (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), - 16); - else - memcpy(pAuthReqPars, - (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), - cplen); + memcpy(pAuthReqPars, + (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), + ((uintptr_t) & + (pAuthReqPars->u2.inner_prefix_sz) - + (uintptr_t)pAuthReqPars)); if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { pAuthReqPars->hash_state_sz = @@ -892,7 +989,7 @@ cpaCySymDpRemoveSession(const CpaInstanceHandle instanceHandle, /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ LAC_CHECK_INSTANCE_HANDLE(instanceHandle); -/* All other param checks are common with trad api */ + /* All other param checks are common with trad api */ return cpaCySymRemoveSession(instanceHandle, sessionCtx); } @@ -932,6 +1029,10 @@ cpaCySymDpEnqueueOp(CpaCySymDpOpData *pRequest, const CpaBoolean performOpNow) return status; } + /* Check if SAL is running in crypto data plane otherwise return an + * error */ + SAL_RUNNING_CHECK(pRequest->instanceHandle); + trans_handle = ((sal_crypto_service_t *)pRequest->instanceHandle) ->trans_handle_sym_tx; @@ -1029,6 +1130,10 @@ cpaCySymDpEnqueueOpBatch(const Cpa32U numberRequests, } } + /* Check if SAL is running in crypto data plane otherwise return an + * error */ + SAL_RUNNING_CHECK(pRequests[0]->instanceHandle); + trans_handle = ((sal_crypto_service_t *)pRequests[0]->instanceHandle) ->trans_handle_sym_tx; pSessionDesc = diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c index e662d0d6d220..a502c7318844 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c @@ -50,17 +50,16 @@ (CPA_CY_SYM_HASH_AES_CMAC == (alg)) || \ (CPA_CY_SYM_HASH_ZUC_EIA3 == (alg))) && \ (CPA_CY_SYM_HASH_MODE_AUTH != (mode))) || \ - (((CPA_CY_SYM_HASH_SHA3_224 == (alg)) || \ - (CPA_CY_SYM_HASH_SHA3_256 == (alg)) || \ - (CPA_CY_SYM_HASH_SHA3_384 == (alg)) || \ - (CPA_CY_SYM_HASH_SHA3_512 == (alg))) && \ - (CPA_CY_SYM_HASH_MODE_NESTED == (mode))) || \ - (((CPA_CY_SYM_HASH_SHAKE_128 == (alg)) || \ - (CPA_CY_SYM_HASH_SHAKE_256 == (alg))) && \ - (CPA_CY_SYM_HASH_MODE_AUTH == (mode)))) - + ((LAC_HASH_IS_SHA3(alg)) && (CPA_CY_SYM_HASH_MODE_NESTED == (mode)))) /**< Macro to check for valid algorithm-mode combination */ +void LacSync_GenBufListVerifyCb(void *pCallbackTag, + CpaStatus status, + CpaCySymOp operationType, + void *pOpData, + CpaBufferList *pDstBuffer, + CpaBoolean opResult); + /** * @ingroup LacHash * This callback function will be invoked whenever a synchronous @@ -98,7 +97,7 @@ LacHash_StatePrefixAadBufferInit( if (pHashStateBufferInfo->pDataPhys == 0) { LAC_LOG_ERROR("Unable to get the physical address of " - "the hash state buffer"); + "the hash state buffer\n"); return CPA_STATUS_FAIL; } @@ -110,9 +109,11 @@ LacHash_StatePrefixAadBufferInit( pHashStateBufferInfo, pReq, pHashSetupData->nestedModeSetupData.pInnerPrefixData, - pHashSetupData->nestedModeSetupData.innerPrefixLenInBytes, + (Cpa8U)pHashSetupData->nestedModeSetupData + .innerPrefixLenInBytes, pHashSetupData->nestedModeSetupData.pOuterPrefixData, - pHashSetupData->nestedModeSetupData.outerPrefixLenInBytes); + (Cpa8U)pHashSetupData->nestedModeSetupData + .outerPrefixLenInBytes); } /* For mode2 HMAC the key gets copied into both the inner and * outer prefix fields */ @@ -121,9 +122,9 @@ LacHash_StatePrefixAadBufferInit( pHashStateBufferInfo, pReq, pHashSetupData->authModeSetupData.authKey, - pHashSetupData->authModeSetupData.authKeyLenInBytes, + (Cpa8U)pHashSetupData->authModeSetupData.authKeyLenInBytes, pHashSetupData->authModeSetupData.authKey, - pHashSetupData->authModeSetupData.authKeyLenInBytes); + (Cpa8U)pHashSetupData->authModeSetupData.authKeyLenInBytes); } /* else do nothing for the other cases */ return CPA_STATUS_SUCCESS; @@ -226,22 +227,32 @@ LacHash_PrecomputeDataCreate(const CpaInstanceHandle instanceHandle, pCallbackTag); } else if (CPA_CY_SYM_HASH_AES_CCM == hashAlgorithm) { /* - * The Inner Hash Initial State2 block must contain K - * (the cipher key) and 16 zeroes which will be replaced with - * EK(Ctr0) by the QAT-ME. + * The Inner Hash Initial State2 block is 32 bytes long. + * Therefore, for keys bigger than 128 bits (16 bytes), + * there is no space for 16 zeroes. */ + if (pSessionSetup->cipherSetupData.cipherKeyLenInBytes == + ICP_QAT_HW_AES_128_KEY_SZ) { + /* + * The Inner Hash Initial State2 block must contain K + * (the cipher key) and 16 zeroes which will be replaced + * with EK(Ctr0) by the QAT-ME. + */ - /* write the auth key which for CCM is equivalent to cipher key - */ - memcpy(pState2, - pSessionSetup->cipherSetupData.pCipherKey, - pSessionSetup->cipherSetupData.cipherKeyLenInBytes); + /* write the auth key which for CCM is equivalent to + * cipher key + */ + memcpy( + pState2, + pSessionSetup->cipherSetupData.pCipherKey, + pSessionSetup->cipherSetupData.cipherKeyLenInBytes); - /* initialize remaining buffer space to all zeroes */ - LAC_OS_BZERO( - pState2 + - pSessionSetup->cipherSetupData.cipherKeyLenInBytes, - ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ); + /* initialize remaining buffer space to all zeroes */ + LAC_OS_BZERO(pState2 + + pSessionSetup->cipherSetupData + .cipherKeyLenInBytes, + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ); + } /* There is no request sent to the QAT for this operation, * so just invoke the user's callback directly to signal @@ -279,8 +290,8 @@ LacHash_PrecomputeDataCreate(const CpaInstanceHandle instanceHandle, if (CPA_STATUS_SUCCESS == status) { /* write len(A) (the length of A) into bytes 16-19 of - * pState2 - * in big-endian format. This field is 8 bytes */ + * pState2 in big-endian format. This field is 8 bytes + */ *(Cpa32U *)&pState2[ICP_QAT_HW_GALOIS_H_SZ] = LAC_MEM_WR_32(pAuthModeSetupData->aadLenInBytes); } @@ -367,26 +378,15 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, CpaCySymCapabilitiesInfo capInfo; /*Protect against value of hash outside the bitmap*/ - if ((pHashSetupData->hashAlgorithm) >= - CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) { + if (pHashSetupData->hashAlgorithm >= CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) { LAC_INVALID_PARAM_LOG("hashAlgorithm"); return CPA_STATUS_INVALID_PARAM; } + cpaCySymQueryCapabilities(instanceHandle, &capInfo); if (!CPA_BITMAP_BIT_TEST(capInfo.hashes, pHashSetupData->hashAlgorithm) && pHashSetupData->hashAlgorithm != CPA_CY_SYM_HASH_AES_CBC_MAC) { - /* Ensure SHAKE algorithms are not supported */ - if ((CPA_CY_SYM_HASH_SHAKE_128 == - pHashSetupData->hashAlgorithm) || - (CPA_CY_SYM_HASH_SHAKE_256 == - pHashSetupData->hashAlgorithm)) { - LAC_INVALID_PARAM_LOG( - "Hash algorithms SHAKE-128 and SHAKE-256 " - "are not supported."); - return CPA_STATUS_UNSUPPORTED; - } - LAC_INVALID_PARAM_LOG("hashAlgorithm"); return CPA_STATUS_INVALID_PARAM; } @@ -405,8 +405,9 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, if (LAC_HASH_ALG_MODE_NOT_SUPPORTED(pHashSetupData->hashAlgorithm, pHashSetupData->hashMode)) { - LAC_INVALID_PARAM_LOG("hashAlgorithm and hashMode combination"); - return CPA_STATUS_INVALID_PARAM; + LAC_UNSUPPORTED_PARAM_LOG( + "hashAlgorithm and hashMode combination"); + return CPA_STATUS_UNSUPPORTED; } LacSymQat_HashAlgLookupGet(instanceHandle, @@ -432,10 +433,8 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, Cpa32U aadDataSize = 0; /* RFC 4106: Implementations MUST support a full-length - * 16-octet - * ICV, and MAY support 8 or 12 octet ICVs, and MUST NOT - * support - * other ICV lengths. */ + * 16-octet ICV, and MAY support 8 or 12 octet ICVs, and + * MUST NOT support other ICV lengths. */ if ((pHashSetupData->digestResultLenInBytes != LAC_HASH_AES_GCM_ICV_SIZE_8) && (pHashSetupData->digestResultLenInBytes != @@ -490,15 +489,13 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, aadDataSize = LAC_HASH_AES_CCM_BLOCK_SIZE; /* then, if there is some 'a' data, the buffer will - * store encoded - * length of 'a' and 'a' itself */ + * store encoded length of 'a' and 'a' itself */ if (pHashSetupData->authModeSetupData.aadLenInBytes > 0) { /* as the QAT API puts the requirement on the * pAdditionalAuthData not to be bigger than 240 - * bytes then we - * just need 2 bytes to store encoded length of - * 'a' */ + * bytes then we just need 2 bytes to store + * encoded length of 'a' */ aadDataSize += sizeof(Cpa16U); aadDataSize += pHashSetupData->authModeSetupData .aadLenInBytes; @@ -536,8 +533,7 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, return CPA_STATUS_INVALID_PARAM; } /* For Snow3g hash aad field contains IV - it needs to - * be 16 - * bytes long + * be 16 bytes long */ if (pHashSetupData->authModeSetupData.aadLenInBytes != ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ) { @@ -570,8 +566,7 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, return CPA_STATUS_INVALID_PARAM; } /* For ZUC EIA3 hash aad field contains IV - it needs to - * be 16 - * bytes long + * be 16 bytes long */ if (pHashSetupData->authModeSetupData.aadLenInBytes != ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ) { @@ -613,19 +608,6 @@ LacHash_HashContextCheck(CpaInstanceHandle instanceHandle, if (!CPA_BITMAP_BIT_TEST(capInfo.hashes, pHashSetupData->nestedModeSetupData .outerHashAlgorithm)) { - /* Ensure SHAKE algorithms are not supported */ - if ((CPA_CY_SYM_HASH_SHAKE_128 == - pHashSetupData->nestedModeSetupData - .outerHashAlgorithm) || - (CPA_CY_SYM_HASH_SHAKE_256 == - pHashSetupData->nestedModeSetupData - .outerHashAlgorithm)) { - LAC_INVALID_PARAM_LOG( - "Hash algorithms SHAKE-128 and SHAKE-256 " - "are not supported."); - return CPA_STATUS_UNSUPPORTED; - } - LAC_INVALID_PARAM_LOG("outerHashAlgorithm"); return CPA_STATUS_INVALID_PARAM; } @@ -689,11 +671,15 @@ LacHash_PerformParamCheck(CpaInstanceHandle instanceHandle, { CpaStatus status = CPA_STATUS_SUCCESS; lac_sym_qat_hash_alg_info_t *pHashAlgInfo = NULL; + CpaBoolean digestIsAppended = pSessionDesc->digestIsAppended; + CpaBoolean digestVerify = pSessionDesc->digestVerify; + CpaCySymOp symOperation = pSessionDesc->symOperation; + CpaCySymHashAlgorithm hashAlgorithm = pSessionDesc->hashAlgorithm; /* digestVerify and digestIsAppended on Hash-Only operation not * supported */ - if (pSessionDesc->digestIsAppended && pSessionDesc->digestVerify && - (CPA_CY_SYM_OP_HASH == pSessionDesc->symOperation)) { + if (digestIsAppended && digestVerify && + (CPA_CY_SYM_OP_HASH == symOperation)) { LAC_INVALID_PARAM_LOG( "digestVerify and digestIsAppended set " "on Hash-Only operation is not supported"); @@ -702,21 +688,19 @@ LacHash_PerformParamCheck(CpaInstanceHandle instanceHandle, /* check the digest result pointer */ if ((CPA_CY_SYM_PACKET_TYPE_PARTIAL != pOpData->packetType) && - !pSessionDesc->digestIsAppended && - (NULL == pOpData->pDigestResult)) { + !digestIsAppended && (NULL == pOpData->pDigestResult)) { LAC_INVALID_PARAM_LOG("pDigestResult is NULL"); return CPA_STATUS_INVALID_PARAM; } /* * Check if the pVerifyResult pointer is not null for hash operation - * when - * the packet is the last one and user has set verifyDigest flag + * when the packet is the last one and user has set verifyDigest flag * Also, this is only needed for symchronous operation, so check if the * callback pointer is the internal synchronous one rather than a user- * supplied one. */ - if ((CPA_TRUE == pSessionDesc->digestVerify) && + if ((CPA_TRUE == digestVerify) && (CPA_CY_SYM_PACKET_TYPE_PARTIAL != pOpData->packetType) && (LacSync_GenBufListVerifyCb == pSessionDesc->pSymCb)) { if (NULL == pVerifyResult) { @@ -732,8 +716,8 @@ LacHash_PerformParamCheck(CpaInstanceHandle instanceHandle, * written anywhere so we cannot check for this been inside a buffer * CCM/GCM specify the auth region using just the cipher params as this * region is the same for auth and cipher. It is not checked here */ - if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) || - (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm)) { + if ((CPA_CY_SYM_HASH_AES_CCM == hashAlgorithm) || + (CPA_CY_SYM_HASH_AES_GCM == hashAlgorithm)) { /* ensure AAD data pointer is non-NULL if AAD len > 0 */ if ((pSessionDesc->aadLenInBytes > 0) && (NULL == pOpData->pAdditionalAuthData)) { @@ -752,8 +736,8 @@ LacHash_PerformParamCheck(CpaInstanceHandle instanceHandle, /* For Snow3g & ZUC hash pAdditionalAuthData field * of OpData should contain IV */ - if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) || - (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) { + if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 == hashAlgorithm) || + (CPA_CY_SYM_HASH_ZUC_EIA3 == hashAlgorithm)) { if (NULL == pOpData->pAdditionalAuthData) { LAC_INVALID_PARAM_LOG("pAdditionalAuthData is NULL"); return CPA_STATUS_INVALID_PARAM; @@ -761,12 +745,11 @@ LacHash_PerformParamCheck(CpaInstanceHandle instanceHandle, } /* partial packets need to be multiples of the algorithm block size in - * hash - * only mode (except for final partial packet) */ + * hash only mode (except for final partial packet) */ if ((CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) && - (CPA_CY_SYM_OP_HASH == pSessionDesc->symOperation)) { + (CPA_CY_SYM_OP_HASH == symOperation)) { LacSymQat_HashAlgLookupGet(instanceHandle, - pSessionDesc->hashAlgorithm, + hashAlgorithm, &pHashAlgInfo); /* check if the message is a multiple of the block size. */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c index 977fb0b84d0b..43d2f44474ee 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c @@ -57,11 +57,7 @@ LacSymQueue_RequestSend(const CpaInstanceHandle instanceHandle, */ if ((CPA_FALSE == pSessionDesc->nonBlockingOpsInProgress) || (NULL != pSessionDesc->pRequestQueueTail)) { - if (CPA_STATUS_SUCCESS != - LAC_SPINLOCK(&pSessionDesc->requestQueueLock)) { - LAC_LOG_ERROR("Failed to lock request queue"); - return CPA_STATUS_RESOURCE; - } + LAC_SPINLOCK(&pSessionDesc->requestQueueLock); /* Re-check blockingOpsInProgress and pRequestQueueTail in case * either @@ -95,10 +91,7 @@ LacSymQueue_RequestSend(const CpaInstanceHandle instanceHandle, /* request is queued, don't send to QAT here */ enqueued = CPA_TRUE; } - if (CPA_STATUS_SUCCESS != - LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock)) { - LAC_LOG_ERROR("Failed to unlock request queue"); - } + LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock); } if (CPA_FALSE == enqueued) { diff --git a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c index c55da0a0d531..61cb7044ada6 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c @@ -31,6 +31,7 @@ #include "sal_string_parse.h" #include "lac_sym_key.h" #include "lac_sym_qat_hash_defs_lookup.h" +#include "lac_sym_qat_constants_table.h" #include "lac_sym_qat_cipher.h" #include "lac_sym_qat_hash.h" @@ -104,6 +105,9 @@ LacSymQat_Init(CpaInstanceHandle instanceHandle) { CpaStatus status = CPA_STATUS_SUCCESS; + /* Initialize the SHRAM constants table */ + LacSymQat_ConstantsInitLookupTables(instanceHandle); + /* Initialise the Hash lookup table */ status = LacSymQat_HashLookupInit(instanceHandle); @@ -130,10 +134,10 @@ LacSymQat_LaPacketCommandFlagSet(Cpa32U qatPacketType, Cpa16U *pLaCommandFlags, Cpa32U ivLenInBytes) { - /* For Chacha ciphers set command flag as partial none to proceed + /* For SM4/Chacha ciphers set command flag as partial none to proceed * with stateless processing */ - if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm) || - LAC_CIPHER_IS_SM4(cipherAlgorithm)) { + if (LAC_CIPHER_IS_SM4(cipherAlgorithm) || + LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) { ICP_QAT_FW_LA_PARTIAL_SET(*pLaCommandFlags, ICP_QAT_FW_LA_PARTIAL_NONE); return; @@ -144,10 +148,10 @@ LacSymQat_LaPacketCommandFlagSet(Cpa32U qatPacketType, * must be disabled always. * For all other ciphers and auth * update state is disabled for full packets and final partials */ - if (((laCmdId != ICP_QAT_FW_LA_CMD_AUTH) && - LAC_CIPHER_IS_ECB_MODE(cipherAlgorithm)) || - (ICP_QAT_FW_LA_PARTIAL_NONE == qatPacketType) || - (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType)) { + if ((ICP_QAT_FW_LA_PARTIAL_NONE == qatPacketType) || + (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType) || + ((laCmdId != ICP_QAT_FW_LA_CMD_AUTH) && + LAC_CIPHER_IS_ECB_MODE(cipherAlgorithm))) { ICP_QAT_FW_LA_UPDATE_STATE_SET(*pLaCommandFlags, ICP_QAT_FW_LA_NO_UPDATE_STATE); } @@ -182,8 +186,9 @@ LacSymQat_packetTypeGet(CpaCySymPacketType packetType, CpaCySymPacketType packetState, Cpa32U *pQatPacketType) { + switch (packetType) { /* partial */ - if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == packetType) { + case CPA_CY_SYM_PACKET_TYPE_PARTIAL: /* if the previous state was full, then this is the first packet */ if (CPA_CY_SYM_PACKET_TYPE_FULL == packetState) { @@ -191,13 +196,15 @@ LacSymQat_packetTypeGet(CpaCySymPacketType packetType, } else { *pQatPacketType = ICP_QAT_FW_LA_PARTIAL_MID; } - } + break; + /* final partial */ - else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == packetType) { + case CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL: *pQatPacketType = ICP_QAT_FW_LA_PARTIAL_END; - } + break; + /* full packet - CPA_CY_SYM_PACKET_TYPE_FULL */ - else { + default: *pQatPacketType = ICP_QAT_FW_LA_PARTIAL_NONE; } } @@ -225,3 +232,101 @@ LacSymQat_LaSetDefaultFlags(icp_qat_fw_serv_specif_flags *laCmdFlags, ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( *laCmdFlags, ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS); } + +CpaBoolean +LacSymQat_UseSymConstantsTable(lac_session_desc_t *pSession, + Cpa8U *pCipherOffset, + Cpa8U *pHashOffset) +{ + + CpaBoolean useOptimisedContentDesc = CPA_FALSE; + CpaBoolean useSHRAMConstants = CPA_FALSE; + + *pCipherOffset = 0; + *pHashOffset = 0; + + /* for chaining can we use the optimised content descritor */ + if (pSession->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH || + pSession->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { + useOptimisedContentDesc = + LacSymQat_UseOptimisedContentDesc(pSession); + } + + /* Cipher-only case or chaining */ + if (pSession->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER || + useOptimisedContentDesc) { + icp_qat_hw_cipher_algo_t algorithm; + icp_qat_hw_cipher_mode_t mode; + icp_qat_hw_cipher_dir_t dir; + icp_qat_hw_cipher_convert_t key_convert; + + if (pSession->cipherKeyLenInBytes > + sizeof(icp_qat_fw_comn_req_hdr_cd_pars_t)) { + return CPA_FALSE; + } + + LacSymQat_CipherGetCfgData( + pSession, &algorithm, &mode, &dir, &key_convert); + + /* Check if cipher config is available in table. */ + LacSymQat_ConstantsGetCipherOffset(pSession->pInstance, + algorithm, + mode, + dir, + key_convert, + pCipherOffset); + if (*pCipherOffset > 0) { + useSHRAMConstants = CPA_TRUE; + } else { + useSHRAMConstants = CPA_FALSE; + } + } + + /* hash only case or when chaining, cipher must be found in SHRAM table + * for + * optimised CD case */ + if (pSession->laCmdId == ICP_QAT_FW_LA_CMD_AUTH || + (useOptimisedContentDesc && useSHRAMConstants)) { + icp_qat_hw_auth_algo_t algorithm; + CpaBoolean nested; + + if (pSession->digestVerify) { + return CPA_FALSE; + } + + if ((!(useOptimisedContentDesc && useSHRAMConstants)) && + (pSession->qatHashMode == ICP_QAT_HW_AUTH_MODE1)) { + /* we can only use the SHA1-mode1 in the SHRAM constants + * table when + * we are using the opimised content desc */ + return CPA_FALSE; + } + + LacSymQat_HashGetCfgData(pSession->pInstance, + pSession->qatHashMode, + pSession->hashMode, + pSession->hashAlgorithm, + &algorithm, + &nested); + + /* Check if config data is available in table. */ + LacSymQat_ConstantsGetAuthOffset(pSession->pInstance, + algorithm, + pSession->qatHashMode, + nested, + pHashOffset); + if (*pHashOffset > 0) { + useSHRAMConstants = CPA_TRUE; + } else { + useSHRAMConstants = CPA_FALSE; + } + } + + return useSHRAMConstants; +} + +CpaBoolean +LacSymQat_UseOptimisedContentDesc(lac_session_desc_t *pSession) +{ + return CPA_FALSE; +} diff --git a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c index b958d3723703..5c554efd61a1 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c @@ -29,6 +29,9 @@ #include "lac_sym_cipher_defs.h" #include "icp_qat_hw.h" #include "icp_qat_fw_la.h" +#include "sal_hw_gen.h" + +#define LAC_UNUSED_POS_MASK 0x3 /***************************************************************************** * Internal data @@ -73,7 +76,7 @@ static const uint8_t key_size_xts[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES128, // ICP_QAT_HW_AES_128_XTS_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES128, /* ICP_QAT_HW_AES_128_XTS_KEY_SZ */ 0, 0, 0, @@ -105,7 +108,7 @@ static const uint8_t key_size_xts[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES256 // ICP_QAT_HW_AES_256_XTS_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_XTS_KEY_SZ */ }; /* LAC_CIPHER_IS_AES */ static const uint8_t key_size_aes[] = { @@ -125,7 +128,7 @@ static const uint8_t key_size_aes[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES128, // ICP_QAT_HW_AES_128_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES128, /* ICP_QAT_HW_AES_128_KEY_SZ */ 0, 0, 0, @@ -133,7 +136,7 @@ static const uint8_t key_size_aes[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES192, // ICP_QAT_HW_AES_192_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES192, /* ICP_QAT_HW_AES_192_KEY_SZ */ 0, 0, 0, @@ -141,7 +144,7 @@ static const uint8_t key_size_aes[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES256 // ICP_QAT_HW_AES_256_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_KEY_SZ */ }; /* LAC_CIPHER_IS_AES_F8 */ static const uint8_t key_size_f8[] = { @@ -177,7 +180,7 @@ static const uint8_t key_size_f8[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES128, // ICP_QAT_HW_AES_128_F8_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES128, /* ICP_QAT_HW_AES_128_F8_KEY_SZ */ 0, 0, 0, @@ -193,7 +196,7 @@ static const uint8_t key_size_f8[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES192, // ICP_QAT_HW_AES_192_F8_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES192, /* ICP_QAT_HW_AES_192_F8_KEY_SZ */ 0, 0, 0, @@ -209,27 +212,7 @@ static const uint8_t key_size_f8[] = { 0, 0, 0, - ICP_QAT_HW_CIPHER_ALGO_AES256 // ICP_QAT_HW_AES_256_F8_KEY_SZ -}; -/* LAC_CIPHER_IS_SM4 */ -static const uint8_t key_size_sm4[] = { - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - ICP_QAT_HW_CIPHER_ALGO_SM4 // ICP_QAT_HW_SM4_KEY_SZ + ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_F8_KEY_SZ */ }; typedef struct _icp_qat_hw_cipher_info { @@ -241,239 +224,237 @@ typedef struct _icp_qat_hw_cipher_info { const uint8_t *pAlgByKeySize; } icp_qat_hw_cipher_info; -static const icp_qat_hw_cipher_info icp_qat_alg_info[] = - { - /* CPA_CY_SYM_CIPHER_NULL */ - { - ICP_QAT_HW_CIPHER_ALGO_NULL, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_ARC4 */ - { - ICP_QAT_HW_CIPHER_ALGO_ARC4, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_AES_ECB */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_ECB_MODE, - /* AES decrypt key needs to be reversed. Instead of reversing the key - * at session registration, it is instead reversed on-the-fly by - * setting the KEY_CONVERT bit here - */ - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_YES, - key_size_aes, - }, - /* CPA_CY_SYM_CIPHER_AES_CBC */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_CBC_MODE, - /* AES decrypt key needs to be reversed. Instead of reversing the key - * at session registration, it is instead reversed on-the-fly by - * setting the KEY_CONVERT bit here - */ - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_YES, - key_size_aes, - }, - /* CPA_CY_SYM_CIPHER_AES_CTR */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_CTR_MODE, - /* AES decrypt key needs to be reversed. Instead of reversing the key - * at session registration, it is instead reversed on-the-fly by - * setting the KEY_CONVERT bit here - */ - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt - * Overriding default values previously set for AES - */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_YES, - key_size_aes, - }, - /* CPA_CY_SYM_CIPHER_AES_CCM */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_CTR_MODE, - /* AES decrypt key needs to be reversed. Instead of reversing the key - * at session registration, it is instead reversed on-the-fly by - * setting the KEY_CONVERT bit here - */ - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt - * Overriding default values previously set for AES - */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_YES, - key_size_aes, - }, - /* CPA_CY_SYM_CIPHER_AES_GCM */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_CTR_MODE, - /* AES decrypt key needs to be reversed. Instead of reversing the key - * at session registration, it is instead reversed on-the-fly by - * setting the KEY_CONVERT bit here - */ - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt - * Overriding default values previously set for AES - */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_YES, - key_size_aes, - }, - /* CPA_CY_SYM_CIPHER_DES_ECB */ - { - ICP_QAT_HW_CIPHER_ALGO_DES, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_DES_CBC */ - { - ICP_QAT_HW_CIPHER_ALGO_DES, - ICP_QAT_HW_CIPHER_CBC_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_3DES_ECB */ - { - ICP_QAT_HW_CIPHER_ALGO_3DES, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_3DES_CBC */ - { - ICP_QAT_HW_CIPHER_ALGO_3DES, - ICP_QAT_HW_CIPHER_CBC_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_3DES_CTR */ - { - ICP_QAT_HW_CIPHER_ALGO_3DES, - ICP_QAT_HW_CIPHER_CTR_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt - * Overriding default values previously set for AES - */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_KASUMI_F8 */ - { - ICP_QAT_HW_CIPHER_ALGO_KASUMI, - ICP_QAT_HW_CIPHER_F8_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_SNOW3G_UEA2 */ - { - /* The KEY_CONVERT bit has to be set for Snow_3G operation */ - ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_AES_F8 */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_F8_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_YES, - key_size_f8, - }, - /* CPA_CY_SYM_CIPHER_AES_XTS */ - { - ICP_QAT_HW_CIPHER_ALGO_AES128, - ICP_QAT_HW_CIPHER_XTS_MODE, - /* AES decrypt key needs to be reversed. Instead of reversing the key - * at session registration, it is instead reversed on-the-fly by - * setting the KEY_CONVERT bit here - */ - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_YES, - key_size_xts, - }, - /* CPA_CY_SYM_CIPHER_ZUC_EEA3 */ - { - ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_CHACHA */ - { - ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305, - ICP_QAT_HW_CIPHER_CTR_MODE, - { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_NO, - NULL, - }, - /* CPA_CY_SYM_CIPHER_SM4_ECB */ - { - ICP_QAT_HW_CIPHER_ALGO_SM4, - ICP_QAT_HW_CIPHER_ECB_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_YES, - key_size_sm4, - }, - /* CPA_CY_SYM_CIPHER_SM4_CBC */ - { - ICP_QAT_HW_CIPHER_ALGO_SM4, - ICP_QAT_HW_CIPHER_CBC_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, - IS_KEY_DEP_YES, - key_size_sm4, - }, - /* CPA_CY_SYM_CIPHER_SM4_CTR */ - { - ICP_QAT_HW_CIPHER_ALGO_SM4, - ICP_QAT_HW_CIPHER_CTR_MODE, - { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, - /* Streaming ciphers are a special case. Decrypt = encrypt */ - { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, - IS_KEY_DEP_YES, - key_size_sm4, - }, - }; +static const icp_qat_hw_cipher_info icp_qat_alg_info[] = { + /* CPA_CY_SYM_CIPHER_NULL */ + { + ICP_QAT_HW_CIPHER_ALGO_NULL, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_ARC4 */ + { + ICP_QAT_HW_CIPHER_ALGO_ARC4, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_AES_ECB */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_ECB_MODE, + /* AES decrypt key needs to be reversed. Instead of reversing the + * key at session registration, it is instead reversed on-the-fly by + * setting the KEY_CONVERT bit here + */ + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_YES, + key_size_aes, + }, + /* CPA_CY_SYM_CIPHER_AES_CBC */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_CBC_MODE, + /* AES decrypt key needs to be reversed. Instead of reversing the + * key at session registration, it is instead reversed on-the-fly by + * setting the KEY_CONVERT bit here + */ + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_YES, + key_size_aes, + }, + /* CPA_CY_SYM_CIPHER_AES_CTR */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_CTR_MODE, + /* AES decrypt key needs to be reversed. Instead of reversing the + * key at session registration, it is instead reversed on-the-fly by + * setting the KEY_CONVERT bit here + */ + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt + * Overriding default values previously set for AES + */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_YES, + key_size_aes, + }, + /* CPA_CY_SYM_CIPHER_AES_CCM */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_CTR_MODE, + /* AES decrypt key needs to be reversed. Instead of reversing the + * key at session registration, it is instead reversed on-the-fly by + * setting the KEY_CONVERT bit here + */ + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt + * Overriding default values previously set for AES + */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_YES, + key_size_aes, + }, + /* CPA_CY_SYM_CIPHER_AES_GCM */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_CTR_MODE, + /* AES decrypt key needs to be reversed. Instead of reversing the + * key at session registration, it is instead reversed on-the-fly by + * setting the KEY_CONVERT bit here + */ + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt + * Overriding default values previously set for AES + */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_YES, + key_size_aes, + }, + /* CPA_CY_SYM_CIPHER_DES_ECB */ + { + ICP_QAT_HW_CIPHER_ALGO_DES, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_DES_CBC */ + { + ICP_QAT_HW_CIPHER_ALGO_DES, + ICP_QAT_HW_CIPHER_CBC_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_3DES_ECB */ + { + ICP_QAT_HW_CIPHER_ALGO_3DES, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_3DES_CBC */ + { + ICP_QAT_HW_CIPHER_ALGO_3DES, + ICP_QAT_HW_CIPHER_CBC_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_3DES_CTR */ + { + ICP_QAT_HW_CIPHER_ALGO_3DES, + ICP_QAT_HW_CIPHER_CTR_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt + * Overriding default values previously set for AES + */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_KASUMI_F8 */ + { + ICP_QAT_HW_CIPHER_ALGO_KASUMI, + ICP_QAT_HW_CIPHER_F8_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_SNOW3G_UEA2 */ + { + /* The KEY_CONVERT bit has to be set for Snow_3G operation */ + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_AES_F8 */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_F8_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + /* Streaming ciphers are a special case. Decrypt = encrypt */ + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_YES, + key_size_f8, + }, + /* CPA_CY_SYM_CIPHER_AES_XTS */ + { + ICP_QAT_HW_CIPHER_ALGO_AES128, + ICP_QAT_HW_CIPHER_XTS_MODE, + /* AES decrypt key needs to be reversed. Instead of reversing the + * key at session registration, it is instead reversed on-the-fly by + * setting the KEY_CONVERT bit here + */ + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_YES, + key_size_xts, + }, + /* CPA_CY_SYM_CIPHER_ZUC_EEA3 */ + { + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_CHACHA */ + { + ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305, + ICP_QAT_HW_CIPHER_CTR_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_SM4_ECB */ + { + ICP_QAT_HW_CIPHER_ALGO_SM4, + ICP_QAT_HW_CIPHER_ECB_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_SM4_CBC */ + { + ICP_QAT_HW_CIPHER_ALGO_SM4, + ICP_QAT_HW_CIPHER_CBC_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT }, + IS_KEY_DEP_NO, + NULL, + }, + /* CPA_CY_SYM_CIPHER_SM4_CTR */ + { + ICP_QAT_HW_CIPHER_ALGO_SM4, + ICP_QAT_HW_CIPHER_CTR_MODE, + { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT }, + { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT }, + IS_KEY_DEP_NO, + NULL, + }, +}; /***************************************************************************** * Internal functions @@ -483,6 +464,7 @@ void LacSymQat_CipherCtrlBlockWrite(icp_qat_la_bulk_req_ftr_t *pMsg, Cpa32U cipherAlgorithm, Cpa32U targetKeyLenInBytes, + Cpa32U sliceType, icp_qat_fw_slice_t nextSlice, Cpa8U cipherCfgOffsetInQuadWord) { @@ -492,33 +474,52 @@ LacSymQat_CipherCtrlBlockWrite(icp_qat_la_bulk_req_ftr_t *pMsg, /* state_padding_sz is nonzero for f8 mode only */ cd_ctrl->cipher_padding_sz = 0; + /* Special handling of AES 192 key for UCS slice. + UCS requires it to have 32 bytes - set is as targetKeyLen + in this case, and add padding. It makes no sense + to force applications to provide such key length for couple reasons: + 1. It won't be possible to distinguish between AES 192 and 256 based + on key lenght only + 2. Only some modes of AES will use UCS slice, then application will + have to know which ones */ + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == sliceType && + ICP_QAT_HW_AES_192_KEY_SZ == targetKeyLenInBytes) { + targetKeyLenInBytes = ICP_QAT_HW_UCS_AES_192_KEY_SZ; + } + + switch (cipherAlgorithm) { /* Base Key is not passed down to QAT in the case of ARC4 or NULL */ - if (LAC_CIPHER_IS_ARC4(cipherAlgorithm) || - LAC_CIPHER_IS_NULL(cipherAlgorithm)) { + case CPA_CY_SYM_CIPHER_ARC4: + case CPA_CY_SYM_CIPHER_NULL: cd_ctrl->cipher_key_sz = 0; - } else if (LAC_CIPHER_IS_KASUMI(cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_KASUMI_F8: cd_ctrl->cipher_key_sz = LAC_BYTES_TO_QUADWORDS(ICP_QAT_HW_KASUMI_F8_KEY_SZ); cd_ctrl->cipher_padding_sz = ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR; - } else if (LAC_CIPHER_IS_SNOW3G_UEA2(cipherAlgorithm)) { - /* For Snow3G UEA2 content descriptor key size is - key size plus iv size */ + break; + /* For Snow3G UEA2 content descriptor key size is + key size plus iv size */ + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: cd_ctrl->cipher_key_sz = LAC_BYTES_TO_QUADWORDS(ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); - } else if (LAC_CIPHER_IS_AES_F8(cipherAlgorithm)) { + break; + case CPA_CY_SYM_CIPHER_AES_F8: cd_ctrl->cipher_key_sz = LAC_BYTES_TO_QUADWORDS(targetKeyLenInBytes); cd_ctrl->cipher_padding_sz = - 2 * ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR; - } else if (LAC_CIPHER_IS_ZUC_EEA3(cipherAlgorithm)) { - /* For ZUC EEA3 content descriptor key size is - key size plus iv size */ + (2 * ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR); + break; + /* For ZUC EEA3 content descriptor key size is + key size plus iv size */ + case CPA_CY_SYM_CIPHER_ZUC_EEA3: cd_ctrl->cipher_key_sz = LAC_BYTES_TO_QUADWORDS(ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); - } else { + break; + default: cd_ctrl->cipher_key_sz = LAC_BYTES_TO_QUADWORDS(targetKeyLenInBytes); } @@ -539,6 +540,8 @@ LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession, icp_qat_hw_cipher_dir_t *pDir, icp_qat_hw_cipher_convert_t *pKey_convert) { + sal_crypto_service_t *pService = + (sal_crypto_service_t *)pSession->pInstance; CpaCySymCipherAlgorithm cipherAlgorithm = 0; icp_qat_hw_cipher_dir_t cipherDirection = 0; @@ -553,8 +556,8 @@ LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession, cipherAlgorithm = pSession->cipherAlgorithm - 1; cipherDirection = pSession->cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ? - ICP_QAT_HW_CIPHER_ENCRYPT : - ICP_QAT_HW_CIPHER_DECRYPT; + ICP_QAT_HW_CIPHER_ENCRYPT : + ICP_QAT_HW_CIPHER_DECRYPT; *pAlgorithm = icp_qat_alg_info[cipherAlgorithm].algorithm; *pMode = icp_qat_alg_info[cipherAlgorithm].mode; @@ -566,23 +569,22 @@ LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession, *pAlgorithm = icp_qat_alg_info[cipherAlgorithm] .pAlgByKeySize[pSession->cipherKeyLenInBytes]; } - /* Set the mode */ - if (LAC_CIPHER_IS_CTR_MODE(pSession->cipherAlgorithm)) { - *pMode = ICP_QAT_HW_CIPHER_CTR_MODE; - *pKey_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; - /* CCP and AES_GCM single pass, despite being limited to - * CTR/AEAD mode, - * support both Encrypt/Decrypt modes - this is because of the - * differences in the hash computation/verification paths in - * encrypt/decrypt modes respectively. - * By default CCP is set as CTR Mode.Set AEAD Mode for AES_GCM. - */ - if (pSession->isSinglePass) { - if (LAC_CIPHER_IS_GCM(pSession->cipherAlgorithm)) - *pMode = ICP_QAT_HW_CIPHER_AEAD_MODE; - if (cipherDirection == ICP_QAT_HW_CIPHER_DECRYPT) - *pDir = ICP_QAT_HW_CIPHER_DECRYPT; - } + + /* CCP and AES_GCM single pass, despite being limited to CTR/AEAD mode, + * support both Encrypt/Decrypt modes - this is because of the + * differences in the hash computation/verification paths in + * encrypt/decrypt modes respectively. + * By default CCP is set as CTR Mode.Set AEAD Mode for AES_GCM. + */ + if (SPC == pSession->singlePassState) { + if (LAC_CIPHER_IS_GCM(pSession->cipherAlgorithm)) + *pMode = ICP_QAT_HW_CIPHER_AEAD_MODE; + else if (isCyGen4x(pService) && + LAC_CIPHER_IS_CCM(pSession->cipherAlgorithm)) + *pMode = ICP_QAT_HW_CIPHER_CCM_MODE; + + if (cipherDirection == ICP_QAT_HW_CIPHER_DECRYPT) + *pDir = ICP_QAT_HW_CIPHER_DECRYPT; } } @@ -597,6 +599,10 @@ LacSymQat_CipherHwBlockPopulateCfgData(lac_session_desc_t *pSession, icp_qat_hw_cipher_convert_t key_convert; icp_qat_hw_cipher_config_t *pCipherConfig = (icp_qat_hw_cipher_config_t *)pCipherHwBlock; + icp_qat_hw_ucs_cipher_config_t *pUCSCipherConfig = + (icp_qat_hw_ucs_cipher_config_t *)pCipherHwBlock; + + Cpa32U val, reserved; Cpa32U aed_hash_cmp_length = 0; *pSizeInBytes = 0; @@ -605,24 +611,38 @@ LacSymQat_CipherHwBlockPopulateCfgData(lac_session_desc_t *pSession, pSession, &algorithm, &mode, &dir, &key_convert); /* Build the cipher config into the hardware setup block */ - if (pSession->isSinglePass) { + if (SPC == pSession->singlePassState) { aed_hash_cmp_length = pSession->hashResultSize; - pCipherConfig->reserved = ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER( + reserved = ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER( pSession->aadLenInBytes); } else { - pCipherConfig->reserved = 0; + reserved = 0; } - pCipherConfig->val = ICP_QAT_HW_CIPHER_CONFIG_BUILD( + val = ICP_QAT_HW_CIPHER_CONFIG_BUILD( mode, algorithm, key_convert, dir, aed_hash_cmp_length); - *pSizeInBytes = sizeof(icp_qat_hw_cipher_config_t); + /* UCS slice has 128-bit configuration register. + Leacy cipher slice has 64-bit config register */ + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSession->cipherSliceType) { + pUCSCipherConfig->val = val; + pUCSCipherConfig->reserved[0] = reserved; + pUCSCipherConfig->reserved[1] = 0; + pUCSCipherConfig->reserved[2] = 0; + *pSizeInBytes = sizeof(icp_qat_hw_ucs_cipher_config_t); + } else { + pCipherConfig->val = val; + pCipherConfig->reserved = reserved; + *pSizeInBytes = sizeof(icp_qat_hw_cipher_config_t); + } } void LacSymQat_CipherHwBlockPopulateKeySetup( + lac_session_desc_t *pSessionDesc, const CpaCySymCipherSetupData *pCipherSetupData, Cpa32U targetKeyLenInBytes, + Cpa32U sliceType, const void *pCipherHwBlock, Cpa32U *pSizeInBytes) { @@ -635,6 +655,20 @@ LacSymQat_CipherHwBlockPopulateKeySetup( * Arc4 and Null cipher */ if (!(LAC_CIPHER_IS_ARC4(pCipherSetupData->cipherAlgorithm) || LAC_CIPHER_IS_NULL(pCipherSetupData->cipherAlgorithm))) { + /* Special handling of AES 192 key for UCS slice. + UCS requires it to have 32 bytes - set is as targetKeyLen + in this case, and add padding. It makes no sense + to force applications to provide such key length for couple + reasons: + 1. It won't be possible to distinguish between AES 192 and + 256 based on key lenght only + 2. Only some modes of AES will use UCS slice, then + application will have to know which ones */ + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == sliceType && + ICP_QAT_HW_AES_192_KEY_SZ == targetKeyLenInBytes) { + targetKeyLenInBytes = ICP_QAT_HW_UCS_AES_192_KEY_SZ; + } + /* Set the Cipher key field in the cipher block */ memcpy(pCipherKey, pCipherSetupData->pCipherKey, @@ -646,9 +680,10 @@ LacSymQat_CipherHwBlockPopulateKeySetup( } *pSizeInBytes += targetKeyLenInBytes; - /* For Kasumi in F8 mode Cipher Key is concatenated with - * Cipher Key XOR-ed with Key Modifier (CK||CK^KM) */ - if (LAC_CIPHER_IS_KASUMI(pCipherSetupData->cipherAlgorithm)) { + switch (pCipherSetupData->cipherAlgorithm) { + /* For Kasumi in F8 mode Cipher Key is concatenated with + * Cipher Key XOR-ed with Key Modifier (CK||CK^KM) */ + case CPA_CY_SYM_CIPHER_KASUMI_F8: { Cpa32U wordIndex = 0; Cpa32U *pu32CipherKey = (Cpa32U *)pCipherSetupData->pCipherKey; @@ -672,11 +707,10 @@ LacSymQat_CipherHwBlockPopulateKeySetup( LAC_OS_BZERO((Cpa8U *)pTempKey + targetKeyLenInBytes, LAC_QUADWORDS_TO_BYTES( ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR)); - } - /* For AES in F8 mode Cipher Key is concatenated with - * Cipher Key XOR-ed with Key Mask (CK||CK^KM) */ - else if (LAC_CIPHER_IS_AES_F8( - pCipherSetupData->cipherAlgorithm)) { + } break; + /* For AES in F8 mode Cipher Key is concatenated with + * Cipher Key XOR-ed with Key Mask (CK||CK^KM) */ + case CPA_CY_SYM_CIPHER_AES_F8: { Cpa32U index = 0; Cpa8U *pTempKey = pCipherKey + (targetKeyLenInBytes / 2); @@ -690,20 +724,59 @@ LacSymQat_CipherHwBlockPopulateKeySetup( /* also add padding for AES F8 */ *pSizeInBytes += 2 * targetKeyLenInBytes; LAC_OS_BZERO(pTempKey, 2 * targetKeyLenInBytes); - } else if (LAC_CIPHER_IS_SNOW3G_UEA2( - pCipherSetupData->cipherAlgorithm)) { + } break; + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: { /* For Snow3G zero area after the key for FW */ LAC_OS_BZERO(pCipherKey + targetKeyLenInBytes, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); *pSizeInBytes += ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; - } else if (LAC_CIPHER_IS_ZUC_EEA3( - pCipherSetupData->cipherAlgorithm)) { + } break; + case CPA_CY_SYM_CIPHER_ZUC_EEA3: { /* For ZUC zero area after the key for FW */ LAC_OS_BZERO(pCipherKey + targetKeyLenInBytes, ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); *pSizeInBytes += ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; + } break; + case CPA_CY_SYM_CIPHER_AES_XTS: { + /* For AES in XTS mode Cipher Key is concatenated with + * second Cipher Key which is used for tweak calculation + * (CK1||CK2). For decryption Cipher Key needs to be + * converted to reverse key.*/ + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == sliceType) { + Cpa32U key_len = + pCipherSetupData->cipherKeyLenInBytes / 2; + memcpy(pSessionDesc->cipherAesXtsKey1Forward, + pCipherSetupData->pCipherKey, + key_len); + + qatUtilsAESKeyExpansionForward( + pSessionDesc->cipherAesXtsKey1Forward, + key_len, + (uint32_t *) + pSessionDesc->cipherAesXtsKey1Reverse); + + memcpy(pSessionDesc->cipherAesXtsKey2, + pCipherSetupData->pCipherKey + key_len, + key_len); + + if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT == + pCipherSetupData->cipherDirection) { + memcpy(pCipherKey, + pSessionDesc + ->cipherAesXtsKey1Reverse, + key_len); + } else { + memcpy(pCipherKey, + pSessionDesc + ->cipherAesXtsKey1Forward, + key_len); + } + } + } break; + default: + break; } } } @@ -715,56 +788,94 @@ LacSymQat_CipherHwBlockPopulateKeySetup( Cpa8U LacSymQat_CipherBlockSizeBytesGet(CpaCySymCipherAlgorithm cipherAlgorithm) { - if (LAC_CIPHER_IS_ARC4(cipherAlgorithm)) { - return LAC_CIPHER_ARC4_BLOCK_LEN_BYTES; - } else if (LAC_CIPHER_IS_AES(cipherAlgorithm) || - LAC_CIPHER_IS_AES_F8(cipherAlgorithm)) { - return ICP_QAT_HW_AES_BLK_SZ; - } else if (LAC_CIPHER_IS_DES(cipherAlgorithm)) { - return ICP_QAT_HW_DES_BLK_SZ; - } else if (LAC_CIPHER_IS_TRIPLE_DES(cipherAlgorithm)) { - return ICP_QAT_HW_3DES_BLK_SZ; - } else if (LAC_CIPHER_IS_KASUMI(cipherAlgorithm)) { - return ICP_QAT_HW_KASUMI_BLK_SZ; - } else if (LAC_CIPHER_IS_SNOW3G_UEA2(cipherAlgorithm)) { - return ICP_QAT_HW_SNOW_3G_BLK_SZ; - } else if (LAC_CIPHER_IS_ZUC_EEA3(cipherAlgorithm)) { - return ICP_QAT_HW_ZUC_3G_BLK_SZ; - } else if (LAC_CIPHER_IS_NULL(cipherAlgorithm)) { - return LAC_CIPHER_NULL_BLOCK_LEN_BYTES; - } else if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) { - return ICP_QAT_HW_CHACHAPOLY_BLK_SZ; - } else if (LAC_CIPHER_IS_SM4(cipherAlgorithm)) { - return ICP_QAT_HW_SM4_BLK_SZ; - } else { - QAT_UTILS_LOG("Algorithm not supported in Cipher\n"); - return 0; + Cpa8U blockSize = 0; + switch (cipherAlgorithm) { + case CPA_CY_SYM_CIPHER_ARC4: + blockSize = LAC_CIPHER_ARC4_BLOCK_LEN_BYTES; + break; + /* Handle AES or AES_F8 */ + case CPA_CY_SYM_CIPHER_AES_ECB: + case CPA_CY_SYM_CIPHER_AES_CBC: + case CPA_CY_SYM_CIPHER_AES_CTR: + case CPA_CY_SYM_CIPHER_AES_CCM: + case CPA_CY_SYM_CIPHER_AES_GCM: + case CPA_CY_SYM_CIPHER_AES_XTS: + case CPA_CY_SYM_CIPHER_AES_F8: + blockSize = ICP_QAT_HW_AES_BLK_SZ; + break; + /* Handle DES */ + case CPA_CY_SYM_CIPHER_DES_ECB: + case CPA_CY_SYM_CIPHER_DES_CBC: + blockSize = ICP_QAT_HW_DES_BLK_SZ; + break; + /* Handle TRIPLE DES */ + case CPA_CY_SYM_CIPHER_3DES_ECB: + case CPA_CY_SYM_CIPHER_3DES_CBC: + case CPA_CY_SYM_CIPHER_3DES_CTR: + blockSize = ICP_QAT_HW_3DES_BLK_SZ; + break; + case CPA_CY_SYM_CIPHER_KASUMI_F8: + blockSize = ICP_QAT_HW_KASUMI_BLK_SZ; + break; + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: + blockSize = ICP_QAT_HW_SNOW_3G_BLK_SZ; + break; + case CPA_CY_SYM_CIPHER_ZUC_EEA3: + blockSize = ICP_QAT_HW_ZUC_3G_BLK_SZ; + break; + case CPA_CY_SYM_CIPHER_NULL: + blockSize = LAC_CIPHER_NULL_BLOCK_LEN_BYTES; + break; + case CPA_CY_SYM_CIPHER_CHACHA: + blockSize = ICP_QAT_HW_CHACHAPOLY_BLK_SZ; + break; + case CPA_CY_SYM_CIPHER_SM4_ECB: + case CPA_CY_SYM_CIPHER_SM4_CBC: + case CPA_CY_SYM_CIPHER_SM4_CTR: + blockSize = ICP_QAT_HW_SM4_BLK_SZ; + break; + default: + QAT_UTILS_LOG("Algorithm not supported in Cipher"); } + return blockSize; } Cpa32U LacSymQat_CipherIvSizeBytesGet(CpaCySymCipherAlgorithm cipherAlgorithm) { - if (CPA_CY_SYM_CIPHER_ARC4 == cipherAlgorithm) { - return LAC_CIPHER_ARC4_STATE_LEN_BYTES; - } else if (LAC_CIPHER_IS_KASUMI(cipherAlgorithm)) { - return ICP_QAT_HW_KASUMI_BLK_SZ; - } else if (LAC_CIPHER_IS_SNOW3G_UEA2(cipherAlgorithm)) { - return ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; - } else if (LAC_CIPHER_IS_ZUC_EEA3(cipherAlgorithm)) { - return ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; - } else if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) { - return ICP_QAT_HW_CHACHAPOLY_IV_SZ; - } else if (LAC_CIPHER_IS_ECB_MODE(cipherAlgorithm)) { - return 0; - } else { - return (Cpa32U)LacSymQat_CipherBlockSizeBytesGet( - cipherAlgorithm); + Cpa32U ivSize = 0; + switch (cipherAlgorithm) { + case CPA_CY_SYM_CIPHER_ARC4: + ivSize = LAC_CIPHER_ARC4_STATE_LEN_BYTES; + break; + case CPA_CY_SYM_CIPHER_KASUMI_F8: + ivSize = ICP_QAT_HW_KASUMI_BLK_SZ; + break; + case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: + ivSize = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; + break; + case CPA_CY_SYM_CIPHER_ZUC_EEA3: + ivSize = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; + break; + case CPA_CY_SYM_CIPHER_CHACHA: + ivSize = ICP_QAT_HW_CHACHAPOLY_IV_SZ; + break; + case CPA_CY_SYM_CIPHER_AES_ECB: + case CPA_CY_SYM_CIPHER_DES_ECB: + case CPA_CY_SYM_CIPHER_3DES_ECB: + case CPA_CY_SYM_CIPHER_SM4_ECB: + case CPA_CY_SYM_CIPHER_NULL: + /* for all ECB Mode IV size is 0 */ + break; + default: + ivSize = LacSymQat_CipherBlockSizeBytesGet(cipherAlgorithm); } + return ivSize; } inline CpaStatus -LacSymQat_CipherRequestParamsPopulate(icp_qat_fw_la_bulk_req_t *pReq, +LacSymQat_CipherRequestParamsPopulate(lac_session_desc_t *pSessionDesc, + icp_qat_fw_la_bulk_req_t *pReq, Cpa32U cipherOffsetInBytes, Cpa32U cipherLenInBytes, Cpa64U ivBufferPhysAddr, @@ -773,6 +884,8 @@ LacSymQat_CipherRequestParamsPopulate(icp_qat_fw_la_bulk_req_t *pReq, icp_qat_fw_la_cipher_req_params_t *pCipherReqParams; icp_qat_fw_cipher_cd_ctrl_hdr_t *pCipherCdCtrlHdr; icp_qat_fw_serv_specif_flags *pCipherSpecificFlags; + Cpa32U usedBufSize = 0; + Cpa32U totalBufSize = 0; pCipherReqParams = (icp_qat_fw_la_cipher_req_params_t *)((Cpa8U *)&(pReq->serv_specif_rqpars) + @@ -805,31 +918,40 @@ LacSymQat_CipherRequestParamsPopulate(icp_qat_fw_la_bulk_req_t *pReq, } else { /* Populate the field with the contents of the buffer, * zero field first as data may be smaller than the field */ - memset(pCipherReqParams->u.cipher_IV_array, - 0, - LAC_LONGWORDS_TO_BYTES(ICP_QAT_FW_NUM_LONGWORDS_4)); - /* We force a specific compiler optimisation here. The length - * to - * be copied turns out to be always 16, and by coding a memcpy - * with - * a literal value the compiler will compile inline code (in - * fact, - * only two vector instructions) to effect the copy. This gives - * us - * a huge performance increase. - */ - unsigned long cplen = - LAC_QUADWORDS_TO_BYTES(pCipherCdCtrlHdr->cipher_state_sz); - - if (cplen == 16) + /* In case of XTS mode using UCS slice always embedd IV. + * IV provided by user needs to be encrypted to calculate + * initial tweak, use pCipherReqParams->u.cipher_IV_array as + * destination buffer for tweak value */ + if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == + pSessionDesc->cipherSliceType && + LAC_CIPHER_IS_XTS_MODE(pSessionDesc->cipherAlgorithm)) { + memset(pCipherReqParams->u.cipher_IV_array, + 0, + LAC_LONGWORDS_TO_BYTES( + ICP_QAT_FW_NUM_LONGWORDS_4)); + qatUtilsAESEncrypt( + pSessionDesc->cipherAesXtsKey2, + pSessionDesc->cipherKeyLenInBytes / 2, + pIvBufferVirt, + (Cpa8U *)pCipherReqParams->u.cipher_IV_array); + } else { + totalBufSize = + LAC_LONGWORDS_TO_BYTES(ICP_QAT_FW_NUM_LONGWORDS_4); + usedBufSize = LAC_QUADWORDS_TO_BYTES( + pCipherCdCtrlHdr->cipher_state_sz); + /* Only initialise unused buffer if applicable*/ + if (usedBufSize < totalBufSize) { + memset( + (&pCipherReqParams->u.cipher_IV_array + [usedBufSize & LAC_UNUSED_POS_MASK]), + 0, + totalBufSize - usedBufSize); + } memcpy(pCipherReqParams->u.cipher_IV_array, pIvBufferVirt, - 16); - else - memcpy(pCipherReqParams->u.cipher_IV_array, - pIvBufferVirt, - cplen); + usedBufSize); + } /* Set the flag indicating the field format */ ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( *pCipherSpecificFlags, ICP_QAT_FW_CIPH_IV_16BYTE_DATA); @@ -851,7 +973,7 @@ LacSymQat_CipherArc4StateInit(const Cpa8U *pKey, pArc4CipherState[i] = (Cpa8U)i; } - for (i = 0; i < LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES; ++i) { + for (i = 0, k = 0; i < LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES; ++i, ++k) { Cpa8U swap = 0; if (k >= keyLenInBytes) @@ -860,7 +982,6 @@ LacSymQat_CipherArc4StateInit(const Cpa8U *pKey, j = (j + pArc4CipherState[i] + pKey[k]); if (j >= LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES) j %= LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES; - ++k; /* Swap state[i] & state[j] */ swap = pArc4CipherState[i]; diff --git a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_constants_table.c b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_constants_table.c new file mode 100644 index 000000000000..00b54ac8842b --- /dev/null +++ b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_constants_table.c @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2022 Intel Corporation */ +/* $FreeBSD$ */ + +/** + *************************************************************************** + * @file lac_sym_qat_constants_table.c + * + * @ingroup LacSymQat + ***************************************************************************/ + +/* +******************************************************************************* +* Include public/global header files +******************************************************************************* +*/ + +#include "cpa.h" + +/* +******************************************************************************* +* Include private header files +******************************************************************************* +*/ + +#include "lac_common.h" +#include "icp_qat_fw_la.h" +#include "lac_log.h" +#include "lac_mem.h" +#include "sal_string_parse.h" +#include "lac_sal_types_crypto.h" +#include "sal_types_compression.h" + +static uint8_t icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_DELIMITER] + [ICP_QAT_HW_CIPHER_MODE_DELIMITER][2] + [2]; /* IA version */ +static uint8_t icp_qat_hw_auth_lookup_tbl[ICP_QAT_HW_AUTH_ALGO_DELIMITER] + [ICP_QAT_HW_AUTH_MODE_DELIMITER] + [2]; /* IA version */ + +#define ICP_QAT_HW_FILL_LOOKUP_TBLS \ + { \ + \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 9; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_DECRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 10; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \ + [ICP_QAT_HW_CIPHER_CBC_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 11; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \ + [ICP_QAT_HW_CIPHER_CBC_MODE] \ + [ICP_QAT_HW_CIPHER_DECRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 12; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \ + [ICP_QAT_HW_CIPHER_CTR_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 13; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 14; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \ + 15; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_DECRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 16; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_DECRYPT] \ + [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \ + 17; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_CBC_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 18; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_CBC_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \ + 19; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_CBC_MODE] \ + [ICP_QAT_HW_CIPHER_DECRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 20; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_CBC_MODE] \ + [ICP_QAT_HW_CIPHER_DECRYPT] \ + [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \ + 21; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_CTR_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 22; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \ + [ICP_QAT_HW_CIPHER_F8_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 23; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_ARC4] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_NO_CONVERT] = \ + 24; \ + icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_ARC4] \ + [ICP_QAT_HW_CIPHER_ECB_MODE] \ + [ICP_QAT_HW_CIPHER_ENCRYPT] \ + [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \ + 25; \ + \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_MD5][ICP_QAT_HW_AUTH_MODE0] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 37; \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_SHA1][ICP_QAT_HW_AUTH_MODE0] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 41; \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_SHA1][ICP_QAT_HW_AUTH_MODE1] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 46; \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_SHA224][ICP_QAT_HW_AUTH_MODE0] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 48; \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_SHA256][ICP_QAT_HW_AUTH_MODE0] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 54; \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_SHA384][ICP_QAT_HW_AUTH_MODE0] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 60; \ + icp_qat_hw_auth_lookup_tbl \ + [ICP_QAT_HW_AUTH_ALGO_SHA512][ICP_QAT_HW_AUTH_MODE0] \ + [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 70; \ + } + +/** + ***************************************************************************** + * @ingroup LacSymQat + * LacSymQat_ConstantsInitLookupTables + * + * + *****************************************************************************/ +void +LacSymQat_ConstantsInitLookupTables(CpaInstanceHandle instanceHandle) +{ + sal_service_t *pService = (sal_service_t *)instanceHandle; + lac_sym_qat_constants_t *pConstantsLookupTables; + + /* Note the global tables are initialised first, then copied + * to the service which probably seems like a waste of memory + * and processing cycles as the global tables are never needed again + * but this allows use of the ICP_QAT_HW_FILL_LOOKUP_TBLS macro + * supplied by FW without modification */ + + if (SAL_SERVICE_TYPE_COMPRESSION == pService->type) { + /* DC chaining not supported yet */ + return; + } else { + pConstantsLookupTables = &( + ((sal_crypto_service_t *)pService)->constantsLookupTables); + } + + /* First fill the global lookup tables with zeroes. */ + memset(icp_qat_hw_cipher_lookup_tbl, + 0, + sizeof(icp_qat_hw_cipher_lookup_tbl)); + memset(icp_qat_hw_auth_lookup_tbl, + 0, + sizeof(icp_qat_hw_auth_lookup_tbl)); + + /* Override lookup tables with the offsets into the SHRAM table + * for supported algorithms/modes */ + ICP_QAT_HW_FILL_LOOKUP_TBLS; + + /* Copy the global tables to the service instance */ + memcpy(pConstantsLookupTables->cipher_offset, + icp_qat_hw_cipher_lookup_tbl, + sizeof(pConstantsLookupTables->cipher_offset)); + memcpy(pConstantsLookupTables->auth_offset, + icp_qat_hw_auth_lookup_tbl, + sizeof(pConstantsLookupTables->auth_offset)); +} + +/** + ***************************************************************************** + * @ingroup LacSymQat + * LacSymQat_ConstantsGetCipherOffset + * + * + *****************************************************************************/ +void +LacSymQat_ConstantsGetCipherOffset(CpaInstanceHandle instanceHandle, + uint8_t algo, + uint8_t mode, + uint8_t direction, + uint8_t convert, + uint8_t *poffset) +{ + sal_service_t *pService = (sal_service_t *)instanceHandle; + lac_sym_qat_constants_t *pConstantsLookupTables; + + if (SAL_SERVICE_TYPE_COMPRESSION == pService->type) { + /* DC chaining not supported yet */ + return; + } else { + pConstantsLookupTables = &( + ((sal_crypto_service_t *)pService)->constantsLookupTables); + } + + *poffset = pConstantsLookupTables + ->cipher_offset[algo][mode][direction][convert]; +} + +/** + ***************************************************************************** + * @ingroup LacSymQat + * LacSymQat_ConstantsGetAuthOffset + * + * + *****************************************************************************/ +void +LacSymQat_ConstantsGetAuthOffset(CpaInstanceHandle instanceHandle, + uint8_t algo, + uint8_t mode, + uint8_t nested, + uint8_t *poffset) +{ + sal_service_t *pService = (sal_service_t *)instanceHandle; + lac_sym_qat_constants_t *pConstantsLookupTables; + + if (SAL_SERVICE_TYPE_COMPRESSION == pService->type) { + /* DC chaining not supported yet */ + return; + } else { + pConstantsLookupTables = &( + ((sal_crypto_service_t *)pService)->constantsLookupTables); + } + + *poffset = pConstantsLookupTables->auth_offset[algo][mode][nested]; +} diff --git a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c index a39734ad16d0..b62ecf271d80 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c @@ -28,8 +28,10 @@ #include "lac_sym_qat.h" #include "lac_list.h" #include "lac_sal_types.h" +#include "lac_sal_types_crypto.h" #include "lac_sym_qat_hash.h" #include "lac_sym_qat_hash_defs_lookup.h" +#include "sal_hw_gen.h" /** * This structure contains pointers into the hash setup block of the @@ -148,10 +150,10 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, icp_qat_hw_auth_mode_t qatHashMode, CpaBoolean useSymConstantsTable, CpaBoolean useOptimisedContentDesc, + CpaBoolean useStatefulSha3ContentDesc, lac_sym_qat_hash_precompute_info_t *pPrecompute, Cpa32U *pHashBlkSizeInBytes) { - icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); lac_sym_qat_hash_defs_t *pHashDefs = NULL; @@ -159,7 +161,7 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, Cpa32U hashSetupBlkSize = 0; /* setup the offset in QuadWords into the hw blk */ - cd_ctrl->hash_cfg_offset = hwBlockOffsetInQuadWords; + cd_ctrl->hash_cfg_offset = (Cpa8U)hwBlockOffsetInQuadWords; ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, nextSlice); ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_AUTH); @@ -170,11 +172,19 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, /* Hmac in mode 2 TLS */ if (IS_HASH_MODE_2(qatHashMode)) { - /* Set bit for nested hashing. - * Make sure not to overwrite other flags in hash_flags byte. - */ - ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( - cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED); + if (isCyGen4x((sal_crypto_service_t *)instanceHandle)) { + /* CPM2.0 has a dedicated bit for HMAC mode2 */ + ICP_QAT_FW_HASH_FLAG_MODE2_SET(cd_ctrl->hash_flags, + QAT_FW_LA_MODE2); + } else { + /* Set bit for nested hashing. + * Make sure not to overwrite other flags in hash_flags + * byte. + */ + ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( + cd_ctrl->hash_flags, + ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED); + } } /* Nested hash in mode 0 */ else if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) { @@ -190,16 +200,32 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED); } + /* Set skip state load flags */ + if (useStatefulSha3ContentDesc) { + /* Here both skip state load flags are set. FW reads them based + * on partial packet type. */ + ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( + cd_ctrl->hash_flags, QAT_FW_LA_SKIP_INNER_STATE1_LOAD); + ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( + cd_ctrl->hash_flags, QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); + } + /* set the final digest size */ - cd_ctrl->final_sz = pHashSetupData->digestResultLenInBytes; + cd_ctrl->final_sz = (Cpa8U)pHashSetupData->digestResultLenInBytes; /* set the state1 size */ - cd_ctrl->inner_state1_sz = - LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state1Length, - LAC_QUAD_WORD_IN_BYTES); + if (useStatefulSha3ContentDesc) { + cd_ctrl->inner_state1_sz = + LAC_ALIGN_POW2_ROUNDUP(LAC_HASH_SHA3_STATEFUL_STATE_SIZE, + LAC_QUAD_WORD_IN_BYTES); + } else { + cd_ctrl->inner_state1_sz = + LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state1Length, + LAC_QUAD_WORD_IN_BYTES); + } /* set the inner result size to the digest length */ - cd_ctrl->inner_res_sz = pHashDefs->algInfo->digestLength; + cd_ctrl->inner_res_sz = (Cpa8U)pHashDefs->algInfo->digestLength; /* set the state2 size - only for mode 1 Auth algos and AES CBC MAC */ if (IS_HASH_MODE_1(qatHashMode) || @@ -212,13 +238,22 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, cd_ctrl->inner_state2_sz = 0; } - cd_ctrl->inner_state2_offset = cd_ctrl->hash_cfg_offset + - LAC_BYTES_TO_QUADWORDS(sizeof(icp_qat_hw_auth_setup_t) + - cd_ctrl->inner_state1_sz); + if (useSymConstantsTable) { + cd_ctrl->inner_state2_offset = + LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state1_sz); - /* size of inner part of hash setup block */ - hashSetupBlkSize = sizeof(icp_qat_hw_auth_setup_t) + - cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; + /* size of inner part of hash setup block */ + hashSetupBlkSize = + cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; + } else { + cd_ctrl->inner_state2_offset = cd_ctrl->hash_cfg_offset + + LAC_BYTES_TO_QUADWORDS(sizeof(icp_qat_hw_auth_setup_t) + + cd_ctrl->inner_state1_sz); + + /* size of inner part of hash setup block */ + hashSetupBlkSize = sizeof(icp_qat_hw_auth_setup_t) + + cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; + } /* For nested hashing - Fill in the outer fields */ if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode || @@ -238,18 +273,24 @@ LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, cd_ctrl->outer_config_offset = cd_ctrl->inner_state2_offset + LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state2_sz); - cd_ctrl->outer_state1_sz = - LAC_ALIGN_POW2_ROUNDUP(pOuterHashDefs->algInfo->stateSize, - LAC_QUAD_WORD_IN_BYTES); + if (useStatefulSha3ContentDesc) { + cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP( + LAC_HASH_SHA3_STATEFUL_STATE_SIZE, + LAC_QUAD_WORD_IN_BYTES); + } else { + cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP( + pOuterHashDefs->algInfo->stateSize, + LAC_QUAD_WORD_IN_BYTES); + } /* outer result size */ - cd_ctrl->outer_res_sz = pOuterHashDefs->algInfo->digestLength; + cd_ctrl->outer_res_sz = + (Cpa8U)pOuterHashDefs->algInfo->digestLength; /* outer_prefix_offset will be the size of the inner prefix data * plus the hash state storage size. */ /* The prefix buffer is part of the ReqParams, so this param - * will be - * setup where ReqParams are set up */ + * will be setup where ReqParams are set up */ /* add on size of outer part of hash block */ hashSetupBlkSize += @@ -325,8 +366,9 @@ LacSymQat_HashSetupReqParamsMetaData( if (IS_HASH_MODE_2(qatHashMode)) { /* Inner and outer prefixes are the block length */ pHashReqParams->u2.inner_prefix_sz = - pHashDefs->algInfo->blockLength; - cd_ctrl->outer_prefix_sz = pHashDefs->algInfo->blockLength; + (Cpa8U)pHashDefs->algInfo->blockLength; + cd_ctrl->outer_prefix_sz = + (Cpa8U)pHashDefs->algInfo->blockLength; cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS( LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz), LAC_QUAD_WORD_IN_BYTES)); @@ -336,9 +378,11 @@ LacSymQat_HashSetupReqParamsMetaData( /* set inner and outer prefixes */ pHashReqParams->u2.inner_prefix_sz = - pHashSetupData->nestedModeSetupData.innerPrefixLenInBytes; + (Cpa8U)pHashSetupData->nestedModeSetupData + .innerPrefixLenInBytes; cd_ctrl->outer_prefix_sz = - pHashSetupData->nestedModeSetupData.outerPrefixLenInBytes; + (Cpa8U)pHashSetupData->nestedModeSetupData + .outerPrefixLenInBytes; cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS( LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz), LAC_QUAD_WORD_IN_BYTES)); @@ -363,8 +407,9 @@ LacSymQat_HashSetupReqParamsMetaData( * just need 2 bytes to store encoded length of * 'a' */ aadDataSize += sizeof(Cpa16U); - aadDataSize += pHashSetupData->authModeSetupData - .aadLenInBytes; + aadDataSize += + (Cpa16U)pHashSetupData->authModeSetupData + .aadLenInBytes; } /* round the aad size to the multiple of CCM block @@ -375,7 +420,8 @@ LacSymQat_HashSetupReqParamsMetaData( } else if (CPA_CY_SYM_HASH_AES_GCM == pHashSetupData->hashAlgorithm) { aadDataSize = - pHashSetupData->authModeSetupData.aadLenInBytes; + (Cpa16U) + pHashSetupData->authModeSetupData.aadLenInBytes; /* round the aad size to the multiple of GCM hash block * size. */ @@ -406,7 +452,7 @@ LacSymQat_HashSetupReqParamsMetaData( /* auth result size in bytes to be read in for a verify * operation */ pHashReqParams->auth_res_sz = - pHashSetupData->digestResultLenInBytes; + (Cpa8U)pHashSetupData->digestResultLenInBytes; } else { pHashReqParams->auth_res_sz = 0; } @@ -453,7 +499,7 @@ LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, { Cpa32U innerConfig = 0; lac_hash_blk_ptrs_t hashBlkPtrs = { 0 }; - Cpa32U aed_hash_cmp_length = 0; + Cpa32U aedHashCmpLength = 0; LacSymQat_HashHwBlockPtrsInit(pHashControlBlock, pHwBlockBase, @@ -610,7 +656,7 @@ LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_ENCRYPT, - aed_hash_cmp_length); + aedHashCmpLength); pCipherConfig->reserved = 0; @@ -633,7 +679,7 @@ LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3, ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_ENCRYPT, - aed_hash_cmp_length); + aedHashCmpLength); pCipherConfig->reserved = 0; @@ -820,7 +866,7 @@ LacSymQat_HashRequestParamsPopulate( CpaBoolean digestVerify, Cpa8U *pAuthResult, CpaCySymHashAlgorithm alg, - void *hkdf_secret) + void *pHKDFSecret) { Cpa64U authResultPhys = 0; icp_qat_fw_la_auth_req_params_t *pHashReqParams; @@ -833,11 +879,11 @@ LacSymQat_HashRequestParamsPopulate( pHashReqParams->auth_len = authLenInBytes; /* Set the physical location of secret for HKDF */ - if (NULL != hkdf_secret) { + if (NULL != pHKDFSecret) { LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL( - (*pService), pHashReqParams->u1.aad_adr, hkdf_secret); + (*pService), pHashReqParams->u1.aad_adr, pHKDFSecret); - if (pHashReqParams->u1.aad_adr == 0) { + if (0 == pHashReqParams->u1.aad_adr) { LAC_LOG_ERROR( "Unable to get the physical address of the" " HKDF secret\n"); @@ -868,7 +914,7 @@ LacSymQat_HashRequestParamsPopulate( if (CPA_TRUE == digestVerify) { /* auth result size in bytes to be read in for a verify * operation */ - pHashReqParams->auth_res_sz = hashResultSize; + pHashReqParams->auth_res_sz = (Cpa8U)hashResultSize; } else { pHashReqParams->auth_res_sz = 0; } diff --git a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c index 23a77f19cca6..5a13241eaf01 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c @@ -74,62 +74,70 @@ static Cpa8U sha224InitialState[LAC_HASH_SHA224_STATE_SIZE] = { }; /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ -static Cpa8U sha256InitialState[LAC_HASH_SHA256_STATE_SIZE] = - { 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, - 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, - 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19 }; +static Cpa8U sha256InitialState[LAC_HASH_SHA256_STATE_SIZE] = { + 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, + 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, + 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19 +}; /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ -static Cpa8U sha384InitialState[LAC_HASH_SHA384_STATE_SIZE] = - { 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, - 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, - 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, - 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, - 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, - 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 }; +static Cpa8U sha384InitialState[LAC_HASH_SHA384_STATE_SIZE] = { + 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, + 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, + 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, + 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, + 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, + 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 +}; /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ -static Cpa8U sha512InitialState[LAC_HASH_SHA512_STATE_SIZE] = - { 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, - 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, - 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, - 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, - 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, - 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 }; +static Cpa8U sha512InitialState[LAC_HASH_SHA512_STATE_SIZE] = { + 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, + 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, + 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, + 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, + 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, + 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 +}; /* SHA3 224 - 28 bytes */ -static Cpa8U sha3_224InitialState[LAC_HASH_SHA3_224_STATE_SIZE] = - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static Cpa8U sha3_224InitialState[LAC_HASH_SHA3_224_STATE_SIZE] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; /* SHA3 256 - 32 bytes */ -static Cpa8U sha3_256InitialState[LAC_HASH_SHA3_256_STATE_SIZE] = - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static Cpa8U sha3_256InitialState[LAC_HASH_SHA3_256_STATE_SIZE] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; /* SHA3 384 - 48 bytes */ -static Cpa8U sha3_384InitialState[LAC_HASH_SHA3_384_STATE_SIZE] = - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static Cpa8U sha3_384InitialState[LAC_HASH_SHA3_384_STATE_SIZE] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; /* SHA3 512 - 64 bytes */ -static Cpa8U sha3_512InitialState[LAC_HASH_SHA3_512_STATE_SIZE] = - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static Cpa8U sha3_512InitialState[LAC_HASH_SHA3_512_STATE_SIZE] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; /* SM3 - 32 bytes */ -static Cpa8U sm3InitialState[LAC_HASH_SM3_STATE_SIZE] = - { 0x73, 0x80, 0x16, 0x6f, 0x49, 0x14, 0xb2, 0xb9, 0x17, 0x24, 0x42, - 0xd7, 0xda, 0x8a, 0x06, 0x00, 0xa9, 0x6f, 0x30, 0xbc, 0x16, 0x31, - 0x38, 0xaa, 0xe3, 0x8d, 0xee, 0x4d, 0xb0, 0xfb, 0x0e, 0x4e }; +static Cpa8U sm3InitialState[LAC_HASH_SM3_STATE_SIZE] = { + 0x73, 0x80, 0x16, 0x6f, 0x49, 0x14, 0xb2, 0xb9, 0x17, 0x24, 0x42, + 0xd7, 0xda, 0x8a, 0x06, 0x00, 0xa9, 0x6f, 0x30, 0xbc, 0x16, 0x31, + 0x38, 0xaa, 0xe3, 0x8d, 0xee, 0x4d, 0xb0, 0xfb, 0x0e, 0x4e +}; /* Constants used in generating K1, K2, K3 from a Key for AES_XCBC_MAC * State defined in RFC 3566 */ @@ -140,12 +148,10 @@ static Cpa8U aesXcbcKeySeed[LAC_SYM_QAT_XCBC_STATE_SIZE] = { 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, }; -static Cpa8U aesCmacKeySeed[LAC_HASH_CMAC_BLOCK_SIZE] = { 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - 0x00 }; +static Cpa8U aesCmacKeySeed[LAC_HASH_CMAC_BLOCK_SIZE] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; /* Hash Algorithm specific structure */ @@ -179,57 +185,57 @@ static lac_sym_qat_hash_alg_info_t sha512Info = { LAC_HASH_SHA512_DIGEST_SIZE, sha512InitialState, LAC_HASH_SHA512_STATE_SIZE }; -static lac_sym_qat_hash_alg_info_t sha3_224Info = - { LAC_HASH_SHA3_224_DIGEST_SIZE, - LAC_HASH_SHA3_224_BLOCK_SIZE, - sha3_224InitialState, - LAC_HASH_SHA3_224_STATE_SIZE }; +static lac_sym_qat_hash_alg_info_t sha3_224Info = { + LAC_HASH_SHA3_224_DIGEST_SIZE, + LAC_HASH_SHA3_224_BLOCK_SIZE, + sha3_224InitialState, + LAC_HASH_SHA3_224_STATE_SIZE +}; -static lac_sym_qat_hash_alg_info_t sha3_256Info = - { LAC_HASH_SHA3_256_DIGEST_SIZE, - LAC_HASH_SHA3_256_BLOCK_SIZE, - sha3_256InitialState, - LAC_HASH_SHA3_256_STATE_SIZE }; +static lac_sym_qat_hash_alg_info_t sha3_256Info = { + LAC_HASH_SHA3_256_DIGEST_SIZE, + LAC_HASH_SHA3_256_BLOCK_SIZE, + sha3_256InitialState, + LAC_HASH_SHA3_256_STATE_SIZE +}; -static lac_sym_qat_hash_alg_info_t sha3_384Info = - { LAC_HASH_SHA3_384_DIGEST_SIZE, - LAC_HASH_SHA3_384_BLOCK_SIZE, - sha3_384InitialState, - LAC_HASH_SHA3_384_STATE_SIZE }; +static lac_sym_qat_hash_alg_info_t sha3_384Info = { + LAC_HASH_SHA3_384_DIGEST_SIZE, + LAC_HASH_SHA3_384_BLOCK_SIZE, + sha3_384InitialState, + LAC_HASH_SHA3_384_STATE_SIZE +}; -static lac_sym_qat_hash_alg_info_t sha3_512Info = - { LAC_HASH_SHA3_512_DIGEST_SIZE, - LAC_HASH_SHA3_512_BLOCK_SIZE, - sha3_512InitialState, - LAC_HASH_SHA3_512_STATE_SIZE }; - -static lac_sym_qat_hash_alg_info_t polyInfo = { LAC_HASH_POLY_DIGEST_SIZE, - LAC_HASH_POLY_BLOCK_SIZE, - NULL, /* intial state */ - LAC_HASH_POLY_STATE_SIZE }; - -static lac_sym_qat_hash_alg_info_t shake_128Info = - { LAC_HASH_SHAKE_128_DIGEST_SIZE, LAC_HASH_SHAKE_128_BLOCK_SIZE, NULL, 0 }; - -static lac_sym_qat_hash_alg_info_t shake_256Info = - { LAC_HASH_SHAKE_256_DIGEST_SIZE, LAC_HASH_SHAKE_256_BLOCK_SIZE, NULL, 0 }; +static lac_sym_qat_hash_alg_info_t sha3_512Info = { + LAC_HASH_SHA3_512_DIGEST_SIZE, + LAC_HASH_SHA3_512_BLOCK_SIZE, + sha3_512InitialState, + LAC_HASH_SHA3_512_STATE_SIZE +}; static lac_sym_qat_hash_alg_info_t sm3Info = { LAC_HASH_SM3_DIGEST_SIZE, LAC_HASH_SM3_BLOCK_SIZE, sm3InitialState, LAC_HASH_SM3_STATE_SIZE }; -static lac_sym_qat_hash_alg_info_t xcbcMacInfo = - { LAC_HASH_XCBC_MAC_128_DIGEST_SIZE, - LAC_HASH_XCBC_MAC_BLOCK_SIZE, - aesXcbcKeySeed, - LAC_SYM_QAT_XCBC_STATE_SIZE }; +static lac_sym_qat_hash_alg_info_t polyInfo = { LAC_HASH_POLY_DIGEST_SIZE, + LAC_HASH_POLY_BLOCK_SIZE, + NULL, /* intial state */ + LAC_HASH_POLY_STATE_SIZE }; -static lac_sym_qat_hash_alg_info_t aesCmacInfo = - { LAC_HASH_CMAC_128_DIGEST_SIZE, - LAC_HASH_CMAC_BLOCK_SIZE, - aesCmacKeySeed, - LAC_SYM_QAT_CMAC_STATE_SIZE }; +static lac_sym_qat_hash_alg_info_t xcbcMacInfo = { + LAC_HASH_XCBC_MAC_128_DIGEST_SIZE, + LAC_HASH_XCBC_MAC_BLOCK_SIZE, + aesXcbcKeySeed, + LAC_SYM_QAT_XCBC_STATE_SIZE +}; + +static lac_sym_qat_hash_alg_info_t aesCmacInfo = { + LAC_HASH_CMAC_128_DIGEST_SIZE, + LAC_HASH_CMAC_BLOCK_SIZE, + aesCmacKeySeed, + LAC_SYM_QAT_CMAC_STATE_SIZE +}; static lac_sym_qat_hash_alg_info_t aesCcmInfo = { LAC_HASH_AES_CCM_DIGEST_SIZE, @@ -259,11 +265,12 @@ static lac_sym_qat_hash_alg_info_t snow3gUia2Info = { 0 /* state size */ }; -static lac_sym_qat_hash_alg_info_t aesCbcMacInfo = - { LAC_HASH_AES_CBC_MAC_DIGEST_SIZE, - LAC_HASH_AES_CBC_MAC_BLOCK_SIZE, - NULL, - 0 }; +static lac_sym_qat_hash_alg_info_t aesCbcMacInfo = { + LAC_HASH_AES_CBC_MAC_DIGEST_SIZE, + LAC_HASH_AES_CBC_MAC_BLOCK_SIZE, + NULL, + 0 +}; static lac_sym_qat_hash_alg_info_t zucEia3Info = { LAC_HASH_ZUC_EIA3_DIGEST_SIZE, @@ -283,145 +290,154 @@ static lac_sym_qat_hash_qat_info_t sha1Config = { ICP_QAT_HW_AUTH_ALGO_SHA1, ICP_QAT_HW_SHA1_STATE1_SZ, ICP_QAT_HW_SHA1_STATE2_SZ }; -static lac_sym_qat_hash_qat_info_t sha224Config = - { ICP_QAT_HW_AUTH_ALGO_SHA224, - LAC_HASH_SHA224_BLOCK_SIZE, - ICP_QAT_HW_SHA224_STATE1_SZ, - ICP_QAT_HW_SHA224_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha224Config = { + ICP_QAT_HW_AUTH_ALGO_SHA224, + LAC_HASH_SHA224_BLOCK_SIZE, + ICP_QAT_HW_SHA224_STATE1_SZ, + ICP_QAT_HW_SHA224_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha256Config = - { ICP_QAT_HW_AUTH_ALGO_SHA256, - LAC_HASH_SHA256_BLOCK_SIZE, - ICP_QAT_HW_SHA256_STATE1_SZ, - ICP_QAT_HW_SHA256_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha256Config = { + ICP_QAT_HW_AUTH_ALGO_SHA256, + LAC_HASH_SHA256_BLOCK_SIZE, + ICP_QAT_HW_SHA256_STATE1_SZ, + ICP_QAT_HW_SHA256_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha384Config = - { ICP_QAT_HW_AUTH_ALGO_SHA384, - LAC_HASH_SHA384_BLOCK_SIZE, - ICP_QAT_HW_SHA384_STATE1_SZ, - ICP_QAT_HW_SHA384_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha384Config = { + ICP_QAT_HW_AUTH_ALGO_SHA384, + LAC_HASH_SHA384_BLOCK_SIZE, + ICP_QAT_HW_SHA384_STATE1_SZ, + ICP_QAT_HW_SHA384_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha512Config = - { ICP_QAT_HW_AUTH_ALGO_SHA512, - LAC_HASH_SHA512_BLOCK_SIZE, - ICP_QAT_HW_SHA512_STATE1_SZ, - ICP_QAT_HW_SHA512_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha512Config = { + ICP_QAT_HW_AUTH_ALGO_SHA512, + LAC_HASH_SHA512_BLOCK_SIZE, + ICP_QAT_HW_SHA512_STATE1_SZ, + ICP_QAT_HW_SHA512_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha3_224Config = - { ICP_QAT_HW_AUTH_ALGO_SHA3_224, - LAC_HASH_SHA3_224_BLOCK_SIZE, - ICP_QAT_HW_SHA3_224_STATE1_SZ, - ICP_QAT_HW_SHA3_224_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha3_224Config = { + ICP_QAT_HW_AUTH_ALGO_SHA3_224, + LAC_HASH_SHA3_224_BLOCK_SIZE, + ICP_QAT_HW_SHA3_224_STATE1_SZ, + ICP_QAT_HW_SHA3_224_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha3_256Config = - { ICP_QAT_HW_AUTH_ALGO_SHA3_256, - LAC_HASH_SHA3_256_BLOCK_SIZE, - ICP_QAT_HW_SHA3_256_STATE1_SZ, - ICP_QAT_HW_SHA3_256_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha3_256Config = { + ICP_QAT_HW_AUTH_ALGO_SHA3_256, + LAC_HASH_SHA3_256_BLOCK_SIZE, + ICP_QAT_HW_SHA3_256_STATE1_SZ, + ICP_QAT_HW_SHA3_256_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha3_384Config = - { ICP_QAT_HW_AUTH_ALGO_SHA3_384, - LAC_HASH_SHA3_384_BLOCK_SIZE, - ICP_QAT_HW_SHA3_384_STATE1_SZ, - ICP_QAT_HW_SHA3_384_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t sha3_384Config = { + ICP_QAT_HW_AUTH_ALGO_SHA3_384, + LAC_HASH_SHA3_384_BLOCK_SIZE, + ICP_QAT_HW_SHA3_384_STATE1_SZ, + ICP_QAT_HW_SHA3_384_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t sha3_512Config = - { ICP_QAT_HW_AUTH_ALGO_SHA3_512, - LAC_HASH_SHA3_512_BLOCK_SIZE, - ICP_QAT_HW_SHA3_512_STATE1_SZ, - ICP_QAT_HW_SHA3_512_STATE2_SZ }; - -static lac_sym_qat_hash_qat_info_t shake_128Config = - { ICP_QAT_HW_AUTH_ALGO_SHAKE_128, LAC_HASH_SHAKE_128_BLOCK_SIZE, 0, 0 }; - -static lac_sym_qat_hash_qat_info_t shake_256Config = - { ICP_QAT_HW_AUTH_ALGO_SHAKE_256, LAC_HASH_SHAKE_256_BLOCK_SIZE, 0, 0 }; - -static lac_sym_qat_hash_qat_info_t polyConfig = { ICP_QAT_HW_AUTH_ALGO_POLY, - LAC_HASH_POLY_BLOCK_SIZE, - 0, - 0 }; +static lac_sym_qat_hash_qat_info_t sha3_512Config = { + ICP_QAT_HW_AUTH_ALGO_SHA3_512, + LAC_HASH_SHA3_512_BLOCK_SIZE, + ICP_QAT_HW_SHA3_512_STATE1_SZ, + ICP_QAT_HW_SHA3_512_STATE2_SZ +}; static lac_sym_qat_hash_qat_info_t sm3Config = { ICP_QAT_HW_AUTH_ALGO_SM3, LAC_HASH_SM3_BLOCK_SIZE, ICP_QAT_HW_SM3_STATE1_SZ, ICP_QAT_HW_SM3_STATE2_SZ }; -static lac_sym_qat_hash_qat_info_t xcbcMacConfig = - { ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC, - 0, - ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ, - LAC_SYM_QAT_XCBC_STATE_SIZE }; +static lac_sym_qat_hash_qat_info_t polyConfig = { ICP_QAT_HW_AUTH_ALGO_POLY, + LAC_HASH_POLY_BLOCK_SIZE, + 0, + 0 }; -static lac_sym_qat_hash_qat_info_t aesCmacConfig = - { ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC, - 0, - ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ, - LAC_SYM_QAT_CMAC_STATE_SIZE }; +static lac_sym_qat_hash_qat_info_t xcbcMacConfig = { + ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC, + 0, + ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ, + LAC_SYM_QAT_XCBC_STATE_SIZE +}; -static lac_sym_qat_hash_qat_info_t aesCcmConfig = - { ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC, - 0, - ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, - ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ }; +static lac_sym_qat_hash_qat_info_t aesCmacConfig = { + ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC, + 0, + ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ, + LAC_SYM_QAT_CMAC_STATE_SIZE +}; -static lac_sym_qat_hash_qat_info_t aesGcmConfig = - { ICP_QAT_HW_AUTH_ALGO_GALOIS_128, - 0, - ICP_QAT_HW_GALOIS_128_STATE1_SZ, - ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ + - ICP_QAT_HW_GALOIS_E_CTR0_SZ }; +static lac_sym_qat_hash_qat_info_t aesCcmConfig = { + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC, + 0, + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, + ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ +}; -static lac_sym_qat_hash_qat_info_t kasumiF9Config = - { ICP_QAT_HW_AUTH_ALGO_KASUMI_F9, - 0, - ICP_QAT_HW_KASUMI_F9_STATE1_SZ, - ICP_QAT_HW_KASUMI_F9_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t aesGcmConfig = { + ICP_QAT_HW_AUTH_ALGO_GALOIS_128, + 0, + ICP_QAT_HW_GALOIS_128_STATE1_SZ, + ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ + + ICP_QAT_HW_GALOIS_E_CTR0_SZ +}; -static lac_sym_qat_hash_qat_info_t snow3gUia2Config = - { ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2, - 0, - ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ, - ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t kasumiF9Config = { + ICP_QAT_HW_AUTH_ALGO_KASUMI_F9, + 0, + ICP_QAT_HW_KASUMI_F9_STATE1_SZ, + ICP_QAT_HW_KASUMI_F9_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t aesCbcMacConfig = - { ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC, - 0, - ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, - ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ }; +static lac_sym_qat_hash_qat_info_t snow3gUia2Config = { + ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2, + 0, + ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ, + ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ +}; -static lac_sym_qat_hash_qat_info_t zucEia3Config = - { ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3, - 0, - ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ, - ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ }; +static lac_sym_qat_hash_qat_info_t aesCbcMacConfig = { + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC, + 0, + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ +}; + +static lac_sym_qat_hash_qat_info_t zucEia3Config = { + ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3, + 0, + ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ, + ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ +}; /* Array of mappings between algorithm and info structure * This array is used to populate the lookup table */ -static lac_sym_qat_hash_def_map_t lacHashDefsMapping[] = - { { CPA_CY_SYM_HASH_MD5, { &md5Info, &md5Config } }, - { CPA_CY_SYM_HASH_SHA1, { &sha1Info, &sha1Config } }, - { CPA_CY_SYM_HASH_SHA224, { &sha224Info, &sha224Config } }, - { CPA_CY_SYM_HASH_SHA256, { &sha256Info, &sha256Config } }, - { CPA_CY_SYM_HASH_SHA384, { &sha384Info, &sha384Config } }, - { CPA_CY_SYM_HASH_SHA512, { &sha512Info, &sha512Config } }, - { CPA_CY_SYM_HASH_SHA3_224, { &sha3_224Info, &sha3_224Config } }, - { CPA_CY_SYM_HASH_SHA3_256, { &sha3_256Info, &sha3_256Config } }, - { CPA_CY_SYM_HASH_SHA3_384, { &sha3_384Info, &sha3_384Config } }, - { CPA_CY_SYM_HASH_SHA3_512, { &sha3_512Info, &sha3_512Config } }, - { CPA_CY_SYM_HASH_SHAKE_128, { &shake_128Info, &shake_128Config } }, - { CPA_CY_SYM_HASH_SHAKE_256, { &shake_256Info, &shake_256Config } }, - { CPA_CY_SYM_HASH_POLY, { &polyInfo, &polyConfig } }, - { CPA_CY_SYM_HASH_SM3, { &sm3Info, &sm3Config } }, - { CPA_CY_SYM_HASH_AES_XCBC, { &xcbcMacInfo, &xcbcMacConfig } }, - { CPA_CY_SYM_HASH_AES_CMAC, { &aesCmacInfo, &aesCmacConfig } }, - { CPA_CY_SYM_HASH_AES_CCM, { &aesCcmInfo, &aesCcmConfig } }, - { CPA_CY_SYM_HASH_AES_GCM, { &aesGcmInfo, &aesGcmConfig } }, - { CPA_CY_SYM_HASH_KASUMI_F9, { &kasumiF9Info, &kasumiF9Config } }, - { CPA_CY_SYM_HASH_SNOW3G_UIA2, { &snow3gUia2Info, &snow3gUia2Config } }, - { CPA_CY_SYM_HASH_AES_GMAC, { &aesGcmInfo, &aesGcmConfig } }, - { CPA_CY_SYM_HASH_ZUC_EIA3, { &zucEia3Info, &zucEia3Config } }, - { CPA_CY_SYM_HASH_AES_CBC_MAC, { &aesCbcMacInfo, &aesCbcMacConfig } } }; +static lac_sym_qat_hash_def_map_t lacHashDefsMapping[] = { + { CPA_CY_SYM_HASH_MD5, { &md5Info, &md5Config } }, + { CPA_CY_SYM_HASH_SHA1, { &sha1Info, &sha1Config } }, + { CPA_CY_SYM_HASH_SHA224, { &sha224Info, &sha224Config } }, + { CPA_CY_SYM_HASH_SHA256, { &sha256Info, &sha256Config } }, + { CPA_CY_SYM_HASH_SHA384, { &sha384Info, &sha384Config } }, + { CPA_CY_SYM_HASH_SHA512, { &sha512Info, &sha512Config } }, + { CPA_CY_SYM_HASH_SHA3_224, { &sha3_224Info, &sha3_224Config } }, + { CPA_CY_SYM_HASH_SHA3_256, { &sha3_256Info, &sha3_256Config } }, + { CPA_CY_SYM_HASH_SHA3_384, { &sha3_384Info, &sha3_384Config } }, + { CPA_CY_SYM_HASH_SHA3_512, { &sha3_512Info, &sha3_512Config } }, + { CPA_CY_SYM_HASH_SM3, { &sm3Info, &sm3Config } }, + { CPA_CY_SYM_HASH_POLY, { &polyInfo, &polyConfig } }, + { CPA_CY_SYM_HASH_AES_XCBC, { &xcbcMacInfo, &xcbcMacConfig } }, + { CPA_CY_SYM_HASH_AES_CMAC, { &aesCmacInfo, &aesCmacConfig } }, + { CPA_CY_SYM_HASH_AES_CCM, { &aesCcmInfo, &aesCcmConfig } }, + { CPA_CY_SYM_HASH_AES_GCM, { &aesGcmInfo, &aesGcmConfig } }, + { CPA_CY_SYM_HASH_KASUMI_F9, { &kasumiF9Info, &kasumiF9Config } }, + { CPA_CY_SYM_HASH_SNOW3G_UIA2, { &snow3gUia2Info, &snow3gUia2Config } }, + { CPA_CY_SYM_HASH_AES_GMAC, { &aesGcmInfo, &aesGcmConfig } }, + { CPA_CY_SYM_HASH_ZUC_EIA3, { &zucEia3Info, &zucEia3Config } }, + { CPA_CY_SYM_HASH_AES_CBC_MAC, { &aesCbcMacInfo, &aesCbcMacConfig } } +}; /* * LacSymQat_HashLookupInit @@ -434,33 +450,35 @@ LacSymQat_HashLookupInit(CpaInstanceHandle instanceHandle) Cpa32U arraySize = 0; CpaStatus status = CPA_STATUS_SUCCESS; CpaCySymHashAlgorithm hashAlg = CPA_CY_SYM_HASH_NONE; - sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; + sal_service_t *pService = (sal_service_t *)instanceHandle; + lac_sym_qat_hash_defs_t **pLacHashLookupDefs; arraySize = (CPA_CY_HASH_ALG_END + 1) * sizeof(lac_sym_qat_hash_defs_t *); /* Size round up for performance */ arraySize = LAC_ALIGN_POW2_ROUNDUP(arraySize, LAC_64BYTE_ALIGNMENT); - pService->pLacHashLookupDefs = LAC_OS_MALLOC(arraySize); - - if (NULL != pService->pLacHashLookupDefs) { - LAC_OS_BZERO(pService->pLacHashLookupDefs, arraySize); - - numEntries = sizeof(lacHashDefsMapping) / - sizeof(lac_sym_qat_hash_def_map_t); - - /* initialise the hash lookup definitions table so that the - * algorithm - * can be used to index into the table */ - for (entry = 0; entry < numEntries; entry++) { - hashAlg = lacHashDefsMapping[entry].hashAlgorithm; - - pService->pLacHashLookupDefs[hashAlg] = - &(lacHashDefsMapping[entry].hashDefs); - } - } else { - status = CPA_STATUS_RESOURCE; + pLacHashLookupDefs = LAC_OS_MALLOC(arraySize); + if (NULL == pLacHashLookupDefs) { + return CPA_STATUS_RESOURCE; } + + LAC_OS_BZERO(pLacHashLookupDefs, arraySize); + numEntries = + sizeof(lacHashDefsMapping) / sizeof(lac_sym_qat_hash_def_map_t); + + /* initialise the hash lookup definitions table so that the algorithm + * can be used to index into the table */ + for (entry = 0; entry < numEntries; entry++) { + hashAlg = lacHashDefsMapping[entry].hashAlgorithm; + + pLacHashLookupDefs[hashAlg] = + &(lacHashDefsMapping[entry].hashDefs); + } + + ((sal_crypto_service_t *)pService)->pLacHashLookupDefs = + pLacHashLookupDefs; + return status; } @@ -472,9 +490,11 @@ LacSymQat_HashAlgLookupGet(CpaInstanceHandle instanceHandle, CpaCySymHashAlgorithm hashAlgorithm, lac_sym_qat_hash_alg_info_t **ppHashAlgInfo) { - sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; + sal_service_t *pService = (sal_service_t *)instanceHandle; + lac_sym_qat_hash_defs_t **pLacHashLookupDefs = + ((sal_crypto_service_t *)pService)->pLacHashLookupDefs; - *ppHashAlgInfo = pService->pLacHashLookupDefs[hashAlgorithm]->algInfo; + *ppHashAlgInfo = pLacHashLookupDefs[hashAlgorithm]->algInfo; } /* @@ -485,7 +505,9 @@ LacSymQat_HashDefsLookupGet(CpaInstanceHandle instanceHandle, CpaCySymHashAlgorithm hashAlgorithm, lac_sym_qat_hash_defs_t **ppHashDefsInfo) { - sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; + sal_service_t *pService = (sal_service_t *)instanceHandle; + lac_sym_qat_hash_defs_t **pLacHashLookupDefs = + ((sal_crypto_service_t *)pService)->pLacHashLookupDefs; - *ppHashDefsInfo = pService->pLacHashLookupDefs[hashAlgorithm]; + *ppHashDefsInfo = pLacHashLookupDefs[hashAlgorithm]; } diff --git a/sys/dev/qat/qat_api/common/ctrl/sal_compression.c b/sys/dev/qat/qat_api/common/ctrl/sal_compression.c index d8523d23b0e0..9827015767bf 100644 --- a/sys/dev/qat/qat_api/common/ctrl/sal_compression.c +++ b/sys/dev/qat/qat_api/common/ctrl/sal_compression.c @@ -47,6 +47,7 @@ #include "sal_service_state.h" #include "lac_buffer_desc.h" #include "icp_qat_fw_comp.h" +#include "icp_qat_hw_20_comp_defs.h" #include "icp_sal_versions.h" /* C string null terminator size */ @@ -124,6 +125,10 @@ static CpaStatus SalCtrl_CompressionInit_CompData(icp_accel_dev_t *device, sal_compression_service_t *pCompService) { + int level = 0; + + pCompService->comp_device_data.uniqueCompressionLevels[0] = CPA_FALSE; + switch (device->deviceType) { case DEVICE_DH895XCC: case DEVICE_DH895XCCVF: @@ -191,13 +196,36 @@ SalCtrl_CompressionInit_CompData(icp_accel_dev_t *device, ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF; pCompService->comp_device_data.inflateContextSize = DC_INFLATE_EH_CONTEXT_SIZE; + pCompService->comp_device_data.highestHwCompressionDepth = + ICP_QAT_HW_COMPRESSION_DEPTH_16; pCompService->comp_device_data.windowSizeMask = - (1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE); + (1 << DC_4K_WINDOW_SIZE | 1 << DC_8K_WINDOW_SIZE | + 1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE); pCompService->comp_device_data.minOutputBuffSize = DC_DEST_BUFFER_STA_MIN_SIZE; + pCompService->comp_device_data.minOutputBuffSizeDynamic = + pCompService->comp_device_data.minOutputBuffSize; pCompService->comp_device_data.enableDmm = ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED; pCompService->comp_device_data.cnvnrSupported = CPA_TRUE; + + for (level = CPA_DC_L1; level <= CPA_DC_L9; level++) { + switch (level) { + case CPA_DC_L1: + case CPA_DC_L2: + case CPA_DC_L3: + case CPA_DC_L4: + pCompService->comp_device_data + .uniqueCompressionLevels[level] = CPA_TRUE; + break; + default: + pCompService->comp_device_data + .uniqueCompressionLevels[level] = CPA_FALSE; + break; + } + } + pCompService->comp_device_data.numCompressionLevels = + DC_NUM_COMPRESSION_LEVELS; break; case DEVICE_C4XXX: case DEVICE_C4XXXVF: @@ -227,6 +255,45 @@ SalCtrl_CompressionInit_CompData(icp_accel_dev_t *device, (1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE); pCompService->comp_device_data.cnvnrSupported = CPA_TRUE; break; + case DEVICE_GEN4: + pCompService->generic_service_info.integrityCrcCheck = CPA_TRUE; + pCompService->numInterBuffs = 0; + pCompService->comp_device_data.minOutputBuffSize = + DC_DEST_BUFFER_STA_MIN_SIZE_GEN4; + pCompService->comp_device_data.minOutputBuffSizeDynamic = + DC_DEST_BUFFER_DYN_MIN_SIZE_GEN4; + pCompService->comp_device_data.oddByteDecompNobFinal = CPA_TRUE; + pCompService->comp_device_data.oddByteDecompInterim = CPA_FALSE; + pCompService->comp_device_data.translatorOverflow = CPA_TRUE; + pCompService->comp_device_data.useDevRam = + ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF; + pCompService->comp_device_data.enableDmm = + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED; + + pCompService->comp_device_data.inflateContextSize = + DC_INFLATE_CONTEXT_SIZE; + pCompService->comp_device_data.highestHwCompressionDepth = + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9; + pCompService->comp_device_data.windowSizeMask = + (1 << DC_4K_WINDOW_SIZE | 1 << DC_8K_WINDOW_SIZE | + 1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE); + for (level = CPA_DC_L1; level <= CPA_DC_L9; level++) { + switch (level) { + case CPA_DC_L1: + case CPA_DC_L6: + case CPA_DC_L9: + pCompService->comp_device_data + .uniqueCompressionLevels[level] = CPA_TRUE; + break; + default: + pCompService->comp_device_data + .uniqueCompressionLevels[level] = CPA_FALSE; + break; + } + } + pCompService->comp_device_data.numCompressionLevels = + DC_NUM_COMPRESSION_LEVELS; + break; default: QAT_UTILS_LOG("Unknown device type! - %d.\n", device->deviceType); diff --git a/sys/dev/qat/qat_api/common/ctrl/sal_create_services.c b/sys/dev/qat/qat_api/common/ctrl/sal_create_services.c index d51cc1bcace0..a512028ab9fe 100644 --- a/sys/dev/qat/qat_api/common/ctrl/sal_create_services.c +++ b/sys/dev/qat/qat_api/common/ctrl/sal_create_services.c @@ -64,6 +64,9 @@ SalCtrl_ServiceCreate(sal_service_type_t serviceType, pCrypto_service->generic_service_info.shutdown = SalCtrl_CryptoShutdown; + /* Force HW MAC validation for GCM and CCM */ + pCrypto_service->forceAEADMacVerify = CPA_TRUE; + *(ppInst) = &(pCrypto_service->generic_service_info); return CPA_STATUS_SUCCESS; diff --git a/sys/dev/qat/qat_api/common/ctrl/sal_crypto.c b/sys/dev/qat/qat_api/common/ctrl/sal_crypto.c index c97d36103c2b..78a559eba891 100644 --- a/sys/dev/qat/qat_api/common/ctrl/sal_crypto.c +++ b/sys/dev/qat/qat_api/common/ctrl/sal_crypto.c @@ -59,7 +59,10 @@ #include "lac_sym_qat.h" #include "icp_sal_versions.h" #include "icp_sal_user.h" +#include "sal_hw_gen.h" +#define HMAC_MODE_1 1 +#define HMAC_MODE_2 2 #define TH_CY_RX_0 0 #define TH_CY_RX_1 1 #define MAX_CY_RX_RINGS 2 @@ -211,7 +214,7 @@ SalCtrl_SymCreateTransHandle(icp_accel_dev_t *device, ICP_TRANS_TYPE_ETR, section, pCryptoService->acceleratorNum, - pCryptoService->bankNum, + pCryptoService->bankNumSym, temp_string, lac_getRingType(SAL_RING_TYPE_A_SYM_HI), NULL, @@ -235,7 +238,7 @@ SalCtrl_SymCreateTransHandle(icp_accel_dev_t *device, ICP_TRANS_TYPE_ETR, section, pCryptoService->acceleratorNum, - pCryptoService->bankNum, + pCryptoService->bankNumSym, temp_string, lac_getRingType(SAL_RING_TYPE_NONE), (icp_trans_callback)LacSymQat_SymRespHandler, @@ -328,6 +331,7 @@ static CpaStatus SalCtrl_SymInit(icp_accel_dev_t *device, sal_service_t *service) { CpaStatus status = CPA_STATUS_SUCCESS; + Cpa32U qatHmacMode = 0; Cpa32U numSymConcurrentReq = 0; char adfGetParam[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; char temp_string[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; @@ -344,6 +348,19 @@ SalCtrl_SymInit(icp_accel_dev_t *device, sal_service_t *service) * (Hash, Cipher, Algorithm-Chaining) (returns void)*/ LacSymCb_CallbacksRegister(); + qatHmacMode = (Cpa32U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC); + switch (qatHmacMode) { + case HMAC_MODE_1: + pCryptoService->qatHmacMode = ICP_QAT_HW_AUTH_MODE1; + break; + case HMAC_MODE_2: + pCryptoService->qatHmacMode = ICP_QAT_HW_AUTH_MODE2; + break; + default: + pCryptoService->qatHmacMode = ICP_QAT_HW_AUTH_MODE1; + break; + } + /* Get num concurrent requests from config file */ status = Sal_StringParsing("Cy", @@ -524,6 +541,32 @@ SalCtrl_DebugInit(icp_accel_dev_t *device, sal_service_t *service) return status; } +static CpaStatus +SalCtrl_GetBankNum(icp_accel_dev_t *device, + Cpa32U inst, + char *section, + char *bank_name, + Cpa16U *bank) +{ + char adfParamValue[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; + char adfParamName[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; + CpaStatus status = CPA_STATUS_SUCCESS; + + status = Sal_StringParsing("Cy", inst, bank_name, adfParamName); + LAC_CHECK_STATUS(status); + status = icp_adf_cfgGetParamValue(device, + section, + adfParamName, + adfParamValue); + if (CPA_STATUS_SUCCESS != status) { + QAT_UTILS_LOG("Failed to get %s from configuration file\n", + adfParamName); + return status; + } + *bank = (Cpa16U)Sal_Strtoul(adfParamValue, NULL, SAL_CFG_BASE_DEC); + return status; +} + static CpaStatus SalCtr_InstInit(icp_accel_dev_t *device, sal_service_t *service) { @@ -545,21 +588,62 @@ SalCtr_InstInit(icp_accel_dev_t *device, sal_service_t *service) pCryptoService->acceleratorNum = 0; - status = - Sal_StringParsing("Cy", - pCryptoService->generic_service_info.instance, - "BankNumber", - temp_string); - LAC_CHECK_STATUS(status); - status = - icp_adf_cfgGetParamValue(device, section, temp_string, adfGetParam); - if (CPA_STATUS_SUCCESS != status) { - QAT_UTILS_LOG("Failed to get %s from configuration file\n", - temp_string); - return status; + /* Gen4, a bank only has 2 rings (1 ring pair), only one type of service + can be assigned one time. asym and sym will be in different bank*/ + if (isCyGen4x(pCryptoService)) { + switch (service->type) { + case SAL_SERVICE_TYPE_CRYPTO_ASYM: + status = SalCtrl_GetBankNum( + device, + pCryptoService->generic_service_info.instance, + section, + "BankNumberAsym", + &pCryptoService->bankNumAsym); + if (CPA_STATUS_SUCCESS != status) + return status; + break; + case SAL_SERVICE_TYPE_CRYPTO_SYM: + status = SalCtrl_GetBankNum( + device, + pCryptoService->generic_service_info.instance, + section, + "BankNumberSym", + &pCryptoService->bankNumSym); + if (CPA_STATUS_SUCCESS != status) + return status; + break; + case SAL_SERVICE_TYPE_CRYPTO: + status = SalCtrl_GetBankNum( + device, + pCryptoService->generic_service_info.instance, + section, + "BankNumberAsym", + &pCryptoService->bankNumAsym); + if (CPA_STATUS_SUCCESS != status) + return status; + status = SalCtrl_GetBankNum( + device, + pCryptoService->generic_service_info.instance, + section, + "BankNumberSym", + &pCryptoService->bankNumSym); + if (CPA_STATUS_SUCCESS != status) + return status; + break; + default: + return CPA_STATUS_FAIL; + } + } else { + status = SalCtrl_GetBankNum( + device, + pCryptoService->generic_service_info.instance, + section, + "BankNumber", + &pCryptoService->bankNumSym); + if (CPA_STATUS_SUCCESS != status) + return status; + pCryptoService->bankNumAsym = pCryptoService->bankNumSym; } - pCryptoService->bankNum = - (Cpa16U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC); status = Sal_StringParsing("Cy", @@ -619,10 +703,18 @@ SalCtr_InstInit(icp_accel_dev_t *device, sal_service_t *service) "", temp_string2); LAC_CHECK_STATUS(status); - status = Sal_StringParsing("Bank", - pCryptoService->bankNum, - "CoreAffinity", - temp_string); + if (service->type == SAL_SERVICE_TYPE_CRYPTO_ASYM) + status = Sal_StringParsing("Bank", + pCryptoService->bankNumAsym, + "CoreAffinity", + temp_string); + else + /* For cy service, asym bank and sym bank will set the + same core affinity. So Just read one*/ + status = Sal_StringParsing("Bank", + pCryptoService->bankNumSym, + "CoreAffinity", + temp_string); LAC_CHECK_STATUS(status); } else { strncpy(temp_string2, section, (strlen(section) + 1)); @@ -817,6 +909,9 @@ cpaCyGetStatusText(const CpaInstanceHandle instanceHandle, case CPA_STATUS_FATAL: LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_FATAL); break; + case CPA_STATUS_UNSUPPORTED: + LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_UNSUPPORTED); + break; default: status = CPA_STATUS_INVALID_PARAM; break; @@ -886,6 +981,10 @@ cpaCyStartInstance(CpaInstanceHandle instanceHandle_in) instanceHandle = instanceHandle_in; } LAC_CHECK_NULL_PARAM(instanceHandle); + SAL_CHECK_INSTANCE_TYPE(instanceHandle, + (SAL_SERVICE_TYPE_CRYPTO | + SAL_SERVICE_TYPE_CRYPTO_ASYM | + SAL_SERVICE_TYPE_CRYPTO_SYM)); pService = (sal_crypto_service_t *)instanceHandle; @@ -930,6 +1029,10 @@ cpaCyStopInstance(CpaInstanceHandle instanceHandle_in) instanceHandle = instanceHandle_in; } LAC_CHECK_NULL_PARAM(instanceHandle); + SAL_CHECK_INSTANCE_TYPE(instanceHandle, + (SAL_SERVICE_TYPE_CRYPTO | + SAL_SERVICE_TYPE_CRYPTO_ASYM | + SAL_SERVICE_TYPE_CRYPTO_SYM)); status = cpaCyInstanceGetInfo2(instanceHandle, &info); if (CPA_STATUS_SUCCESS != status) { @@ -1431,23 +1534,31 @@ cpaCySymQueryCapabilities(const CpaInstanceHandle instanceHandle_in, } CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_NULL); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_ARC4); CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_ECB); CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_CBC); CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_CTR); CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_CCM); CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_GCM); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_DES_ECB); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_DES_CBC); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_3DES_ECB); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_3DES_CBC); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_3DES_CTR); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_KASUMI_F8); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_SNOW3G_UEA2); - CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_F8); CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_XTS); + if (isCyGen2x(pCryptoService)) { + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_ARC4); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_DES_ECB); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_DES_CBC); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_3DES_ECB); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_3DES_CBC); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_3DES_CTR); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_KASUMI_F8); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, + CPA_CY_SYM_CIPHER_SNOW3G_UEA2); + CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_F8); + } - CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_MD5); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SHA1); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SHA224); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SHA256); @@ -1456,11 +1567,15 @@ cpaCySymQueryCapabilities(const CpaInstanceHandle instanceHandle_in, CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_XCBC); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_CCM); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_GCM); - CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_KASUMI_F9); - CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SNOW3G_UIA2); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_CMAC); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_GMAC); CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_CBC_MAC); + if (isCyGen2x(pCryptoService)) { + CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_MD5); + CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_KASUMI_F9); + CPA_BITMAP_BIT_SET(pCapInfo->hashes, + CPA_CY_SYM_HASH_SNOW3G_UIA2); + } if (pGenericService->capabilitiesMask & ICP_ACCEL_CAPABILITIES_CRYPTO_ZUC) { @@ -1660,18 +1775,33 @@ static CpaInstanceHandle Lac_GetFirstAsymHandle(icp_accel_dev_t *adfInsts[ADF_MAX_DEVICES], Cpa16U num_dev) { + CpaStatus status = CPA_STATUS_SUCCESS; icp_accel_dev_t *dev_addr = NULL; sal_t *base_addr = NULL; sal_list_t *list_temp = NULL; CpaInstanceHandle cyInst = NULL; + CpaInstanceInfo2 info; Cpa16U i = 0; for (i = 0; i < num_dev; i++) { dev_addr = (icp_accel_dev_t *)adfInsts[i]; base_addr = dev_addr->pSalHandle; - if ((NULL != base_addr) && (NULL != base_addr->asym_services)) { - list_temp = base_addr->asym_services; + if (NULL == base_addr) { + continue; + } + list_temp = base_addr->asym_services; + while (NULL != list_temp) { cyInst = SalList_getObject(list_temp); + status = cpaCyInstanceGetInfo2(cyInst, &info); + list_temp = SalList_next(list_temp); + if (CPA_STATUS_SUCCESS != status || + CPA_TRUE != info.isPolled) { + cyInst = NULL; + continue; + } + break; + } + if (cyInst) { break; } } @@ -1684,18 +1814,33 @@ static CpaInstanceHandle Lac_GetFirstSymHandle(icp_accel_dev_t *adfInsts[ADF_MAX_DEVICES], Cpa16U num_dev) { + CpaStatus status = CPA_STATUS_SUCCESS; icp_accel_dev_t *dev_addr = NULL; sal_t *base_addr = NULL; sal_list_t *list_temp = NULL; CpaInstanceHandle cyInst = NULL; + CpaInstanceInfo2 info; Cpa16U i = 0; for (i = 0; i < num_dev; i++) { dev_addr = (icp_accel_dev_t *)adfInsts[i]; base_addr = dev_addr->pSalHandle; - if ((NULL != base_addr) && (NULL != base_addr->sym_services)) { - list_temp = base_addr->sym_services; + if (NULL == base_addr) { + continue; + } + list_temp = base_addr->sym_services; + while (NULL != list_temp) { cyInst = SalList_getObject(list_temp); + status = cpaCyInstanceGetInfo2(cyInst, &info); + list_temp = SalList_next(list_temp); + if (CPA_STATUS_SUCCESS != status || + CPA_TRUE != info.isPolled) { + cyInst = NULL; + continue; + } + break; + } + if (cyInst) { break; } } @@ -1709,22 +1854,37 @@ Lac_GetFirstSymHandle(icp_accel_dev_t *adfInsts[ADF_MAX_DEVICES], static CpaInstanceHandle Lac_GetFirstCyHandle(icp_accel_dev_t *adfInsts[ADF_MAX_DEVICES], Cpa16U num_dev) { + CpaStatus status = CPA_STATUS_SUCCESS; icp_accel_dev_t *dev_addr = NULL; sal_t *base_addr = NULL; sal_list_t *list_temp = NULL; CpaInstanceHandle cyInst = NULL; + CpaInstanceInfo2 info; Cpa16U i = 0; for (i = 0; i < num_dev; i++) { dev_addr = (icp_accel_dev_t *)adfInsts[i]; base_addr = dev_addr->pSalHandle; - if ((NULL != base_addr) && - (NULL != base_addr->crypto_services)) { - list_temp = base_addr->crypto_services; + if (NULL == base_addr) { + continue; + } + list_temp = base_addr->crypto_services; + while (NULL != list_temp) { cyInst = SalList_getObject(list_temp); + status = cpaCyInstanceGetInfo2(cyInst, &info); + list_temp = SalList_next(list_temp); + if (CPA_STATUS_SUCCESS != status || + CPA_TRUE != info.isPolled) { + cyInst = NULL; + continue; + } + break; + } + if (cyInst) { break; } } + return cyInst; } @@ -1835,3 +1995,16 @@ icp_sal_dp_SymGetInflightRequests(CpaInstanceHandle instanceHandle, numInflightRequests); } + +CpaStatus +icp_sal_setForceAEADMACVerify(CpaInstanceHandle instanceHandle, + CpaBoolean forceAEADMacVerify) +{ + sal_crypto_service_t *crypto_handle = NULL; + + crypto_handle = (sal_crypto_service_t *)instanceHandle; + LAC_CHECK_NULL_PARAM(crypto_handle); + crypto_handle->forceAEADMacVerify = forceAEADMacVerify; + + return CPA_STATUS_SUCCESS; +} diff --git a/sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c b/sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c index 4a41340dfabb..10ce54c0ce43 100644 --- a/sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c +++ b/sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c @@ -458,6 +458,37 @@ SalCtrl_ServiceShutdown(icp_accel_dev_t *device, return status; } +static CpaStatus +selectGeneration(device_type_t deviceType, sal_service_t *pInst) +{ + switch (deviceType) { + case DEVICE_C62X: + case DEVICE_C62XVF: + case DEVICE_DH895XCC: + case DEVICE_DH895XCCVF: + case DEVICE_C3XXX: + case DEVICE_C3XXXVF: + case DEVICE_200XX: + case DEVICE_200XXVF: + pInst->gen = GEN2; + break; + + case DEVICE_C4XXX: + case DEVICE_C4XXXVF: + pInst->gen = GEN3; + break; + + case DEVICE_GEN4: + pInst->gen = GEN4; + break; + + default: + QAT_UTILS_LOG("deviceType not initialised\n"); + return CPA_STATUS_FAIL; + } + return CPA_STATUS_SUCCESS; +} + /************************************************************************* * @ingroup SalCtrl * @description @@ -523,7 +554,12 @@ SalCtrl_ServiceInit(icp_accel_dev_t *device, } pInst->debug_parent_dir = debug_dir; pInst->capabilitiesMask = device->accelCapabilitiesMask; - status = SalList_add(services, &tail_list, pInst); + + status = selectGeneration(device->deviceType, pInst); + if (CPA_STATUS_SUCCESS == status) { + status = + SalList_add(services, &tail_list, pInst); + } if (CPA_STATUS_SUCCESS != status) { free(pInst, M_QAT); } diff --git a/sys/dev/qat/qat_api/common/include/lac_common.h b/sys/dev/qat/qat_api/common/include/lac_common.h index fb2fdd95300e..dacf43c3a072 100644 --- a/sys/dev/qat/qat_api/common/include/lac_common.h +++ b/sys/dev/qat/qat_api/common/include/lac_common.h @@ -756,17 +756,14 @@ typedef struct mtx *lac_lock_t; #define LAC_SPINLOCK(lock) \ ({ \ (void)qatUtilsLock(lock); \ - CPA_STATUS_SUCCESS; \ }) #define LAC_SPINUNLOCK(lock) \ ({ \ (void)qatUtilsUnlock(lock); \ - CPA_STATUS_SUCCESS; \ }) #define LAC_SPINLOCK_DESTROY(lock) \ ({ \ (void)qatUtilsLockDestroy(lock); \ - CPA_STATUS_SUCCESS; \ }) #define LAC_CONST_PTR_CAST(castee) ((void *)(LAC_ARCH_UINT)(castee)) diff --git a/sys/dev/qat/qat_api/common/include/lac_sal.h b/sys/dev/qat/qat_api/common/include/lac_sal.h index 031cb4204084..69d47628496e 100644 --- a/sys/dev/qat/qat_api/common/include/lac_sal.h +++ b/sys/dev/qat/qat_api/common/include/lac_sal.h @@ -54,8 +54,7 @@ CpaStatus SalCtrl_ServiceCreate(sal_service_type_t service, Cpa32U instance_num, sal_service_t **pObj); -/** -******************************************************************************* +/****************************************************************************** * @ingroup SalCtl * @description * This macro goes through the 'list' passed in as a parameter. For each diff --git a/sys/dev/qat/qat_api/common/include/lac_sal_types.h b/sys/dev/qat/qat_api/common/include/lac_sal_types.h index 36e146665787..eb2edf586438 100644 --- a/sys/dev/qat/qat_api/common/include/lac_sal_types.h +++ b/sys/dev/qat/qat_api/common/include/lac_sal_types.h @@ -67,6 +67,17 @@ typedef enum { SAL_SERVICE_TYPE_QAT = 32 } sal_service_type_t; +/** + ***************************************************************************** + * @ingroup SalCtrl + * Device generations + * + * @description + * List in an enum all the QAT device generations. + * + *****************************************************************************/ +typedef enum { GEN2, GEN3, GEN4 } sal_generation_t; + /** ***************************************************************************** * @ingroup SalCtrl @@ -128,6 +139,9 @@ typedef struct sal_service_s { CpaBoolean integrityCrcCheck; /** < True if the device supports end to end data integrity checks */ + + sal_generation_t gen; + /** Generation of devices */ } sal_service_t; /** diff --git a/sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h b/sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h index c7db231260ed..68213278672e 100644 --- a/sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h +++ b/sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h @@ -16,6 +16,7 @@ #define LAC_SAL_TYPES_CRYPTO_H_ #include "lac_sym_qat_hash_defs_lookup.h" +#include "lac_sym_qat_constants_table.h" #include "lac_sym_key.h" #include "cpa_cy_sym_dp.h" @@ -84,6 +85,8 @@ typedef struct sal_crypto_service_s { QatUtilsAtomic *pLacDrbgStatsArr; /**< pointer to an array of atomic stats for DRBG */ + icp_qat_hw_auth_mode_t qatHmacMode; + /**< Hmac Mode */ Cpa32U pkeFlowId; /**< Flow ID for all pke requests from this instance - identifies @@ -105,6 +108,8 @@ typedef struct sal_crypto_service_s { Cpa16U acceleratorNum; Cpa16U bankNum; + Cpa16U bankNumAsym; + Cpa16U bankNumSym; Cpa16U pkgID; Cpa8U isPolled; Cpa8U executionEngine; @@ -119,6 +124,9 @@ typedef struct sal_crypto_service_s { /**< table of pointers to standard defined information for all hash algorithms. We support an extra hash algo that is not exported by cy api which is why we need the extra +1 */ + + lac_sym_qat_constants_t constantsLookupTables; + Cpa8U **ppHmacContentDesc; /**< table of pointers to CD for Hmac precomputes - used at session init */ @@ -137,6 +145,10 @@ typedef struct sal_crypto_service_s { debug_file_info_t *debug_file; /**< Statistics handler */ + + CpaBoolean forceAEADMacVerify; + /**< internal flag to enable/disable forcing HW digest verification for + GCM and CCM algorithms */ } sal_crypto_service_t; /************************************************************************* diff --git a/sys/dev/qat/qat_api/common/include/lac_sync.h b/sys/dev/qat/qat_api/common/include/lac_sync.h index b842cce30d87..7052c656396a 100644 --- a/sys/dev/qat/qat_api/common/include/lac_sync.h +++ b/sys/dev/qat/qat_api/common/include/lac_sync.h @@ -61,7 +61,7 @@ typedef struct lac_sync_op_data_s { * Timeout for wait for init messages response in msecs */ -#define DC_SYNC_CALLBACK_TIMEOUT (1000) +#define DC_SYNC_CALLBACK_TIMEOUT (2000) /**< @ingroup LacSyn * Timeout for wait for compression response in msecs */ diff --git a/sys/dev/qat/qat_api/common/include/sal_hw_gen.h b/sys/dev/qat/qat_api/common/include/sal_hw_gen.h new file mode 100644 index 000000000000..38deb3cbc013 --- /dev/null +++ b/sys/dev/qat/qat_api/common/include/sal_hw_gen.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2022 Intel Corporation */ +/* $FreeBSD$ */ +/** + *************************************************************************** + * @file sal_hw_gen.h + * + * @ingroup SalHwGen + * + * @description + * Functions which return a value corresponding to qat device generation + * + ***************************************************************************/ + +#ifndef SAL_HW_GEN_H +#define SAL_HW_GEN_H + +#include "cpa.h" +#include "sal_types_compression.h" +#include "lac_sal_types_crypto.h" + +/** + *************************************************************************** + * @ingroup SalHwGen + * + * @description This function returns whether qat device is gen 4 or not + * + * @param[in] pService pointer to compression service + * + ***************************************************************************/ + +static inline CpaBoolean +isDcGen4x(const sal_compression_service_t *pService) +{ + return (pService->generic_service_info.gen == GEN4); +} + +/** + *************************************************************************** + * @ingroup SalHwGen + * + * @description This function returns whether qat device is gen 2/3 or not + * + * @param[in] pService pointer to compression service + * + ***************************************************************************/ + +static inline CpaBoolean +isDcGen2x(const sal_compression_service_t *pService) +{ + return ((pService->generic_service_info.gen == GEN2) || + (pService->generic_service_info.gen == GEN3)); +} + +/** + *************************************************************************** + * @ingroup SalHwGen + * + * @description This function returns whether qat device is gen 4 or not + * + * @param[in] pService pointer to crypto service + * + ***************************************************************************/ + +static inline CpaBoolean +isCyGen4x(const sal_crypto_service_t *pService) +{ + return (pService->generic_service_info.gen == GEN4); +} + +/** + *************************************************************************** + * @ingroup SalHwGen + * + * @description This function returns whether qat device is gen 2/3 or not + * + * @param[in] pService pointer to crypto service + * + ***************************************************************************/ + +static inline CpaBoolean +isCyGen2x(const sal_crypto_service_t *pService) +{ + return ((pService->generic_service_info.gen == GEN2) || + (pService->generic_service_info.gen == GEN3)); +} + +#endif /* SAL_HW_GEN_H */ diff --git a/sys/dev/qat/qat_api/common/include/sal_types_compression.h b/sys/dev/qat/qat_api/common/include/sal_types_compression.h index 12f9f673ac43..80e3c89ea699 100644 --- a/sys/dev/qat/qat_api/common/include/sal_types_compression.h +++ b/sys/dev/qat/qat_api/common/include/sal_types_compression.h @@ -23,6 +23,7 @@ #include "icp_adf_transport.h" #define DC_NUM_RX_RINGS (1) +#define DC_NUM_COMPRESSION_LEVELS (CPA_DC_L9) /** ***************************************************************************** @@ -37,6 +38,9 @@ typedef struct sal_compression_device_data { /* Device specific minimum output buffer size for static compression */ Cpa32U minOutputBuffSize; + /* Device specific minimum output buffer size for dynamic compression */ + Cpa32U minOutputBuffSizeDynamic; + /* Enable/disable secureRam/acceleratorRam for intermediate buffers*/ Cpa8U useDevRam; @@ -62,6 +66,11 @@ typedef struct sal_compression_device_data { /* Mask that reports supported window sizes for comp/decomp */ Cpa8U windowSizeMask; + /* List representing compression levels that are the first to have + a unique search depth. */ + CpaBoolean uniqueCompressionLevels[DC_NUM_COMPRESSION_LEVELS + 1]; + Cpa8U numCompressionLevels; + /* Flag to indicate CompressAndVerifyAndRecover feature support */ CpaBoolean cnvnrSupported; } sal_compression_device_data_t; diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h index 7fe1bc3d1056..1280f3c1c8a9 100644 --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h @@ -486,12 +486,12 @@ typedef struct icp_qat_fw_comn_resp_s { /* ========================================================================= */ /* Common QAT FW request header - structure of LW0 - * + ===== + ---- + ----------- + ----------- + ----------- + ----------- + - * | Bit | 31 | 30 - 24 | 21 - 16 | 15 - 8 | 7 - 0 | - * + ===== + ---- + ----------- + ----------- + ----------- + ----------- + - * | Flags | V | Reserved | Serv Type | Serv Cmd Id | Reserved | - * + ===== + ---- + ----------- + ----------- + ----------- + ----------- + -*/ + * + ===== + ------- + ----------- + ----------- + ----------- + -------- + + * | Bit | 31/30 | 29 - 24 | 21 - 16 | 15 - 8 | 7 - 0 | + * + ===== + ------- + ----------- + ----------- + ----------- + -------- + + * | Flags | V/Gen | Reserved | Serv Type | Serv Cmd Id | Rsv | + * + ===== + ------- + ----------- + ----------- + ----------- + -------- + + */ /**< @ingroup icp_qat_fw_comn * Definition of the setting of the header's valid flag */ @@ -505,6 +505,20 @@ typedef struct icp_qat_fw_comn_resp_s { * hdr_flags field of LW0 (service request and response) */ #define ICP_QAT_FW_COMN_VALID_FLAG_BITPOS 7 #define ICP_QAT_FW_COMN_VALID_FLAG_MASK 0x1 + +/**< @ingroup icp_qat_fw_comn + * Macros defining the bit position and mask of the 'generation' flag, within + * the hdr_flags field of LW0 (service request and response) */ +#define ICP_QAT_FW_COMN_GEN_FLAG_BITPOS 6 +#define ICP_QAT_FW_COMN_GEN_FLAG_MASK 0x1 +/**< @ingroup icp_qat_fw_comn + * The request is targeted for QAT2.0 */ +#define ICP_QAT_FW_COMN_GEN_2 1 +/**< @ingroup icp_qat_fw_comn +* The request is targeted for QAT1.x. QAT2.0 FW will return + 'unsupported request' if GEN1 request type is sent to QAT2.0 FW */ +#define ICP_QAT_FW_COMN_GEN_1 0 + #define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK 0x7F /* Common QAT FW response header - structure of LW0 @@ -525,6 +539,13 @@ typedef struct icp_qat_fw_comn_resp_s { #define ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS 5 #define ICP_QAT_FW_COMN_CNVNR_FLAG_MASK 0x1 +/**< @ingroup icp_qat_fw_comn + * Macros defining the bit position and mask of Stored Blocks flag + * within the hdr_flags field of LW0 (service response only) + */ +#define ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS 4 +#define ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK 0x1 + /** ****************************************************************************** * @ingroup icp_qat_fw_comn @@ -660,6 +681,89 @@ typedef struct icp_qat_fw_comn_resp_s { ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ ICP_QAT_FW_COMN_VALID_FLAG_MASK) +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comn + * + * @description + * Extract the Stored Block flag from the header flags in the + * response only. + * + * @param hdr_flags Response 'hdr' structure to extract the + * Stored Block bit from the 'hdr_flags' field. + * + *****************************************************************************/ +#define ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comn + * + * @description + * Set the Stored Block bit in the response's header flags. + * + * @param hdr_t Response 'hdr_t' structure to set the ST_BLK bit + * @param val Value of the ST_BLK bit flag. + * + *****************************************************************************/ +#define ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), \ + (val), \ + ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comn + * + * @description + * Set the generation bit in the request's header flags. + * + * @param hdr_t Request or Response 'hdr_t' structure to set the gen bit + * @param val Value of the generation bit flag. + * + *****************************************************************************/ +#define ICP_QAT_FW_COMN_HDR_GENERATION_FLAG_SET(hdr_t, val) \ + ICP_QAT_FW_COMN_GENERATION_FLAG_SET(hdr_t, val) + +/** +****************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Common macro to set the generation bit in the common header +* +* @param hdr_t Structure (request or response) containing the header +* flags field, to allow the generation bit to be set. +* @param val Value of the generation bit flag. +* +*****************************************************************************/ +#define ICP_QAT_FW_COMN_GENERATION_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), \ + (val), \ + ICP_QAT_FW_COMN_GEN_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_GEN_FLAG_MASK) + +/** +****************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Common macro to extract the generation flag from the header flags field +* within the header structure (request or response). +* +* @param hdr_t Structure (request or response) to extract the +* generation bit from the 'hdr_flags' field. +* +*****************************************************************************/ + +#define ICP_QAT_FW_COMN_HDR_GENERATION_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_GEN_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_GEN_FLAG_MASK) /** ****************************************************************************** * @ingroup icp_qat_fw_comn diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h index 1a9d7e727bd0..c4bb5632b318 100644 --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h @@ -61,6 +61,8 @@ typedef enum { * | | |as intmd| | | | | | | | * | | | buf | | | | | | | | * + ===== + ------ + ----- + --- + ------ + ----- + ----- + -- + ---- + --- + + * Note: For QAT 2.0 Disable Secure Ram, DisType0 Header and Enhanced ASB bits + * are don't care. i.e., these features are removed from QAT 2.0. */ /** Flag usage */ @@ -183,6 +185,28 @@ typedef enum { ((secure_ram & ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) \ << ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS)) +/** +****************************************************************************** +* @ingroup icp_qat_fw_comp +* +* @description +* Macro used for the generation of the command flags for Compression Request. +* This should always be used for the generation of the flags. No direct sets or +* masks should be performed on the flags data +* +* @param sesstype Session Type +* @param autoselect AutoSelectBest +* Selects between compressed and uncompressed output. +* No distinction made between static and dynamic +* compressed data. +* +*********************************************************************************/ +#define ICP_QAT_FW_COMP_20_FLAGS_BUILD(sesstype, autoselect) \ + (((sesstype & ICP_QAT_FW_COMP_SESSION_TYPE_MASK) \ + << ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS) | \ + ((autoselect & ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK) \ + << ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS)) + /** ****************************************************************************** * @ingroup icp_qat_fw_comp @@ -375,14 +399,16 @@ typedef struct icp_qat_fw_comp_req_params_s { * *****************************************************************************/ #define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( \ - sop, eop, bfinal, cnv, cnvnr, crc) \ + sop, eop, bfinal, cnv, cnvnr, cnvdfx, crc) \ (((sop & ICP_QAT_FW_COMP_SOP_MASK) << ICP_QAT_FW_COMP_SOP_BITPOS) | \ ((eop & ICP_QAT_FW_COMP_EOP_MASK) << ICP_QAT_FW_COMP_EOP_BITPOS) | \ ((bfinal & ICP_QAT_FW_COMP_BFINAL_MASK) \ << ICP_QAT_FW_COMP_BFINAL_BITPOS) | \ ((cnv & ICP_QAT_FW_COMP_CNV_MASK) << ICP_QAT_FW_COMP_CNV_BITPOS) | \ - ((cnvnr & ICP_QAT_FW_COMP_CNV_RECOVERY_MASK) \ - << ICP_QAT_FW_COMP_CNV_RECOVERY_BITPOS) | \ + ((cnvnr & ICP_QAT_FW_COMP_CNVNR_MASK) \ + << ICP_QAT_FW_COMP_CNVNR_BITPOS) | \ + ((cnvdfx & ICP_QAT_FW_COMP_CNV_DFX_MASK) \ + << ICP_QAT_FW_COMP_CNV_DFX_BITPOS) | \ ((crc & ICP_QAT_FW_COMP_CRC_MODE_MASK) \ << ICP_QAT_FW_COMP_CRC_MODE_BITPOS)) @@ -443,6 +469,14 @@ typedef struct icp_qat_fw_comp_req_params_s { /**< @ingroup icp_qat_fw_comp * Flag indicating that a cnv recovery is to be performed on the request */ +#define ICP_QAT_FW_COMP_NO_CNV_DFX 0 +/**< @ingroup icp_qat_fw_comp + * Flag indicating that NO CNV inject error is to be performed on the request */ + +#define ICP_QAT_FW_COMP_CNV_DFX 1 +/**< @ingroup icp_qat_fw_comp + * Flag indicating that CNV inject error is to be performed on the request */ + #define ICP_QAT_FW_COMP_CRC_MODE_LEGACY 0 /**< @ingroup icp_qat_fw_comp * Flag representing to use the legacy CRC mode */ @@ -491,6 +525,22 @@ typedef struct icp_qat_fw_comp_req_params_s { /**< @ingroup icp_qat_fw_comp * Starting bit position for the CNV Recovery bit */ +#define ICP_QAT_FW_COMP_CNVNR_MASK 0x1 +/**< @ingroup icp_qat_fw_comp + * One bit mask for the CNV Recovery bit */ + +#define ICP_QAT_FW_COMP_CNVNR_BITPOS 17 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for the CNV Recovery bit */ + +#define ICP_QAT_FW_COMP_CNV_DFX_BITPOS 18 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for the CNV DFX bit */ + +#define ICP_QAT_FW_COMP_CNV_DFX_MASK 0x1 +/**< @ingroup icp_qat_fw_comp + * One bit mask for the CNV DFX bit */ + #define ICP_QAT_FW_COMP_CRC_MODE_BITPOS 19 /**< @ingroup icp_qat_fw_comp * Starting bit position for CRC mode */ @@ -499,6 +549,14 @@ typedef struct icp_qat_fw_comp_req_params_s { /**< @ingroup icp_qat_fw_comp * One bit mask used to determine CRC mode */ +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS 20 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for xxHash accumulate mode */ + +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK 0x1 +/**< @ingroup icp_qat_fw_comp + * One bit mask used to determine xxHash accumulate mode */ + /** ****************************************************************************** * @ingroup icp_qat_fw_comp @@ -574,6 +632,38 @@ typedef struct icp_qat_fw_comp_req_params_s { ICP_QAT_FW_COMP_CRC_MODE_BITPOS, \ ICP_QAT_FW_COMP_CRC_MODE_MASK) +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comp + * + * @description + * Macro for extraction of the xxHash accumulate mode bit + * + * @param flags Flags to extract the xxHash accumulate mode bit from + * + *****************************************************************************/ +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_GET(flags) \ + QAT_FIELD_GET(flags, \ + ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \ + ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comp + * + * @description + * Macro for setting of the xxHash accumulate mode bit + * + * @param flags Flags to set the xxHash accumulate mode bit to + * @param val xxHash accumulate mode to set + * + *****************************************************************************/ +#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \ + ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK) + /** ****************************************************************************** * @ingroup icp_qat_fw_comp @@ -589,10 +679,11 @@ typedef struct icp_qat_fw_xlt_req_params_s { /**< LWs 20-21 */ uint64_t inter_buff_ptr; /**< This field specifies the physical address of an intermediate - * buffer SGL array. The array contains a pair of 64-bit - * intermediate buffer pointers to SGL buffer descriptors, one pair - * per CPM. Please refer to the CPM1.6 Firmware Interface HLD - * specification for more details. */ + * buffer SGL array. The array contains a pair of 64-bit + * intermediate buffer pointers to SGL buffer descriptors, one pair + * per CPM. Please refer to the CPM1.6 Firmware Interface HLD + * specification for more details. + * Placeholder for QAT2.0. */ } icp_qat_fw_xlt_req_params_t; /** @@ -1026,4 +1117,31 @@ typedef enum { (((bank_a_enable)&QAT_FW_COMP_BANK_FLAG_MASK) \ << QAT_FW_COMP_BANK_A_BITPOS)) +/** + ***************************************************************************** + * @ingroup icp_qat_fw_comp + * Definition of the xxhash32 acc state buffer + * @description + * This is data structure used in stateful lite for xxhash32 + * + *****************************************************************************/ +typedef struct xxhash_acc_state_buff_s { + /**< LW 0 */ + uint32_t in_counter; + /**< Accumulated (total) consumed bytes. As oppose to the per request + * IBC in the response.*/ + + /**< LW 1 */ + uint32_t out_counter; + /**< OBC as in the response.*/ + + /**< LW 2-5 */ + uint32_t xxhash_state[4]; + /**< Initial value is set by IA to the values stated in HAS.*/ + + /**< LW 6-9 */ + uint32_t clear_txt[4]; + /**< Set to 0 for the first request.*/ +} xxhash_acc_state_buff_t; + #endif /* _ICP_QAT_FW_COMP_H_ */ diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h index 447b1a848809..8a59b314c74b 100644 --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h @@ -92,6 +92,44 @@ typedef enum { /**< Delimiter type */ } icp_qat_fw_la_cmd_id_t; +typedef struct icp_qat_fw_la_cipher_20_req_params_s { + /**< LW 14 */ + uint32_t cipher_offset; + /**< Cipher offset long word. */ + + /**< LW 15 */ + uint32_t cipher_length; + /**< Cipher length long word. */ + + /**< LWs 16-19 */ + union { + uint32_t cipher_IV_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + /**< Cipher IV array */ + + struct { + uint64_t cipher_IV_ptr; + /**< Cipher IV pointer or Partial State Pointer */ + + uint64_t resrvd1; + /**< reserved */ + + } s; + + } u; + /**< LW 20 */ + uint32_t spc_aad_offset; + /**< LW 21 */ + uint32_t spc_aad_sz; + /**< LW 22 - 23 */ + uint64_t spc_aad_addr; + /**< LW 24 - 25 */ + uint64_t spc_auth_res_addr; + /**< LW 26 */ + uint8_t reserved[3]; + uint8_t spc_auth_res_sz; + +} icp_qat_fw_la_cipher_20_req_params_t; + /* For the definitions of the bits in the status field of the common * response, refer to icp_qat_fw.h. * The return values specific to Lookaside service are given below. @@ -165,6 +203,34 @@ typedef struct icp_qat_fw_la_bulk_req_s { /* Private defines */ +/* bits 15:14 */ +#define ICP_QAT_FW_LA_USE_WIRELESS_SLICE_TYPE 2 +/**< @ingroup icp_qat_fw_la + * FW Selects Wireless Cipher Slice + * Cipher Algorithms: AES-{F8}, Snow3G, ZUC + * Auth Algorithms : Snow3G, ZUC */ + +#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1 +/**< @ingroup icp_qat_fw_la + * FW Selects UCS Cipher Slice + * Cipher Algorithms: AES-{CTR/XTS}, Single Pass AES-GCM + * Auth Algorithms : SHA1/ SHA{2/3}-{224/256/384/512} */ + +#define ICP_QAT_FW_LA_USE_LEGACY_SLICE_TYPE 0 +/**< @ingroup icp_qat_fw_la + * FW Selects Legacy Cipher/Auth Slice + * Cipher Algorithms: AES-{CBC/ECB}, SM4, Single Pass AES-CCM + * Auth Algorithms : SHA1/ SHA{2/3}-{224/256/384/512} */ + +#define QAT_LA_SLICE_TYPE_BITPOS 14 +/**< @ingroup icp_qat_fw_la + * Starting bit position for the slice type selection. + * Refer to HAS for Slice type assignment details on QAT2.0 */ + +#define QAT_LA_SLICE_TYPE_MASK 0x3 +/**< @ingroup icp_qat_fw_la + * Two bit mask used to determine the Slice type */ + /* bit 11 */ #define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 /**< @ingroup icp_qat_fw_la @@ -482,7 +548,7 @@ typedef struct icp_qat_fw_la_bulk_req_s { * * @description * Macro for extraction of the Cipher IV field contents (bit 2) - * + * * @param flags Flags to extract the Cipher IV field contents * *****************************************************************************/ @@ -496,7 +562,7 @@ typedef struct icp_qat_fw_la_bulk_req_s { * @description * Macro for extraction of the Cipher/Auth Config * offset type (bit 3) - * + * * @param flags Flags to extract the Cipher/Auth Config * offset type * @@ -640,6 +706,19 @@ typedef struct icp_qat_fw_la_bulk_req_s { QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_BITPOS, \ QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_MASK) +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for extraction of the slice type information from the flags. + * + * @param flags Flags to extract the protocol state + * + *****************************************************************************/ +#define ICP_QAT_FW_LA_SLICE_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_SLICE_TYPE_BITPOS, QAT_LA_SLICE_TYPE_MASK) + /* Macros for setting field bits */ /** ****************************************************************************** @@ -647,7 +726,7 @@ typedef struct icp_qat_fw_la_bulk_req_s { * * @description * Macro for setting the Cipher IV field contents - * + * * @param flags Flags to set with the Cipher IV field contents * @param val Field contents indicator value * @@ -665,7 +744,7 @@ typedef struct icp_qat_fw_la_bulk_req_s { * @description * Macro for setting the Cipher/Auth Config * offset type - * + * * @param flags Flags to set the Cipher/Auth Config offset type * @param val Offset type value * @@ -840,6 +919,23 @@ typedef struct icp_qat_fw_la_bulk_req_s { QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_BITPOS, \ QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_MASK) +/** +****************************************************************************** +* @ingroup icp_qat_fw_la +* +* @description +* Macro for setting the "slice type" field in la flags +* +* @param flags Flags to set the slice type +* @param val Value of the slice type to be set. +* +*****************************************************************************/ +#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + QAT_LA_SLICE_TYPE_BITPOS, \ + QAT_LA_SLICE_TYPE_MASK) + /** ***************************************************************************** * @ingroup icp_qat_fw_la @@ -860,14 +956,10 @@ typedef union icp_qat_fw_cipher_req_hdr_cd_pars_s { uint8_t content_desc_params_sz; /**< Size of the content descriptor parameters in quad words. - * These - * parameters describe the session setup configuration info for - * the - * slices that this request relies upon i.e. the configuration - * word and - * cipher key needed by the cipher slice if there is a request - * for - * cipher processing. */ + * These parameters describe the session setup configuration + * info for the slices that this request relies upon i.e. the + * configuration word and cipher key needed by the cipher slice + * if there is a request for cipher processing. */ uint8_t content_desc_hdr_resrvd2; /**< Content descriptor reserved field */ @@ -916,14 +1008,10 @@ typedef union icp_qat_fw_cipher_auth_req_hdr_cd_pars_s { uint8_t content_desc_params_sz; /**< Size of the content descriptor parameters in quad words. - * These - * parameters describe the session setup configuration info for - * the - * slices that this request relies upon i.e. the configuration - * word and - * cipher key needed by the cipher slice if there is a request - * for - * cipher processing. */ + * These parameters describe the session setup configuration + * info for the slices that this request relies upon i.e. the + * configuration word and cipher key needed by the cipher slice + * if there is a request for cipher processing. */ uint8_t content_desc_hdr_resrvd2; /**< Content descriptor reserved field */ @@ -955,8 +1043,7 @@ typedef struct icp_qat_fw_cipher_cd_ctrl_hdr_s { /**< LW 27 */ uint8_t cipher_state_sz; /**< State size in quad words of the cipher algorithm used in this - * session. - * Set to zero if the algorithm doesnt provide any state */ + * session. Set to zero if the algorithm doesnt provide any state */ uint8_t cipher_key_sz; /**< Key size in quad words of the cipher algorithm used in this session @@ -964,17 +1051,16 @@ typedef struct icp_qat_fw_cipher_cd_ctrl_hdr_s { uint8_t cipher_cfg_offset; /**< Quad word offset from the content descriptor parameters address - * i.e. - * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher - * processing */ + * i.e. (content_address + (cd_hdr_sz << 3)) to the parameters for the + * cipher processing */ uint8_t next_curr_id; /**< This field combines the next and current id (each four bits) - - * the next id is the most significant nibble. - * Next Id: Set to the next slice to pass the ciphered data through. - * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through - * any more slices after cipher. - * Current Id: Initialised with the cipher slice type */ + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the ciphered data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after cipher. + * Current Id: Initialised with the cipher slice type */ /**< LW 28 */ uint8_t cipher_padding_sz; @@ -1021,17 +1107,15 @@ typedef struct icp_qat_fw_auth_cd_ctrl_hdr_s { uint8_t hash_cfg_offset; /**< Quad word offset from the content descriptor parameters address to - * the - * parameters for the auth processing */ + * the parameters for the auth processing */ uint8_t next_curr_id; /**< This field combines the next and current id (each four bits) - - * the next id is the most significant nibble. - * Next Id: Set to the next slice to pass the authentication data - * through. - * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through - * any more slices after authentication. - * Current Id: Initialised with the authentication slice type */ + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the authentication data + * through. Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go + * through any more slices after authentication. + * Current Id: Initialised with the authentication slice type */ /**< LW 29 */ uint8_t resrvd3; @@ -1057,8 +1141,7 @@ typedef struct icp_qat_fw_auth_cd_ctrl_hdr_s { uint8_t inner_state2_offset; /**< Quad word offset from the content descriptor parameters pointer to - * the - * inner state2 value */ + * the inner state2 value */ uint8_t inner_state2_sz; /**< Size in bytes of inner hash state2 data. Must be a qword multiple @@ -1067,8 +1150,7 @@ typedef struct icp_qat_fw_auth_cd_ctrl_hdr_s { /**< LW 31 */ uint8_t outer_config_offset; /**< Quad word offset from the content descriptor parameters pointer to - * the - * outer configuration information */ + * the outer configuration information */ uint8_t outer_state1_sz; /**< Size in bytes of the outer state1 value */ @@ -1078,10 +1160,8 @@ typedef struct icp_qat_fw_auth_cd_ctrl_hdr_s { uint8_t outer_prefix_offset; /**< Quad word offset from the start of the inner prefix data to the - * outer - * prefix information. Should equal the rounded inner prefix size, - * converted - * to qwords */ + * outer prefix information. Should equal the rounded inner prefix size, + * converted to qwords */ } icp_qat_fw_auth_cd_ctrl_hdr_t; @@ -1100,8 +1180,7 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { /**< LW 27 */ uint8_t cipher_state_sz; /**< State size in quad words of the cipher algorithm used in this - * session. - * Set to zero if the algorithm doesnt provide any state */ + * session. Set to zero if the algorithm doesnt provide any state */ uint8_t cipher_key_sz; /**< Key size in quad words of the cipher algorithm used in this session @@ -1109,17 +1188,16 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { uint8_t cipher_cfg_offset; /**< Quad word offset from the content descriptor parameters address - * i.e. - * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher - * processing */ + * i.e. (content_address + (cd_hdr_sz << 3)) to the parameters for the + * cipher processing */ uint8_t next_curr_id_cipher; /**< This field combines the next and current id (each four bits) - - * the next id is the most significant nibble. - * Next Id: Set to the next slice to pass the ciphered data through. - * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through - * any more slices after cipher. - * Current Id: Initialised with the cipher slice type */ + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the ciphered data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after cipher. + * Current Id: Initialised with the cipher slice type */ /**< LW 28 */ uint8_t cipher_padding_sz; @@ -1134,16 +1212,14 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { uint8_t hash_cfg_offset; /**< Quad word offset from the content descriptor parameters address to - * the - * parameters for the auth processing */ + * the parameters for the auth processing */ uint8_t next_curr_id_auth; /**< This field combines the next and current id (each four bits) - * the next id is the most significant nibble. * Next Id: Set to the next slice to pass the authentication data - * through. - * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through - * any more slices after authentication. + * through. Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go + * through any more slices after authentication. * Current Id: Initialised with the authentication slice type */ /**< LW 29 */ @@ -1170,8 +1246,7 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { uint8_t inner_state2_offset; /**< Quad word offset from the content descriptor parameters pointer to - * the - * inner state2 value */ + * the inner state2 value */ uint8_t inner_state2_sz; /**< Size in bytes of inner hash state2 data. Must be a qword multiple @@ -1180,8 +1255,7 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { /**< LW 31 */ uint8_t outer_config_offset; /**< Quad word offset from the content descriptor parameters pointer to - * the - * outer configuration information */ + * the outer configuration information */ uint8_t outer_state1_sz; /**< Size in bytes of the outer state1 value */ @@ -1191,10 +1265,8 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { uint8_t outer_prefix_offset; /**< Quad word offset from the start of the inner prefix data to the - * outer - * prefix information. Should equal the rounded inner prefix size, - * converted - * to qwords */ + * outer prefix information. Should equal the rounded inner prefix size, + * converted to qwords */ } icp_qat_fw_cipher_auth_cd_ctrl_hdr_t; @@ -1204,9 +1276,9 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { * + ===== + --- + --- + --- + --- + --- + --- + --- + ---- + * | Bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | * + ===== + --- + --- + --- + --- + --- + --- + --- + ---- + - * | Flags | Rsv | Rsv | Rsv | ZUC |SNOW | Rsv | Rsv |NESTED| - * | | | | |EIA3 | 3G | | | | - * | | | | | |UIA2 | | | | + * | Flags | Rsv | Rsv | Rsv | ZUC |SNOW |SKIP |SKIP |NESTED| + * | | | | |EIA3 | 3G |LOAD |LOAD | | + * | | | | | |UIA2 |OUTER|INNER| | * + ===== + --- + --- + --- + --- + --- + --- + --- + ---- + */ @@ -1233,6 +1305,42 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { * requires nested hashing */ +/* Bit 1 */ + +#define QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS 1 +/**< @ingroup icp_qat_fw_comn + * Bit position of the Skipping Inner State1 Load bit */ + +#define QAT_FW_LA_SKIP_INNER_STATE1_LOAD 1 +/**< @ingroup icp_qat_fw_comn + * Value indicating the skipping of inner hash state load */ + +#define QAT_FW_LA_NO_SKIP_INNER_STATE1_LOAD 0 +/**< @ingroup icp_qat_fw_comn + * Value indicating the no skipping of inner hash state load */ + +#define QAT_FW_LA_SKIP_INNER_STATE1_LOAD_MASK 0x1 +/**< @ingroup icp_qat_fw_comn + * Bit mask of Skipping Inner State1 Load bit */ + +/* Bit 2 */ + +#define QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS 2 +/**< @ingroup icp_qat_fw_comn + * Bit position of the Skipping Outer State1 Load bit */ + +#define QAT_FW_LA_SKIP_OUTER_STATE1_LOAD 1 +/**< @ingroup icp_qat_fw_comn + * Value indicating the skipping of outer hash state load */ + +#define QAT_FW_LA_NO_SKIP_OUTER_STATE1_LOAD 0 +/**< @ingroup icp_qat_fw_comn + * Value indicating the no skipping of outer hash state load */ + +#define QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK 0x1 +/**< @ingroup icp_qat_fw_comn + * Bit mask of Skipping Outer State1 Load bit */ + /* Bit 3 */ #define QAT_FW_LA_SNOW3G_UIA2_BITPOS 3 @@ -1261,6 +1369,24 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { /**< @ingroup icp_qat_fw_la * One bit mask used to determine the use of hash algorithm ZUC-EIA3 */ +/* Bit 5 */ + +#define QAT_FW_LA_MODE2_BITPOS 5 +/**< @ingroup icp_qat_fw_comn + * Bit position of the Mode 2 bit */ + +#define QAT_FW_LA_MODE2 1 +/**< @ingroup icp_qat_fw_comn + * Value indicating the Mode 2*/ + +#define QAT_FW_LA_NO_MODE2 0 +/**< @ingroup icp_qat_fw_comn + * Value indicating the no Mode 2*/ + +#define QAT_FW_LA_MODE2_MASK 0x1 +/**< @ingroup icp_qat_fw_comn + * Bit mask of Mode 2 */ + /* Macros for extracting hash flags */ /** @@ -1279,6 +1405,67 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { QAT_FW_LA_AUTH_HDR_NESTED_BITPOS, \ QAT_FW_LA_AUTH_HDR_NESTED_MASK) +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for extraction of the "Skipping Inner State1 Load state" hash flag + * + * @param flags Hash Flags + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_GET(flags) \ + QAT_FIELD_GET(flags, \ + QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS, \ + QAT_FW_LA_INNER_STATE1_LOAD_MASK) + +/** + ****************************************************************************** + * Macro for setting the "Skipping Inner State1 Load" hash flag + * + * @param flags Hash Flags + * @param val Value of the flag + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS, \ + QAT_FW_LA_SKIP_INNER_STATE1_LOAD_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for extraction of the "Skipping Outer State1 Load state" hash flag + * + * @param flags Hash Flags + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_GET(flags) \ + QAT_FIELD_GET(flags, \ + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS, \ + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for setting the "Skipping Outer State1 Load" hash flag + * + * @param flags Hash Flags + * @param val Value of the flag + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS, \ + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK) + /** ****************************************************************************** * @ingroup icp_qat_fw_la @@ -1328,6 +1515,40 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { QAT_FW_LA_AUTH_HDR_NESTED_BITPOS, \ QAT_FW_LA_AUTH_HDR_NESTED_MASK) +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for setting the "Skipping Inner State1 Load" hash flag + * + * @param flags Hash Flags + * @param val Value of the flag + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS, \ + QAT_FW_LA_SKIP_INNER_STATE1_LOAD_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for setting the "Skipping Outer State1 Load" hash flag + * + * @param flags Hash Flags + * @param val Value of the flag + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS, \ + QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK) + /** ****************************************************************************** * @ingroup icp_qat_fw_la @@ -1362,6 +1583,20 @@ typedef struct icp_qat_fw_cipher_auth_cd_ctrl_hdr_s { QAT_FW_LA_ZUC_EIA3_BITPOS, \ QAT_FW_LA_ZUC_EIA3_MASK) +/** + ****************************************************************************** + * @ingroup icp_qat_fw_la + * + * @description + * Macro for setting the "Mode 2" hash flag + * + * @param flags Hash Flags + * @param val Value of the flag + * + *****************************************************************************/ +#define ICP_QAT_FW_HASH_FLAG_MODE2_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_FW_LA_MODE2_BITPOS, QAT_FW_LA_MODE2_MASK) + #define ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX 240 #define ICP_QAT_FW_SPC_AAD_SZ_MAX 0x3FFF @@ -1494,8 +1729,7 @@ typedef struct icp_qat_fw_la_auth_req_params_s { uint8_t aad_sz; /**< Size in bytes of padded AAD data to prefix to the packet - * for CCM - * or GCM processing */ + * for CCM or GCM processing */ } u2; uint8_t resrvd1; @@ -1535,8 +1769,7 @@ typedef struct icp_qat_fw_la_auth_req_params_resrvd_flds_s { uint8_t aad_sz; /**< Size in bytes of padded AAD data to prefix to the packet - * for CCM - * or GCM processing */ + * for CCM or GCM processing */ } u2; uint8_t resrvd1; @@ -1565,8 +1798,7 @@ typedef struct icp_qat_fw_la_key_gen_common_s { /**< SSL3 */ uint16_t secret_lgth_ssl; /**< Length of Secret information for SSL. In the case of TLS - * the - * secret is supplied in the content descriptor */ + * the secret is supplied in the content descriptor */ /**< MGF */ uint16_t mask_length; @@ -1692,8 +1924,7 @@ typedef struct icp_qat_fw_la_ssl3_req_params_s { uint8_t aad_sz; /**< Size in bytes of padded AAD data to prefix to the packet - * for CCM - * or GCM processing */ + * for CCM or GCM processing */ } u2; uint8_t resrvd1; @@ -1731,8 +1962,7 @@ typedef struct icp_qat_fw_la_mgf_req_params_s { uint8_t aad_sz; /**< Size in bytes of padded AAD data to prefix to the packet - * for CCM - * or GCM processing */ + * for CCM or GCM processing */ } u2; uint8_t resrvd1; @@ -1779,16 +2009,14 @@ typedef struct icp_qat_fw_la_trng_req_mid_s { /**< LWs 6-13 */ uint64_t opaque_data; /**< Opaque data passed unmodified from the request to response messages - * by - * firmware (fw) */ + * by firmware (fw) */ uint64_t resrvd1; /**< Reserved, unused for TRNG */ uint64_t dest_data_addr; /**< Generic definition of the destination data supplied to the QAT AE. - * The - * common flags are used to further describe the attributes of this + * The common flags are used to further describe the attributes of this * field */ uint32_t resrvd2; @@ -1796,7 +2024,7 @@ typedef struct icp_qat_fw_la_trng_req_mid_s { uint32_t entropy_length; /**< Size of the data in bytes to process. Used by the get_random - * command. Set to 0 for commands that dont need a length parameter */ + * command. Set to 0 for commands that dont need a length parameter */ } icp_qat_fw_la_trng_req_mid_t; @@ -2235,10 +2463,8 @@ struct icp_qat_fw_hkdf_label { union { uint8_t label_flags; /**< For first-level labels: each bit in [0..3] will trigger a - * child - * Expand-Label operation on the corresponding sublabel. Bits - * [4..7] - * are reserved. + * child Expand-Label operation on the corresponding sublabel. + * Bits [4..7] are reserved. */ uint8_t sublabel_flags; diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h index 5ed607aa3c3b..629b6a3d675c 100644 --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h @@ -25,22 +25,22 @@ /* ========================================================================= */ typedef enum { - ICP_QAT_HW_AE_0 = 0, /*!< ID of AE0 */ - ICP_QAT_HW_AE_1 = 1, /*!< ID of AE1 */ - ICP_QAT_HW_AE_2 = 2, /*!< ID of AE2 */ - ICP_QAT_HW_AE_3 = 3, /*!< ID of AE3 */ - ICP_QAT_HW_AE_4 = 4, /*!< ID of AE4 */ - ICP_QAT_HW_AE_5 = 5, /*!< ID of AE5 */ - ICP_QAT_HW_AE_6 = 6, /*!< ID of AE6 */ - ICP_QAT_HW_AE_7 = 7, /*!< ID of AE7 */ - ICP_QAT_HW_AE_8 = 8, /*!< ID of AE8 */ - ICP_QAT_HW_AE_9 = 9, /*!< ID of AE9 */ - ICP_QAT_HW_AE_10 = 10, /*!< ID of AE10 */ - ICP_QAT_HW_AE_11 = 11, /*!< ID of AE11 */ - ICP_QAT_HW_AE_12 = 12, /*!< ID of AE12 */ - ICP_QAT_HW_AE_13 = 13, /*!< ID of AE13 */ - ICP_QAT_HW_AE_14 = 14, /*!< ID of AE14 */ - ICP_QAT_HW_AE_15 = 15, /*!< ID of AE15 */ + ICP_QAT_HW_AE_0 = 0, /*!< ID of AE0 */ + ICP_QAT_HW_AE_1 = 1, /*!< ID of AE1 */ + ICP_QAT_HW_AE_2 = 2, /*!< ID of AE2 */ + ICP_QAT_HW_AE_3 = 3, /*!< ID of AE3 */ + ICP_QAT_HW_AE_4 = 4, /*!< ID of AE4 */ + ICP_QAT_HW_AE_5 = 5, /*!< ID of AE5 */ + ICP_QAT_HW_AE_6 = 6, /*!< ID of AE6 */ + ICP_QAT_HW_AE_7 = 7, /*!< ID of AE7 */ + ICP_QAT_HW_AE_8 = 8, /*!< ID of AE8 */ + ICP_QAT_HW_AE_9 = 9, /*!< ID of AE9 */ + ICP_QAT_HW_AE_10 = 10, /*!< ID of AE10 */ + ICP_QAT_HW_AE_11 = 11, /*!< ID of AE11 */ + ICP_QAT_HW_AE_12 = 12, /*!< ID of AE12 */ + ICP_QAT_HW_AE_13 = 13, /*!< ID of AE13 */ + ICP_QAT_HW_AE_14 = 14, /*!< ID of AE14 */ + ICP_QAT_HW_AE_15 = 15, /*!< ID of AE15 */ ICP_QAT_HW_AE_DELIMITER = 16 /**< Delimiter type */ } icp_qat_hw_ae_id_t; @@ -49,12 +49,12 @@ typedef enum { /* ========================================================================= */ typedef enum { - ICP_QAT_HW_QAT_0 = 0, /*!< ID of QAT0 */ - ICP_QAT_HW_QAT_1 = 1, /*!< ID of QAT1 */ - ICP_QAT_HW_QAT_2 = 2, /*!< ID of QAT2 */ - ICP_QAT_HW_QAT_3 = 3, /*!< ID of QAT3 */ - ICP_QAT_HW_QAT_4 = 4, /*!< ID of QAT4 */ - ICP_QAT_HW_QAT_5 = 5, /*!< ID of QAT5 */ + ICP_QAT_HW_QAT_0 = 0, /*!< ID of QAT0 */ + ICP_QAT_HW_QAT_1 = 1, /*!< ID of QAT1 */ + ICP_QAT_HW_QAT_2 = 2, /*!< ID of QAT2 */ + ICP_QAT_HW_QAT_3 = 3, /*!< ID of QAT3 */ + ICP_QAT_HW_QAT_4 = 4, /*!< ID of QAT4 */ + ICP_QAT_HW_QAT_5 = 5, /*!< ID of QAT5 */ ICP_QAT_HW_QAT_DELIMITER = 6 /**< Delimiter type */ } icp_qat_hw_qat_id_t; @@ -79,24 +79,24 @@ typedef enum { ICP_QAT_HW_AUTH_ALGO_SHA256 = 4, /*!< SHA-256 hashing */ ICP_QAT_HW_AUTH_ALGO_SHA384 = 5, /*!< SHA-384 hashing */ ICP_QAT_HW_AUTH_ALGO_SHA512 = 6, /*!< SHA-512 hashing */ - ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, /*!< AES-XCBC-MAC hashing */ - ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, /*!< AES-CBC-MAC hashing */ + ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, /*!< AES-XCBC-MAC hashing */ + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, /*!< AES-CBC-MAC hashing */ ICP_QAT_HW_AUTH_ALGO_AES_F9 = 9, /*!< AES F9 hashing */ - ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, /*!< Galois 128 bit hashing */ - ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, /*!< Galois 64 hashing */ - ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, /*!< Kasumi F9 hashing */ + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, /*!< Galois 128 bit hashing */ + ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, /*!< Galois 64 hashing */ + ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, /*!< Kasumi F9 hashing */ ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, /*!< UIA2/SNOW_3G F9 hashing */ ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 = - 14, /*!< 128_EIA3/ZUC_3G hashing */ - ICP_QAT_HW_AUTH_ALGO_SM3 = 15, /*!< SM3 hashing */ - ICP_QAT_HW_AUTH_ALGO_SHA3_224 = 16, /*!< SHA3-224 hashing */ - ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, /*!< SHA3-256 hashing */ - ICP_QAT_HW_AUTH_ALGO_SHA3_384 = 18, /*!< SHA3-384 hashing */ - ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, /*!< SHA3-512 hashing */ - ICP_QAT_HW_AUTH_ALGO_SHAKE_128 = 20, /*!< SHAKE-128 hashing */ - ICP_QAT_HW_AUTH_ALGO_SHAKE_256 = 21, /*!< SHAKE-256 hashing */ - ICP_QAT_HW_AUTH_ALGO_POLY = 22, /*!< POLY hashing */ - ICP_QAT_HW_AUTH_ALGO_DELIMITER = 23 /**< Delimiter type */ + 14, /*!< 128_EIA3/ZUC_3G hashing */ + ICP_QAT_HW_AUTH_ALGO_SM3 = 15, /*!< SM3 hashing */ + ICP_QAT_HW_AUTH_ALGO_SHA3_224 = 16, /*!< SHA3-224 hashing */ + ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, /*!< SHA3-256 hashing */ + ICP_QAT_HW_AUTH_ALGO_SHA3_384 = 18, /*!< SHA3-384 hashing */ + ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, /*!< SHA3-512 hashing */ + ICP_QAT_HW_AUTH_RESERVED_4 = 20, /*!< Reserved */ + ICP_QAT_HW_AUTH_RESERVED_5 = 21, /*!< Reserved */ + ICP_QAT_HW_AUTH_ALGO_POLY = 22, /*!< POLY hashing */ + ICP_QAT_HW_AUTH_ALGO_DELIMITER = 23 /**< Delimiter type */ } icp_qat_hw_auth_algo_t; /** @@ -118,9 +118,9 @@ typedef enum { *****************************************************************************/ typedef enum { - ICP_QAT_HW_AUTH_MODE0 = 0, /*!< QAT Auth Mode0 configuration */ - ICP_QAT_HW_AUTH_MODE1 = 1, /*!< QAT Auth Mode1 configuration */ - ICP_QAT_HW_AUTH_MODE2 = 2, /*!< QAT AuthMode2 configuration */ + ICP_QAT_HW_AUTH_MODE0 = 0, /*!< QAT Auth Mode0 configuration */ + ICP_QAT_HW_AUTH_MODE1 = 1, /*!< QAT Auth Mode1 configuration */ + ICP_QAT_HW_AUTH_MODE2 = 2, /*!< QAT AuthMode2 configuration */ ICP_QAT_HW_AUTH_MODE_DELIMITER = 3 /**< Delimiter type */ } icp_qat_hw_auth_mode_t; @@ -269,7 +269,7 @@ typedef struct icp_qat_hw_auth_config_s { /**< Flag usage - see additional notes @description for * ICP_QAT_HW_AUTH_CONFIG_BUILD and * ICP_QAT_HW_AUTH_CONFIG_BUILD_UPPER macros. -*/ + */ #define QAT_AUTH_SHA3_HW_PADDING_ENABLE 0 /**< @ingroup icp_qat_hw_defs @@ -461,7 +461,7 @@ typedef struct icp_qat_hw_auth_setup_s { #define QAT_HW_ROUND_UP(val, n) (((val) + ((n)-1)) & (~(n - 1))) /* State1 */ -#define ICP_QAT_HW_NULL_STATE1_SZ 64 +#define ICP_QAT_HW_NULL_STATE1_SZ 32 /**< @ingroup icp_qat_hw_defs * State1 block size for NULL hashing */ #define ICP_QAT_HW_MD5_STATE1_SZ 16 @@ -474,7 +474,7 @@ typedef struct icp_qat_hw_auth_setup_s { #define ICP_QAT_HW_SHA224_STATE1_SZ 32 /**< @ingroup icp_qat_hw_defs * State1 block size for SHA24 */ -#define ICP_QAT_HW_SHA3_224_STATE1_SZ 32 +#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28 /**< @ingroup icp_qat_hw_defs * State1 block size for SHA3_224 */ #define ICP_QAT_HW_SHA256_STATE1_SZ 32 @@ -486,7 +486,7 @@ typedef struct icp_qat_hw_auth_setup_s { #define ICP_QAT_HW_SHA384_STATE1_SZ 64 /**< @ingroup icp_qat_hw_defs * State1 block size for SHA384 */ -#define ICP_QAT_HW_SHA3_384_STATE1_SZ 64 +#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48 /**< @ingroup icp_qat_hw_defs * State1 block size for SHA3_384 */ #define ICP_QAT_HW_SHA512_STATE1_SZ 64 @@ -516,15 +516,15 @@ typedef struct icp_qat_hw_auth_setup_s { #define ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ 8 /**< @ingroup icp_cpm_hw_defs * State1 block size for EIA3 */ +#define ICP_QAT_HW_SM3_STATE1_SZ 32 +/**< @ingroup icp_qat_hw_defs + * State1 block size for SM3 */ #define ICP_QAT_HW_SHA3_STATEFUL_STATE1_SZ 200 /** <@ingroup icp_cpm_hw_defs * State1 block size for stateful SHA3 processing*/ -#define ICP_QAT_HW_SM3_STATE1_SZ 32 -/**< @ingroup icp_cpm_hw_defs - * State1 block size for SM3 */ /* State2 */ -#define ICP_QAT_HW_NULL_STATE2_SZ 64 +#define ICP_QAT_HW_NULL_STATE2_SZ 32 /**< @ingroup icp_qat_hw_defs * State2 block size for NULL hashing */ #define ICP_QAT_HW_MD5_STATE2_SZ 16 @@ -537,25 +537,25 @@ typedef struct icp_qat_hw_auth_setup_s { #define ICP_QAT_HW_SHA224_STATE2_SZ 32 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA224 */ -#define ICP_QAT_HW_SHA3_224_STATE2_SZ 32 +#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA3_224 */ #define ICP_QAT_HW_SHA256_STATE2_SZ 32 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA256 */ -#define ICP_QAT_HW_SHA3_256_STATE2_SZ 32 +#define ICP_QAT_HW_SHA3_256_STATE2_SZ 0 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA3_256 */ #define ICP_QAT_HW_SHA384_STATE2_SZ 64 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA384 */ -#define ICP_QAT_HW_SHA3_384_STATE2_SZ 64 +#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA3_384 */ #define ICP_QAT_HW_SHA512_STATE2_SZ 64 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA512 */ -#define ICP_QAT_HW_SHA3_512_STATE2_SZ 64 +#define ICP_QAT_HW_SHA3_512_STATE2_SZ 0 /**< @ingroup icp_qat_hw_defs * State2 block size for SHA3_512 */ #define ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ 16 @@ -598,6 +598,9 @@ typedef struct icp_qat_hw_auth_setup_s { #define ICP_QAT_HW_SM3_STATE2_SZ 32 /**< @ingroup icp_qat_hw_defs * State2 block size for SM3 */ +#define ICP_QAT_HW_SHA3_STATEFUL_STATE2_SZ 208 +/** <@ingroup icp_cpm_hw_defs + * State2 block size for stateful SHA3 processing*/ /* ************************************************************************* */ /* ************************************************************************* */ @@ -647,10 +650,33 @@ typedef struct icp_qat_hw_auth_sha3_512_s { icp_qat_hw_auth_setup_t outer_setup; /**< Outer configuration word for the slice */ - /* State2 size is zero - this may change for future implementations */ - uint8_t state2[ICP_QAT_HW_SHA3_512_STATE2_SZ]; } icp_qat_hw_auth_sha3_512_t; +/** + ***************************************************************************** + * @ingroup icp_qat_hw_defs + * Definition of stateful SHA3 auth algorithm processing struct + * @description + * This structs described the parameters to pass to the slice for + * configuring it for stateful SHA3 processing. This is the largest + * possible setup block for authentication + * + *****************************************************************************/ +typedef struct icp_qat_hw_auth_sha3_stateful_s { + icp_qat_hw_auth_setup_t inner_setup; + /**< Inner loop configuration word for the slice */ + + uint8_t inner_state1[ICP_QAT_HW_SHA3_STATEFUL_STATE1_SZ]; + /**< Inner hash block */ + + icp_qat_hw_auth_setup_t outer_setup; + /**< Outer configuration word for the slice */ + + uint8_t outer_state1[ICP_QAT_HW_SHA3_STATEFUL_STATE1_SZ]; + /**< Outer hash block */ + +} icp_qat_hw_auth_sha3_stateful_t; + /** ***************************************************************************** * @ingroup icp_qat_hw_defs @@ -662,6 +688,8 @@ typedef struct icp_qat_hw_auth_sha3_512_s { typedef union icp_qat_hw_auth_algo_blk_u { icp_qat_hw_auth_sha512_t sha512; /**< SHA512 Hashing */ + icp_qat_hw_auth_sha3_stateful_t sha3_stateful; + /**< Stateful SHA3 Hashing */ } icp_qat_hw_auth_algo_blk_t; @@ -691,11 +719,11 @@ typedef enum { ICP_QAT_HW_CIPHER_ALGO_NULL = 0, /*!< Null ciphering */ ICP_QAT_HW_CIPHER_ALGO_DES = 1, /*!< DES ciphering */ ICP_QAT_HW_CIPHER_ALGO_3DES = 2, /*!< 3DES ciphering */ - ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, /*!< AES-128 ciphering */ - ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, /*!< AES-192 ciphering */ - ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, /*!< AES-256 ciphering */ + ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, /*!< AES-128 ciphering */ + ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, /*!< AES-192 ciphering */ + ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, /*!< AES-256 ciphering */ ICP_QAT_HW_CIPHER_ALGO_ARC4 = 6, /*!< ARC4 ciphering */ - ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, /*!< Kasumi */ + ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, /*!< Kasumi */ ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, /*!< Snow_3G */ ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, /*!< ZUC_3G */ ICP_QAT_HW_CIPHER_ALGO_SM4 = 10, /*!< SM4 ciphering */ @@ -723,7 +751,7 @@ typedef enum { ICP_QAT_HW_CIPHER_CTR_MODE = 2, /*!< CTR mode */ ICP_QAT_HW_CIPHER_F8_MODE = 3, /*!< F8 mode */ ICP_QAT_HW_CIPHER_AEAD_MODE = 4, /*!< AES-GCM SPC AEAD mode */ - ICP_QAT_HW_CIPHER_RESERVED_MODE = 5, /*!< Reserved */ + ICP_QAT_HW_CIPHER_CCM_MODE = 5, /*!< AES-CCM SPC AEAD mode */ ICP_QAT_HW_CIPHER_XTS_MODE = 6, /*!< XTS mode */ ICP_QAT_HW_CIPHER_MODE_DELIMITER = 7 /**< Delimiter type */ } icp_qat_hw_cipher_mode_t; @@ -746,6 +774,23 @@ typedef struct icp_qat_hw_cipher_config_s { /**< Reserved */ } icp_qat_hw_cipher_config_t; +/** + ***************************************************************************** + * @ingroup icp_qat_hw_defs + * Cipher Configuration Struct + * + * @description + * Configuration data used for setting up the QAT UCS Cipher Slice + * + *****************************************************************************/ +typedef struct icp_qat_hw_ucs_cipher_config_s { + uint32_t val; + /**< Cipher slice configuration */ + + uint32_t reserved[3]; + /**< Reserved */ +} icp_qat_hw_ucs_cipher_config_t; + /** ***************************************************************************** * @ingroup icp_qat_hw_defs @@ -851,6 +896,10 @@ typedef enum { /**< @ingroup icp_qat_hw_defs * Define for the cipher XTS mode key size */ +#define QAT_CIPHER_MODE_UCS_XTS_KEY_SZ_MULT 1 +/**< @ingroup icp_qat_hw_defs + * Define for the UCS cipher XTS mode key size */ + /** ****************************************************************************** * @ingroup icp_qat_hw_defs @@ -931,6 +980,16 @@ typedef enum { #define ICP_QAT_HW_AES_256_KEY_SZ 32 /**< @ingroup icp_qat_hw_defs * Define the key size for AES256 */ +/* AES UCS */ +#define ICP_QAT_HW_UCS_AES_128_KEY_SZ ICP_QAT_HW_AES_128_KEY_SZ +/**< @ingroup icp_qat_hw_defs + * Define the key size for AES128 for UCS slice*/ +#define ICP_QAT_HW_UCS_AES_192_KEY_SZ 32 +/**< @ingroup icp_qat_hw_defs + * Define the key size for AES192 for UCS slice*/ +#define ICP_QAT_HW_UCS_AES_256_KEY_SZ ICP_QAT_HW_AES_256_KEY_SZ +/**< @ingroup icp_qat_hw_defs + * Define the key size for AES256 for UCS slice*/ #define ICP_QAT_HW_AES_128_F8_KEY_SZ \ (ICP_QAT_HW_AES_128_KEY_SZ * QAT_CIPHER_MODE_F8_KEY_SZ_MULT) /**< @ingroup icp_qat_hw_defs @@ -951,6 +1010,14 @@ typedef enum { (ICP_QAT_HW_AES_256_KEY_SZ * QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) /**< @ingroup icp_qat_hw_defs * Define the key size for AES256 XTS */ +#define ICP_QAT_HW_UCS_AES_128_XTS_KEY_SZ \ + (ICP_QAT_HW_UCS_AES_128_KEY_SZ * QAT_CIPHER_MODE_UCS_XTS_KEY_SZ_MULT) +/**< @ingroup icp_qat_hw_defs + * Define the key size for AES128 XTS for the UCS Slice*/ +#define ICP_QAT_HW_UCS_AES_256_XTS_KEY_SZ \ + (ICP_QAT_HW_UCS_AES_256_KEY_SZ * QAT_CIPHER_MODE_UCS_XTS_KEY_SZ_MULT) +/**< @ingroup icp_qat_hw_defs + * Define the key size for AES256 XTS for the UCS Slice*/ #define ICP_QAT_HW_KASUMI_KEY_SZ 16 /**< @ingroup icp_qat_hw_defs * Define the key size for Kasumi */ @@ -1090,10 +1157,10 @@ typedef enum { *****************************************************************************/ typedef enum { - ICP_QAT_HW_TRNG_NEG_0 = 0, /*!< TRNG Neg Zero Test */ - ICP_QAT_HW_TRNG_NEG_1 = 1, /*!< TRNG Neg One Test */ + ICP_QAT_HW_TRNG_NEG_0 = 0, /*!< TRNG Neg Zero Test */ + ICP_QAT_HW_TRNG_NEG_1 = 1, /*!< TRNG Neg One Test */ ICP_QAT_HW_TRNG_POS = 2, /*!< TRNG POS Test */ - ICP_QAT_HW_TRNG_POS_VNC = 3, /*!< TRNG POS VNC Test */ + ICP_QAT_HW_TRNG_POS_VNC = 3, /*!< TRNG POS VNC Test */ ICP_QAT_HW_TRNG_KAT_DELIMITER = 4 /**< Delimiter type */ } icp_qat_hw_trng_kat_mode_t; @@ -1388,7 +1455,7 @@ typedef enum { typedef enum { ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE = 0, /*!< Deflate compression */ - ICP_QAT_HW_COMPRESSION_DEPRECATED = 1, /*!< Deprecated */ + ICP_QAT_HW_COMPRESSION_DEPRECATED = 1, /*!< Deprecated */ ICP_QAT_HW_COMPRESSION_ALGO_DELIMITER = 2 /**< Delimiter type */ } icp_qat_hw_compression_algo_t; @@ -1470,11 +1537,11 @@ typedef enum { *****************************************************************************/ typedef struct icp_qat_hw_compression_config_s { - uint32_t val; - /**< Compression slice configuration */ + uint32_t lower_val; + /**< Compression slice configuration lower LW */ - uint32_t reserved; - /**< Reserved */ + uint32_t upper_val; + /**< Compression slice configuration upper LW */ } icp_qat_hw_compression_config_t; /* Private defines */ diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp.h new file mode 100644 index 000000000000..6e3ee4ce0446 --- /dev/null +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp.h @@ -0,0 +1,292 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2022 Intel Corporation */ +/* $FreeBSD$ */ +/** + ***************************************************************************** + * @file icp_qat_hw_2x_comp.h + * @defgroup ICP QAT HW accessors for using the for 2.x Compression Slice + * definitions + * @ingroup icp_qat_hw_2x_comp + * @description + * This file documents definitions for the QAT HW COMP SLICE + * + *****************************************************************************/ + +#ifndef _ICP_QAT_HW_20_COMP_H_ +#define _ICP_QAT_HW_20_COMP_H_ + +#include "icp_qat_hw_20_comp_defs.h" // For HW definitions +#include "icp_qat_fw.h" //For Set Field Macros. + +#ifdef WIN32 +#include // built in support for _byteswap_ulong +#define BYTE_SWAP_32 _byteswap_ulong +#else +#define BYTE_SWAP_32 __builtin_bswap32 +#endif + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Definition of the hw config csr. This representation has to be further +* processed by the corresponding config build function. +* +*****************************************************************************/ +typedef struct icp_qat_hw_comp_20_config_csr_lower_s { + // Fields programmable directly by the SW. + icp_qat_hw_comp_20_extended_delay_match_mode_t edmm; + icp_qat_hw_comp_20_hw_comp_format_t algo; + icp_qat_hw_comp_20_search_depth_t sd; + icp_qat_hw_comp_20_hbs_control_t hbs; + // Fields programmable directly by the FW. + // Block Drop enable. (Set by FW) + icp_qat_hw_comp_20_abd_t abd; + icp_qat_hw_comp_20_lllbd_ctrl_t lllbd; + // Advanced HW control (Set to default vals) + icp_qat_hw_comp_20_skip_hash_collision_t hash_col; + icp_qat_hw_comp_20_skip_hash_update_t hash_update; + icp_qat_hw_comp_20_byte_skip_t skip_ctrl; + +} icp_qat_hw_comp_20_config_csr_lower_t; + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Build the longword as expected by the HW +* +*****************************************************************************/ +static inline uint32_t +ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(icp_qat_hw_comp_20_config_csr_lower_t csr) +{ + uint32_t val32 = 0; + // Programmable values + QAT_FIELD_SET(val32, + csr.algo, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK); + + QAT_FIELD_SET(val32, + csr.sd, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK); + + QAT_FIELD_SET( + val32, + csr.edmm, + ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK); + + QAT_FIELD_SET(val32, + csr.hbs, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.lllbd, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK); + + QAT_FIELD_SET(val32, + csr.hash_col, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK); + + QAT_FIELD_SET(val32, + csr.hash_update, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK); + + QAT_FIELD_SET(val32, + csr.skip_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK); + // Default values. + + QAT_FIELD_SET(val32, + csr.abd, + ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK); + + QAT_FIELD_SET(val32, + csr.lllbd, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK); + + return BYTE_SWAP_32(val32); +} + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Definition of the hw config csr. This representation has to be further +* processed by the corresponding config build function. +* +*****************************************************************************/ +typedef struct icp_qat_hw_comp_20_config_csr_upper_s { + icp_qat_hw_comp_20_scb_control_t scb_ctrl; + icp_qat_hw_comp_20_rmb_control_t rmb_ctrl; + icp_qat_hw_comp_20_som_control_t som_ctrl; + icp_qat_hw_comp_20_skip_hash_rd_control_t skip_hash_ctrl; + icp_qat_hw_comp_20_scb_unload_control_t scb_unload_ctrl; + icp_qat_hw_comp_20_disable_token_fusion_control_t + disable_token_fusion_ctrl; + icp_qat_hw_comp_20_scb_mode_reset_mask_t scb_mode_reset; + uint16_t lazy; + uint16_t nice; +} icp_qat_hw_comp_20_config_csr_upper_t; + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Build the longword as expected by the HW +* +*****************************************************************************/ +static inline uint32_t +ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(icp_qat_hw_comp_20_config_csr_upper_t csr) +{ + uint32_t val32 = 0; + + QAT_FIELD_SET(val32, + csr.scb_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.rmb_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.som_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.skip_hash_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.scb_unload_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK); + + QAT_FIELD_SET( + val32, + csr.disable_token_fusion_ctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.scb_mode_reset, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); + + QAT_FIELD_SET(val32, + csr.lazy, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK); + + QAT_FIELD_SET(val32, + csr.nice, + ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK); + + return BYTE_SWAP_32(val32); +} + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Definition of the hw config csr. This representation has to be further +* processed by the corresponding config build function. +* +*****************************************************************************/ +typedef struct icp_qat_hw_decomp_20_config_csr_lower_s { + /* Fields programmable directly by the SW. */ + icp_qat_hw_decomp_20_hbs_control_t hbs; + /* Advanced HW control (Set to default vals) */ + icp_qat_hw_decomp_20_hw_comp_format_t algo; +} icp_qat_hw_decomp_20_config_csr_lower_t; + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Build the longword as expected by the HW +* +*****************************************************************************/ +static inline uint32_t +ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER( + icp_qat_hw_decomp_20_config_csr_lower_t csr) +{ + uint32_t val32 = 0; + + QAT_FIELD_SET(val32, + csr.hbs, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.algo, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK); + + return BYTE_SWAP_32(val32); +} + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Definition of the hw config csr. This representation has to be further +* processed by the corresponding config build function. +* +*****************************************************************************/ +typedef struct icp_qat_hw_decomp_20_config_csr_upper_s { + /* Advanced HW control (Set to default vals) */ + icp_qat_hw_decomp_20_speculative_decoder_control_t sdc; + icp_qat_hw_decomp_20_mini_cam_control_t mcc; +} icp_qat_hw_decomp_20_config_csr_upper_t; + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comn +* +* @description +* Build the longword as expected by the HW +* +*****************************************************************************/ +static inline uint32_t +ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_UPPER( + icp_qat_hw_decomp_20_config_csr_upper_t csr) +{ + uint32_t val32 = 0; + + QAT_FIELD_SET( + val32, + csr.sdc, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK); + + QAT_FIELD_SET(val32, + csr.mcc, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK); + + return BYTE_SWAP_32(val32); +} + +#endif /* ICP_QAT_HW__2X_COMP_H_ */ diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h new file mode 100644 index 000000000000..496f2e5c8de5 --- /dev/null +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h @@ -0,0 +1,443 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2022 Intel Corporation */ +/* $FreeBSD$ */ +/* + **************************************************************************** + * @file icp_qat_hw_20_comp_defs.h, (autogenerated at 04-19-18 16:06) + * @defgroup icp_qat_hw_comp_20 + * @ingroup icp_qat_hw_comp_20 + * @description + * This file represents the HW configuration CSR definitions + **************************************************************************** + */ + +#ifndef _ICP_QAT_HW_20_COMP_DEFS_H +#define _ICP_QAT_HW_20_COMP_DEFS_H + +/*****************************************************************************/ +/* SCB Disabled - Set by FW, located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS 31 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SCB_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SCB_CONTROL_ENABLE = 0x0, + /* Normal Mode using SCB (Default) */ + ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE = 0x1, + /* Legacy CPM1.x Mode with SCB disabled. */ +} icp_qat_hw_comp_20_scb_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE + +/*****************************************************************************/ +/* Reset Bit Mask Disabled - Set by FW , located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS 30 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible RMB_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL = 0x0, + /* Reset all data structures with a set_config command. (Set by FW) */ + ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_FC_ONLY = 0x1, + /* Reset only the Frequency Counters (LFCT) with a set_config command. + */ +} icp_qat_hw_comp_20_rmb_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL + +/*****************************************************************************/ +/* Slice Operation Mode (SOM) - Set By FW, located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS 28 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK 0x3 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SOM_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE = 0x0, + /* Normal mode. */ + ICP_QAT_HW_COMP_20_SOM_CONTROL_REPLAY_MODE = 0x1, + /* Replay mode */ + ICP_QAT_HW_COMP_20_SOM_CONTROL_INPUT_CRC = 0x2, + /* Input CRC Mode */ + ICP_QAT_HW_COMP_20_SOM_CONTROL_RESERVED_MODE = 0x3, + /* Reserved. */ +} icp_qat_hw_comp_20_som_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE + +/*****************************************************************************/ +/* Skip Hash Read (Set By FW) , located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SKIP_HASH_RD_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0, + /* When set to 0, hash reads are not skipped. */ + ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1, + /* Hash reads are skipped. */ +} icp_qat_hw_comp_20_skip_hash_rd_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP + +/*****************************************************************************/ +/* SCB Unload Disable, located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS 26 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SCB_UNLOAD_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD = 0x0, + /* Unloads the LFCT and flushes the State Registers. */ + ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_NO_UNLOAD = 0x1, + /* Does not unload the LFCT, but flushes the State Registers. */ +} icp_qat_hw_comp_20_scb_unload_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD + +/*****************************************************************************/ +/* Disable token fusion, located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS 21 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible DISABLE_TOKEN_FUSION_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE = 0x0, + /* Enables token fusion. */ + ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_DISABLE = 0x1, + /* Disables token fusion. */ +} icp_qat_hw_comp_20_disable_token_fusion_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE + +/*****************************************************************************/ +/* SCB Mode Reset Mask (Set By FW) , located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SCB_MODE_RESET_MASK field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS = 0x0, + /* iLZ77 mode: Reset LFCT, OBC */ + ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS_AND_HISTORY = 0x1, + /* iLZ77 mode: Reset LFCT, OBC, HB, HT */ +} icp_qat_hw_comp_20_scb_mode_reset_mask_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS + +/*****************************************************************************/ +/* Lazy - For iLZ77 and Static DEFLATE, Lazy = 102h , located in upper + * 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS 9 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK 0x1ff +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL 258 + +/*****************************************************************************/ +/* Nice - For iLZ77 and Static DEFLATE, Nice = 103h , located in upper + * 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS 0 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK 0x1ff +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL 259 + +/*****************************************************************************/ +/* History Buffer Size (Set By the Driver/ Application), located in lower 32bit + */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible HBS_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, + /* 000b - 32KB */ + ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_64KB = 0x1, + /* 001b - 64KB */ +} icp_qat_hw_comp_20_hbs_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB + +/*****************************************************************************/ +/* Adaptive Block Drop (Set By FW if Dynamic), located in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS 13 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible ABD field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED = 0x0, + /* 0b - Feature enabled. */ + ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED = 0x1, + /* 1b - Feature disabled. */ +} icp_qat_hw_comp_20_abd_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED + +/*****************************************************************************/ +/* Literal+Length Limit Block Drop Block Drop, (Set By FW if Dynamic) , located + * in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS 12 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible LLLBD_CTRL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED = 0x0, + /* 0b - Feature enabled. */ + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED = 0x1, + /* 1b - Feature disabled. */ +} icp_qat_hw_comp_20_lllbd_ctrl_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED + +/*****************************************************************************/ +/* Search Depth (SD) (Set By Driver/Application), located in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK 0xf +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SEARCH_DEPTH field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 = 0x1, + /* 0001b - Level 1 (search depth = 2^1 = 2) */ + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6 = 0x3, + /* 0011b - Level 6 (search depth = 2^3 = 8) */ + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9 = 0x4, + /* 0100b - Level 9 (search depth = 2^4 = 16) */ +} icp_qat_hw_comp_20_search_depth_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 + +/*****************************************************************************/ +/* Compression Format (Set By Driver/Application. Also See CMD ID), located in + * lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS 5 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK 0x7 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible HW_COMP_FORMAT field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77 = 0x0, + /* 000 - iLZ77. (Must set Min_Match = 3 bytes and HB size = 32KB.) */ + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE = 0x1, + /* 001 - Static DEFLATE. (Must set Min_Match = 3 bytes and HB size = + 32KB.) */ +} icp_qat_hw_comp_20_hw_comp_format_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE + +/*****************************************************************************/ +/* Skip Hash Collision (Set By FW to default value), located in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SKIP_HASH_COLLISION field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW = 0x0, + /* When set to 0, hash collisions are allowed. */ + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1, + /* When set to 0, hash collisions are allowed. */ +} icp_qat_hw_comp_20_skip_hash_collision_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW + +/*****************************************************************************/ +/* Skip Hash Update (Set By FW to default value) , located in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SKIP_HASH_UPDATE field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW = 0x0, + /* 0 - hash updates are not skipped. */ + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1, + /* 1 - hash updates are skipped. */ +} icp_qat_hw_comp_20_skip_hash_update_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW + +/*****************************************************************************/ +/* 3-Byte Match Skip (Set By FW to default value), located in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS 1 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible BYTE_SKIP field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN = 0x0, + /* 0 - Use 3-byte token */ + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL = 0x1, + /* 0 - Use 3-byte literal */ +} icp_qat_hw_comp_20_byte_skip_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN + +/*****************************************************************************/ +/* Extended Delayed Match Mode enabled (Set By the Driver), located in lower + * 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS 0 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible EXTENDED_DELAY_MATCH_MODE field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED = 0x0, + /* 0 - EXTENDED_DELAY_MATCH_MODE disabled */ + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED = 0x1, + /* 1 - EXTENDED_DELAY_MATCH_MODE enabled */ +} icp_qat_hw_comp_20_extended_delay_match_mode_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED + +/*****************************************************************************/ +/* Speculative Decoder Disable (Set By the Driver/ Application), located in + * upper 32bit */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS 31 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible SPECULATIVE_DECODER_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE = 0x0, + /* 0b - Enabled */ + ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_DISABLE = 0x1, + /* 1b - Disabled */ +} icp_qat_hw_decomp_20_speculative_decoder_control_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE + +/*****************************************************************************/ +/* Mini CAM Disable (Set By the Driver/ Application), located in upper 32bit */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS 30 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible MINI_CAM_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE = 0x0, + /* 0b - Enabled */ + ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_DISABLE = 0x1, + /* 1b - Disabled */ +} icp_qat_hw_decomp_20_mini_cam_control_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE + +/*****************************************************************************/ +/* History Buffer Size (Set By the Driver/ Application), located in lower 32bit + */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible HBS_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, + /* 000b - 32KB */ +} icp_qat_hw_decomp_20_hbs_control_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB + +/*****************************************************************************/ +/* Decompression Format (Set By Driver/Application. Also See CMD ID), located in + * lower 32bit */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS 5 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK 0x7 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible HW_DECOMP_FORMAT field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE = 0x1, + /* 001 - Static DEFLATE. (Must set Min_Match = 3 bytes and HB size = + 32KB.) */ +} icp_qat_hw_decomp_20_hw_comp_format_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE + +#endif //_ICP_QAT_HW_20_COMP_DEFS_H diff --git a/sys/dev/qat/qat_api/include/icp_sal_user.h b/sys/dev/qat/qat_api/include/icp_sal_user.h index af34d5aa194a..6c46210f6a1f 100644 --- a/sys/dev/qat/qat_api/include/icp_sal_user.h +++ b/sys/dev/qat/qat_api/include/icp_sal_user.h @@ -868,4 +868,41 @@ CpaStatus icp_sal_dp_SymGetInflightRequests(CpaInstanceHandle instanceHandle, * *****************************************************************************/ CpaStatus icp_sal_AsymPerformOpNow(CpaInstanceHandle instanceHandle); + +/** + ***************************************************************************** + * @ingroup icp_sal_setForceAEADMACVerify + * Sets forceAEADMacVerify for particular instance to force HW MAC + * validation. + * + * @description + * By default HW MAC verification is set to CPA_TRUE - this utility + * function allows to change default behavior. + * + * @assumptions + * None + * @sideEffects + * None + * @blocking + * None + * @reentrant + * No + * @threadSafe + * No + * + * @param[in] instanceHandle Crypto API instance handle. + * @param[in] forceAEADMacVerify new value + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @pre + * None + * @post + * None + * @see + * None + * + *****************************************************************************/ +CpaStatus icp_sal_setForceAEADMACVerify(CpaInstanceHandle instanceHandle, + CpaBoolean forceAEADMacVerify); #endif diff --git a/sys/dev/qat/qat_api/include/icp_sal_versions.h b/sys/dev/qat/qat_api/include/icp_sal_versions.h index 0a013fe46b81..91719587445d 100644 --- a/sys/dev/qat/qat_api/include/icp_sal_versions.h +++ b/sys/dev/qat/qat_api/include/icp_sal_versions.h @@ -26,8 +26,8 @@ /**< Max length of hardware version string */ /* Part name and number of the accelerator device */ -#define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3 -#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 11 +#define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3 +#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 12 #define SAL_INFO2_DRIVER_SW_VERSION_PATCH_NUMBER 0 /** diff --git a/sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h b/sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h index 3733fec2c9ea..82df3bb00d19 100644 --- a/sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h +++ b/sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h @@ -97,7 +97,8 @@ typedef enum device_type_e { DEVICE_200XX, DEVICE_200XXVF, DEVICE_C4XXX, - DEVICE_C4XXXVF + DEVICE_C4XXXVF, + DEVICE_GEN4 } device_type_t; /* diff --git a/sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c b/sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c index 739cd026eedf..910854446442 100644 --- a/sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c +++ b/sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c @@ -372,13 +372,20 @@ void icp_adf_updateQueueTail(icp_comms_trans_handle trans_handle) { struct adf_etr_ring_data *ring = trans_handle; + struct adf_hw_csr_ops *csr_ops; ICP_CHECK_FOR_NULL_PARAM_VOID(ring); + ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank); + ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank->accel_dev); - WRITE_CSR_RING_TAIL(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, - ring->tail); + csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + + ICP_CHECK_FOR_NULL_PARAM_VOID(csr_ops); + + csr_ops->write_csr_ring_tail(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, + ring->tail); ring->csr_tail_offset = ring->tail; } diff --git a/sys/dev/qat/qat_api/qat_utils/include/qat_utils.h b/sys/dev/qat/qat_api/qat_utils/include/qat_utils.h index c30a19785980..a9874eb89215 100644 --- a/sys/dev/qat/qat_api/qat_utils/include/qat_utils.h +++ b/sys/dev/qat/qat_api/qat_utils/include/qat_utils.h @@ -848,4 +848,25 @@ CpaStatus qatUtilsAESEncrypt(uint8_t *key, uint32_t keyLenInBytes, uint8_t *in, uint8_t *out); + +/** + * @ingroup QatUtils + * + * @brief Converts AES forward key to reverse key + * + * @param key - pointer to symetric key. + * keyLenInBytes - key length + * out - pointer to output buffer for reversed key + * The in and out buffers need to be at least AES block size long + * as defined in rfc3686 (16 bytes) + * + * @li Reentrant: yes + * @li IRQ safe: yes + * + * @return - CPA_STATUS_SUCCESS/CPA_STATUS_FAIL + * + */ +CpaStatus qatUtilsAESKeyExpansionForward(uint8_t *key, + uint32_t keyLenInBytes, + uint32_t *out); #endif diff --git a/sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c b/sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c index 3be2b8c362a0..6026faa52fa5 100644 --- a/sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c +++ b/sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c @@ -3,6 +3,10 @@ /* $FreeBSD$ */ #include "qat_utils.h" +#define AES_128_KEY_LEN_BYTES 16 +#define AES_192_KEY_LEN_BYTES 24 +#define AES_256_KEY_LEN_BYTES 32 + CpaStatus qatUtilsHashMD5(uint8_t *in, uint8_t *out) { @@ -150,3 +154,38 @@ qatUtilsAESEncrypt(uint8_t *key, return CPA_STATUS_SUCCESS; } + +CpaStatus +qatUtilsAESKeyExpansionForward(uint8_t *key, + uint32_t keyLenInBytes, + uint32_t *out) +{ + rijndael_ctx ctx; + uint32_t i = 0, j = 0; + uint32_t lw_per_round = 4; + int32_t lw_left_to_copy = keyLenInBytes / lw_per_round; + uint32_t *key_pointer = NULL; + + /* Error check for wrong input key len */ + if (AES_128_KEY_LEN_BYTES != keyLenInBytes && + AES_192_KEY_LEN_BYTES != keyLenInBytes && + AES_256_KEY_LEN_BYTES != keyLenInBytes) { + return CPA_STATUS_INVALID_PARAM; + } + + rijndael_set_key(&ctx, key, keyLenInBytes << BYTE_TO_BITS_SHIFT); + + /* Pointer to the last round of expanded key. */ + key_pointer = &ctx.ek[lw_per_round * ctx.Nr]; + + while (lw_left_to_copy > 0) { + for (i = 0; i < MIN(lw_left_to_copy, lw_per_round); i++, j++) { + out[j] = __builtin_bswap32(key_pointer[i]); + } + + lw_left_to_copy -= lw_per_round; + key_pointer -= lw_left_to_copy; + } + + return CPA_STATUS_SUCCESS; +} diff --git a/sys/dev/qat/qat_common/adf_accel_engine.c b/sys/dev/qat/qat_common/adf_accel_engine.c index 3b41ab5764e8..d430605be522 100644 --- a/sys/dev/qat/qat_common/adf_accel_engine.c +++ b/sys/dev/qat/qat_common/adf_accel_engine.c @@ -99,11 +99,24 @@ adf_ae_fw_load(struct adf_accel_dev *accel_dev) */ if (hw_device->get_obj_name && hw_device->get_obj_cfg_ae_mask) { unsigned long service_mask = hw_device->service_mask; + enum adf_accel_unit_services service_type = + ADF_ACCEL_SERVICE_NULL; - if (hw_device->service_mask && - !(test_bit(i, &service_mask))) + if (hw_device->get_service_type) + service_type = + hw_device->get_service_type(accel_dev, i); + else + service_type = BIT(i); + + if (service_mask && !(service_mask & service_type)) continue; - obj_name = hw_device->get_obj_name(accel_dev, BIT(i)); + + obj_name = + hw_device->get_obj_name(accel_dev, service_type); + cfg_ae_mask = + hw_device->get_obj_cfg_ae_mask(accel_dev, + service_type); + if (!obj_name) { device_printf( GET_DEV(accel_dev), @@ -111,10 +124,8 @@ adf_ae_fw_load(struct adf_accel_dev *accel_dev) BIT(i)); goto out_err; } - if (!hw_device->get_obj_cfg_ae_mask(accel_dev, BIT(i))) + if (!cfg_ae_mask) continue; - cfg_ae_mask = - hw_device->get_obj_cfg_ae_mask(accel_dev, BIT(i)); if (qat_uclo_set_cfg_ae_mask(loader_data->fw_loader, cfg_ae_mask)) { device_printf(GET_DEV(accel_dev), @@ -172,17 +183,12 @@ adf_ae_start(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + uint32_t ae_ctr; if (!hw_data->fw_name) return 0; - for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { - if (hw_data->ae_mask & (1 << ae)) { - qat_hal_start(loader_data->fw_loader, ae, 0xFF); - ae_ctr++; - } - } + ae_ctr = qat_hal_start(loader_data->fw_loader); device_printf(GET_DEV(accel_dev), "qat_dev%d started %d acceleration engines\n", accel_dev->accel_id, diff --git a/sys/dev/qat/qat_common/adf_cfg_bundle.c b/sys/dev/qat/qat_common/adf_cfg_bundle.c index edb2ef942417..a7faf6c05730 100644 --- a/sys/dev/qat/qat_common/adf_cfg_bundle.c +++ b/sys/dev/qat/qat_common/adf_cfg_bundle.c @@ -157,16 +157,23 @@ adf_cfg_init_and_insert_inst(struct adf_cfg_bundle *bundle, { struct adf_cfg_instance *cfg_instance = NULL; int ring_pair_index = 0; + int ring_index = 0; int i = 0; u8 serv_type; - int num_req_rings = bundle->num_of_rings / 2; - int num_rings_per_srv = num_req_rings / ADF_CFG_NUM_SERVICES; + int num_rings_per_srv = 0; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map; /* init the bundle with instance information */ - for (ring_pair_index = 0; ring_pair_index < ADF_CFG_NUM_SERVICES; + for (ring_pair_index = 0; ring_pair_index < bundle->max_cfg_svc_num; ring_pair_index++) { - serv_type = GET_SRV_TYPE(ring_to_svc_map, ring_pair_index); + adf_get_ring_svc_map_data(hw_data, + bundle->number, + ring_pair_index, + &serv_type, + &ring_index, + &num_rings_per_srv); + for (i = 0; i < num_rings_per_srv; i++) { cfg_instance = malloc(sizeof(*cfg_instance), M_QAT, @@ -219,8 +226,9 @@ adf_cfg_bundle_init(struct adf_cfg_bundle *bundle, { int i = 0; + bundle->number = bank_num; /* init ring to service mapping for this bundle */ - adf_cfg_init_ring2serv_mapping(accel_dev, bundle); + adf_cfg_init_ring2serv_mapping(accel_dev, bundle, device); /* init the bundle with instance information */ adf_cfg_init_and_insert_inst(bundle, device, bank_num, accel_dev); @@ -229,7 +237,6 @@ adf_cfg_bundle_init(struct adf_cfg_bundle *bundle, bundle->type = FREE; bundle->polling_mode = -1; bundle->section_index = 0; - bundle->number = bank_num; bundle->sections = malloc(sizeof(char *) * bundle->max_section, M_QAT, @@ -262,18 +269,25 @@ adf_cfg_bundle_clear(struct adf_cfg_bundle *bundle, } static void -adf_cfg_assign_serv_to_rings(struct adf_cfg_bundle *bundle, u16 ring_to_svc_map) +adf_cfg_assign_serv_to_rings(struct adf_hw_device_data *hw_data, + struct adf_cfg_bundle *bundle, + struct adf_cfg_device *device) { int ring_pair_index = 0; int ring_index = 0; u8 serv_type = 0; int num_req_rings = bundle->num_of_rings / 2; - int num_rings_per_srv = num_req_rings / ADF_CFG_NUM_SERVICES; + int num_rings_per_srv = 0; - for (ring_pair_index = 0; ring_pair_index < ADF_CFG_NUM_SERVICES; + for (ring_pair_index = 0; ring_pair_index < bundle->max_cfg_svc_num; ring_pair_index++) { - serv_type = GET_SRV_TYPE(ring_to_svc_map, ring_pair_index); - ring_index = num_rings_per_srv * ring_pair_index; + adf_get_ring_svc_map_data(hw_data, + bundle->number, + ring_pair_index, + &serv_type, + &ring_index, + &num_rings_per_srv); + switch (serv_type) { case CRYPTO: ASSIGN_SERV_TO_RINGS(bundle, @@ -283,7 +297,7 @@ adf_cfg_assign_serv_to_rings(struct adf_cfg_bundle *bundle, u16 ring_to_svc_map) num_rings_per_srv); ring_pair_index++; ring_index = num_rings_per_srv * ring_pair_index; - if (ring_pair_index == ADF_CFG_NUM_SERVICES) + if (ring_pair_index == bundle->max_cfg_svc_num) break; ASSIGN_SERV_TO_RINGS(bundle, ring_index, @@ -324,7 +338,7 @@ adf_cfg_assign_serv_to_rings(struct adf_cfg_bundle *bundle, u16 ring_to_svc_map) /* unknown service type */ pr_err("Unknown service type %d, mask 0x%x.\n", serv_type, - ring_to_svc_map); + hw_data->ring_to_svc_map); } } @@ -333,13 +347,18 @@ adf_cfg_assign_serv_to_rings(struct adf_cfg_bundle *bundle, u16 ring_to_svc_map) void adf_cfg_init_ring2serv_mapping(struct adf_accel_dev *accel_dev, - struct adf_cfg_bundle *bundle) + struct adf_cfg_bundle *bundle, + struct adf_cfg_device *device) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct adf_cfg_ring *ring_in_bundle; int ring_num = 0; bundle->num_of_rings = hw_data->num_rings_per_bank; + if (hw_data->num_rings_per_bank >= (2 * ADF_CFG_NUM_SERVICES)) + bundle->max_cfg_svc_num = ADF_CFG_NUM_SERVICES; + else + bundle->max_cfg_svc_num = 1; bundle->rings = malloc(bundle->num_of_rings * sizeof(struct adf_cfg_ring *), @@ -356,7 +375,7 @@ adf_cfg_init_ring2serv_mapping(struct adf_accel_dev *accel_dev, bundle->rings[ring_num] = ring_in_bundle; } - adf_cfg_assign_serv_to_rings(bundle, hw_data->ring_to_svc_map); + adf_cfg_assign_serv_to_rings(hw_data, bundle, device); return; } diff --git a/sys/dev/qat/qat_common/adf_cfg_bundle.h b/sys/dev/qat/qat_common/adf_cfg_bundle.h index 50ad7b007ef7..b9ec5e7a0af5 100644 --- a/sys/dev/qat/qat_common/adf_cfg_bundle.h +++ b/sys/dev/qat/qat_common/adf_cfg_bundle.h @@ -49,7 +49,29 @@ void adf_cfg_bundle_clear(struct adf_cfg_bundle *bundle, struct adf_accel_dev *accel_dev); void adf_cfg_init_ring2serv_mapping(struct adf_accel_dev *accel_dev, - struct adf_cfg_bundle *bundle); + struct adf_cfg_bundle *bundle, + struct adf_cfg_device *device); int adf_cfg_rel_ring2serv_mapping(struct adf_cfg_bundle *bundle); + +static inline void +adf_get_ring_svc_map_data(struct adf_hw_device_data *hw_data, + int bundle_num, + int ring_pair_index, + u8 *serv_type, + int *ring_index, + int *num_rings_per_srv) +{ + if (hw_data->get_ring_svc_map_data) + return hw_data->get_ring_svc_map_data(ring_pair_index, + hw_data->ring_to_svc_map, + serv_type, + ring_index, + num_rings_per_srv, + bundle_num); + *serv_type = GET_SRV_TYPE(hw_data->ring_to_svc_map, ring_pair_index); + *num_rings_per_srv = + hw_data->num_rings_per_bank / (2 * ADF_CFG_NUM_SERVICES); + *ring_index = (*num_rings_per_srv) * ring_pair_index; +} #endif diff --git a/sys/dev/qat/qat_common/adf_cfg_device.c b/sys/dev/qat/qat_common/adf_cfg_device.c index ecd8e1599eeb..9e59c038f2f3 100644 --- a/sys/dev/qat/qat_common/adf_cfg_device.c +++ b/sys/dev/qat/qat_common/adf_cfg_device.c @@ -249,7 +249,6 @@ adf_cfg_get_ring_pairs(struct adf_cfg_device *device, int i = 0; int ret = EFAULT; struct adf_cfg_instance *free_inst = NULL; - struct adf_cfg_bundle *first_free_bundle = NULL; enum adf_cfg_bundle_type free_bundle_type; int first_user_bundle = 0; @@ -304,29 +303,25 @@ adf_cfg_get_ring_pairs(struct adf_cfg_device *device, return ret; - } else if (!first_free_bundle && - adf_cfg_is_free(device->bundles[i])) { - first_free_bundle = device->bundles[i]; } } + for (i = 0; i < device->bundle_num; i++) { + if (adf_cfg_is_free(device->bundles[i])) { + free_inst = adf_cfg_get_free_instance( + device, + device->bundles[i], + inst, + process_name); + if (!free_inst) + continue; - if (first_free_bundle) { - free_inst = adf_cfg_get_free_instance(device, - first_free_bundle, - inst, - process_name); - - if (!free_inst) + ret = adf_cfg_get_ring_pairs_from_bundle( + device->bundles[i], + inst, + process_name, + free_inst); return ret; - - ret = adf_cfg_get_ring_pairs_from_bundle( - first_free_bundle, inst, process_name, free_inst); - - if (free_bundle_type == KERNEL) { - device->max_kernel_bundle_nr = - first_free_bundle->number; } - return ret; } } pr_err("Don't have enough rings for instance %s in process %s\n", diff --git a/sys/dev/qat/qat_common/adf_cfg_section.c b/sys/dev/qat/qat_common/adf_cfg_section.c index efb07784c57a..7cadd32bfc9c 100644 --- a/sys/dev/qat/qat_common/adf_cfg_section.c +++ b/sys/dev/qat/qat_common/adf_cfg_section.c @@ -286,10 +286,16 @@ adf_cfg_add_asym_inst_info(struct adf_accel_dev *accel_dev, key = malloc(ADF_CFG_MAX_KEY_LEN_IN_BYTES, M_QAT, M_WAITOK | M_ZERO); - snprintf(key, - ADF_CFG_MAX_KEY_LEN_IN_BYTES, - ADF_CY_BANK_NUM_FORMAT, - inst_index); + if (adf_cy_inst_cross_banks(accel_dev)) + snprintf(key, + ADF_CFG_MAX_KEY_LEN_IN_BYTES, + ADF_CY_ASYM_BANK_NUM_FORMAT, + inst_index); + else + snprintf(key, + ADF_CFG_MAX_KEY_LEN_IN_BYTES, + ADF_CY_BANK_NUM_FORMAT, + inst_index); bank_number = asym_inst->bundle; adf_cfg_add_key_value_param( accel_dev, derived_sec, key, (void *)&bank_number, ADF_DEC); @@ -337,10 +343,17 @@ adf_cfg_add_sym_inst_info(struct adf_accel_dev *accel_dev, key = malloc(ADF_CFG_MAX_KEY_LEN_IN_BYTES, M_QAT, M_WAITOK | M_ZERO); - snprintf(key, - ADF_CFG_MAX_KEY_LEN_IN_BYTES, - ADF_CY_BANK_NUM_FORMAT, - inst_index); + if (adf_cy_inst_cross_banks(accel_dev)) + snprintf(key, + ADF_CFG_MAX_KEY_LEN_IN_BYTES, + ADF_CY_SYM_BANK_NUM_FORMAT, + inst_index); + else + snprintf(key, + ADF_CFG_MAX_KEY_LEN_IN_BYTES, + ADF_CY_BANK_NUM_FORMAT, + inst_index); + bank_number = sym_inst->bundle; adf_cfg_add_key_value_param( accel_dev, derived_sec, key, (void *)&bank_number, ADF_DEC); diff --git a/sys/dev/qat/qat_common/adf_freebsd_admin.c b/sys/dev/qat/qat_common/adf_freebsd_admin.c index 2ad919887944..403fbcec3835 100644 --- a/sys/dev/qat/qat_common/adf_freebsd_admin.c +++ b/sys/dev/qat/qat_common/adf_freebsd_admin.c @@ -27,10 +27,7 @@ #define ADF_CONST_TABLE_VERSION (1) /* Admin Messages Registers */ -#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 -#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_MAILBOX_STRIDE 0x1000 #define ADF_ADMINMSG_LEN 32 #define FREEBSD_ALLIGNMENT_SIZE 64 #define ADF_INIT_CONFIG_SIZE 1024 @@ -146,7 +143,7 @@ adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, hw_data->get_admin_info(&admin_csrs_info); int offset = ae * ADF_ADMINMSG_LEN * 2; int mb_offset = - ae * ADF_DH895XCC_MAILBOX_STRIDE + admin_csrs_info.mailbox_offset; + ae * ADF_MAILBOX_STRIDE + admin_csrs_info.mailbox_offset; int times, received; struct icp_qat_fw_init_admin_req *request = in; @@ -294,7 +291,7 @@ adf_set_fw_constants(struct adf_accel_dev *accel_dev) struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; + u32 ae_mask = hw_device->admin_ae_mask; explicit_bzero(&req, sizeof(req)); req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; diff --git a/sys/dev/qat/qat_common/adf_freebsd_transport_debug.c b/sys/dev/qat/qat_common/adf_freebsd_transport_debug.c index b8e2bf2ca336..2a597601ffb6 100644 --- a/sys/dev/qat/qat_common/adf_freebsd_transport_debug.c +++ b/sys/dev/qat/qat_common/adf_freebsd_transport_debug.c @@ -20,6 +20,7 @@ static int adf_ring_show(SYSCTL_HANDLER_ARGS) { struct adf_etr_ring_data *ring = arg1; struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); struct resource *csr = ring->bank->csr_addr; struct sbuf sb; int error, word; @@ -29,13 +30,13 @@ static int adf_ring_show(SYSCTL_HANDLER_ARGS) { int head, tail, empty; - head = READ_CSR_RING_HEAD(csr, - bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, - bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, + bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, + bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); sbuf_cat(&sb, "\n------- Ring configuration -------\n"); sbuf_printf(&sb, @@ -119,6 +120,7 @@ static int adf_bank_show(SYSCTL_HANDLER_ARGS) { struct adf_etr_bank_data *bank; struct adf_accel_dev *accel_dev = NULL; + struct adf_hw_csr_ops *csr_ops = NULL; struct adf_hw_device_data *hw_data = NULL; u8 num_rings_per_bank = 0; struct sbuf sb; @@ -127,6 +129,7 @@ static int adf_bank_show(SYSCTL_HANDLER_ARGS) sbuf_new_for_sysctl(&sb, NULL, 128, req); bank = arg1; accel_dev = bank->accel_dev; + csr_ops = GET_CSR_OPS(bank->accel_dev); hw_data = accel_dev->hw_device; num_rings_per_bank = hw_data->num_rings_per_bank; sbuf_printf(&sb, @@ -140,13 +143,13 @@ static int adf_bank_show(SYSCTL_HANDLER_ARGS) if (!(bank->ring_mask & 1 << ring_id)) continue; - head = READ_CSR_RING_HEAD(csr, - bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, - bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, + bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, + bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); sbuf_printf(&sb, "ring num %02d, head %04x, tail %04x, empty: %d\n", diff --git a/sys/dev/qat/qat_common/adf_gen2_hw_data.c b/sys/dev/qat/qat_common/adf_gen2_hw_data.c new file mode 100644 index 000000000000..d3babf8800ba --- /dev/null +++ b/sys/dev/qat/qat_common/adf_gen2_hw_data.c @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2021 Intel Corporation */ +/* $FreeBSD$ */ +#include "adf_gen2_hw_data.h" +#include "icp_qat_hw.h" + +static u64 +build_csr_ring_base_addr(bus_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 +read_csr_ring_head(struct resource *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void +write_csr_ring_head(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 +read_csr_ring_tail(struct resource *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void +write_csr_ring_tail(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 +read_csr_e_stat(struct resource *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void +write_csr_ring_config(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void +write_csr_ring_base(struct resource *csr_base_addr, + u32 bank, + u32 ring, + bus_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void +write_csr_int_flag(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void +write_csr_int_srcsel(struct resource *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void +write_csr_int_col_en(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void +write_csr_int_col_ctl(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void +write_csr_int_flag_and_col(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static u32 +read_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank) +{ + return READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank); +} + +static void +write_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void +adf_gen2_init_hw_csr_info(struct adf_hw_csr_info *csr_info) +{ + struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops; + + csr_info->arb_enable_mask = 0xFF; + + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->read_csr_ring_srv_arb_en = read_csr_ring_srv_arb_en; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_info); diff --git a/sys/dev/qat/qat_common/adf_gen4_hw_data.c b/sys/dev/qat/qat_common/adf_gen4_hw_data.c new file mode 100644 index 000000000000..aae54898afb1 --- /dev/null +++ b/sys/dev/qat/qat_common/adf_gen4_hw_data.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2021 Intel Corporation */ +/* $FreeBSD$ */ +#include "adf_accel_devices.h" +#include "adf_gen4_hw_data.h" + +static u64 +build_csr_ring_base_addr(bus_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 +read_csr_ring_head(struct resource *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void +write_csr_ring_head(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 +read_csr_ring_tail(struct resource *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void +write_csr_ring_tail(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 +read_csr_e_stat(struct resource *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void +write_csr_ring_config(struct resource *csr_base_addr, + u32 bank, + u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void +write_csr_ring_base(struct resource *csr_base_addr, + u32 bank, + u32 ring, + bus_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void +write_csr_int_flag(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void +write_csr_int_srcsel(struct resource *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void +write_csr_int_col_en(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void +write_csr_int_col_ctl(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void +write_csr_int_flag_and_col(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static u32 +read_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank) +{ + return READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank); +} + +static void +write_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void +adf_gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info) +{ + struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops; + + csr_info->arb_enable_mask = 0x1; + + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->read_csr_ring_srv_arb_en = read_csr_ring_srv_arb_en; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_info); + +static inline void +adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, u32 *lower) +{ + *lower = lower_32_bits(value); + *upper = upper_32_bits(value); +} + +int +adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; + u32 ssm_wdt_pke_high = 0; + u32 ssm_wdt_pke_low = 0; + u32 ssm_wdt_high = 0; + u32 ssm_wdt_low = 0; + struct resource *pmisc_addr; + struct adf_bar *pmisc; + int pmisc_id; + + pmisc_id = hw_data->get_misc_bar_id(hw_data); + pmisc = &GET_BARS(accel_dev)[pmisc_id]; + pmisc_addr = pmisc->virt_addr; + + /* Convert 64bit WDT timer value into 32bit values for + * mmio write to 32bit CSRs. + */ + adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low); + adf_gen4_unpack_ssm_wdtimer(timer_val_pke, + &ssm_wdt_pke_high, + &ssm_wdt_pke_low); + + /* Enable WDT for sym and dc */ + ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low); + ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high); + /* Enable WDT for pke */ + ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low); + ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); diff --git a/sys/dev/qat/qat_common/adf_heartbeat.c b/sys/dev/qat/qat_common/adf_heartbeat.c index cf56ca98af33..1feaa37d48bf 100644 --- a/sys/dev/qat/qat_common/adf_heartbeat.c +++ b/sys/dev/qat/qat_common/adf_heartbeat.c @@ -68,10 +68,14 @@ adf_get_hb_timer(struct adf_accel_dev *accel_dev, unsigned int *value) unsigned int timer_val = ADF_CFG_HB_DEFAULT_VALUE; u32 clk_per_sec = 0; - if (!hw_data->get_ae_clock) + /* HB clock may be different than AE clock */ + if (hw_data->get_hb_clock) { + clk_per_sec = (u32)hw_data->get_hb_clock(hw_data); + } else if (hw_data->get_ae_clock) { + clk_per_sec = (u32)hw_data->get_ae_clock(hw_data); + } else { return EINVAL; - - clk_per_sec = (u32)hw_data->get_ae_clock(hw_data); + } /* Get Heartbeat Timer value from the configuration */ if (!adf_cfg_get_param_value(accel_dev, @@ -99,24 +103,19 @@ adf_get_hb_timer(struct adf_accel_dev *accel_dev, unsigned int *value) return 0; } -struct adf_hb_count { - u16 ae_thread[ADF_NUM_HB_CNT_PER_AE]; -}; - int adf_get_heartbeat_status(struct adf_accel_dev *accel_dev) { + struct icp_qat_fw_init_admin_hb_cnt *live_s, *last_s, *curr_s; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct icp_qat_fw_init_admin_hb_stats *live_s = - (struct icp_qat_fw_init_admin_hb_stats *) - accel_dev->admin->virt_hb_addr; const size_t max_aes = hw_device->get_num_aes(hw_device); + const size_t hb_ctrs = hw_device->heartbeat_ctr_num; const size_t stats_size = - max_aes * sizeof(struct icp_qat_fw_init_admin_hb_stats); + max_aes * hb_ctrs * sizeof(struct icp_qat_fw_init_admin_hb_cnt); int ret = 0; size_t ae, thr; + u16 *count_s; unsigned long ae_mask = 0; - int num_threads_per_ae = ADF_NUM_HB_CNT_PER_AE; /* * Memory layout of Heartbeat @@ -127,13 +126,19 @@ adf_get_heartbeat_status(struct adf_accel_dev *accel_dev) * \_______________/\_______________/\________/ * ^ ^ ^ * | | | - * | | max_aes * sizeof(adf_hb_count) - * | max_aes * sizeof(icp_qat_fw_init_admin_hb_stats) - * max_aes * sizeof(icp_qat_fw_init_admin_hb_stats) + * | | max_aes * hb_ctrs * + * | | sizeof(u16) + * | | + * | max_aes * hb_ctrs * + * | sizeof(icp_qat_fw_init_admin_hb_cnt) + * | + * max_aes * hb_ctrs * + * sizeof(icp_qat_fw_init_admin_hb_cnt) */ - struct icp_qat_fw_init_admin_hb_stats *curr_s; - struct icp_qat_fw_init_admin_hb_stats *last_s = live_s + max_aes; - struct adf_hb_count *count = (struct adf_hb_count *)(last_s + max_aes); + live_s = (struct icp_qat_fw_init_admin_hb_cnt *) + accel_dev->admin->virt_hb_addr; + last_s = live_s + (max_aes * hb_ctrs); + count_s = (u16 *)(last_s + (max_aes * hb_ctrs)); curr_s = malloc(stats_size, M_QAT, M_WAITOK | M_ZERO); @@ -142,23 +147,25 @@ adf_get_heartbeat_status(struct adf_accel_dev *accel_dev) for_each_set_bit(ae, &ae_mask, max_aes) { - for (thr = 0; thr < num_threads_per_ae; ++thr) { - struct icp_qat_fw_init_admin_hb_cnt *curr = - &curr_s[ae].stats[thr]; - struct icp_qat_fw_init_admin_hb_cnt *prev = - &last_s[ae].stats[thr]; - u16 req = curr->req_heartbeat_cnt; - u16 resp = curr->resp_heartbeat_cnt; - u16 last = prev->resp_heartbeat_cnt; + struct icp_qat_fw_init_admin_hb_cnt *curr = + curr_s + ae * hb_ctrs; + struct icp_qat_fw_init_admin_hb_cnt *prev = + last_s + ae * hb_ctrs; + u16 *count = count_s + ae * hb_ctrs; + + for (thr = 0; thr < hb_ctrs; ++thr) { + u16 req = curr[thr].req_heartbeat_cnt; + u16 resp = curr[thr].resp_heartbeat_cnt; + u16 last = prev[thr].resp_heartbeat_cnt; if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) { - u16 retry = ++count[ae].ae_thread[thr]; + u16 retry = ++count[thr]; if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) ret = EIO; } else { - count[ae].ae_thread[thr] = 0; + count[thr] = 0; } } } diff --git a/sys/dev/qat/qat_common/adf_hw_arbiter.c b/sys/dev/qat/qat_common/adf_hw_arbiter.c index 586b871f2dfd..e89a5f6ce68c 100644 --- a/sys/dev/qat/qat_common/adf_hw_arbiter.c +++ b/sys/dev/qat/qat_common/adf_hw_arbiter.c @@ -97,14 +97,30 @@ adf_init_gen2_arb(struct adf_accel_dev *accel_dev) void adf_update_ring_arb(struct adf_etr_ring_data *ring) { - WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, - ring->bank->bank_number, - ring->bank->ring_mask & 0xFF); + int shift; + u32 arben, arben_tx, arben_rx, arb_mask; + struct adf_accel_dev *accel_dev = ring->bank->accel_dev; + struct adf_hw_csr_info *csr_info = &accel_dev->hw_device->csr_info; + struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops; + + arb_mask = csr_info->arb_enable_mask; + shift = hweight32(arb_mask); + + arben_tx = ring->bank->ring_mask & arb_mask; + arben_rx = (ring->bank->ring_mask >> shift) & arb_mask; + arben = arben_tx & arben_rx; + csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr, + ring->bank->bank_number, + arben); } void -adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask) +adf_enable_ring_arb(struct adf_accel_dev *accel_dev, + void *csr_addr, + unsigned int bank_nr, + unsigned int mask) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); struct resource *csr = csr_addr; u32 arbenable; @@ -112,16 +128,20 @@ adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask) return; mutex_lock(&csr_arb_lock); - arbenable = READ_CSR_ARB_RINGSRVARBEN(csr, bank_nr); + arbenable = csr_ops->read_csr_ring_srv_arb_en(csr, bank_nr); arbenable |= mask & 0xFF; - WRITE_CSR_ARB_RINGSRVARBEN(csr, bank_nr, arbenable); + csr_ops->write_csr_ring_srv_arb_en(csr, bank_nr, arbenable); mutex_unlock(&csr_arb_lock); } void -adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask) +adf_disable_ring_arb(struct adf_accel_dev *accel_dev, + void *csr_addr, + unsigned int bank_nr, + unsigned int mask) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); struct resource *csr = csr_addr; u32 arbenable; @@ -129,15 +149,16 @@ adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask) return; mutex_lock(&csr_arb_lock); - arbenable = READ_CSR_ARB_RINGSRVARBEN(csr, bank_nr); + arbenable = csr_ops->read_csr_ring_srv_arb_en(csr, bank_nr); arbenable &= ~mask & 0xFF; - WRITE_CSR_ARB_RINGSRVARBEN(csr, bank_nr, arbenable); + csr_ops->write_csr_ring_srv_arb_en(csr, bank_nr, arbenable); mutex_unlock(&csr_arb_lock); } void adf_exit_arb(struct adf_accel_dev *accel_dev) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct arb_info info; struct resource *csr; @@ -166,12 +187,13 @@ adf_exit_arb(struct adf_accel_dev *accel_dev) /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); + csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } void adf_disable_arb(struct adf_accel_dev *accel_dev) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); struct resource *csr; unsigned int i; @@ -182,5 +204,5 @@ adf_disable_arb(struct adf_accel_dev *accel_dev) /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); + csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } diff --git a/sys/dev/qat/qat_common/adf_init.c b/sys/dev/qat/qat_common/adf_init.c index 2e5f77d22ea6..0fb8618b1f32 100644 --- a/sys/dev/qat/qat_common/adf_init.c +++ b/sys/dev/qat/qat_common/adf_init.c @@ -213,7 +213,7 @@ adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) unsigned int mask; u32 clk_per_sec = hw_data->get_clock_speed(hw_data); u32 timer_val = ADF_WDT_TIMER_SYM_COMP_MS * (clk_per_sec / 1000); - u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; + u32 timer_val_pke = ADF_GEN2_SSM_WDT_PKE_DEFAULT_VALUE; char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; /* Get Watch Dog Timer for CySym+Comp from the configuration */ @@ -289,6 +289,12 @@ adf_dev_init(struct adf_accel_dev *accel_dev) return EFAULT; } + if (hw_data->init_device && hw_data->init_device(accel_dev)) { + device_printf(GET_DEV(accel_dev), + "Failed to initialize device\n"); + return EFAULT; + } + if (hw_data->init_accel_units && hw_data->init_accel_units(accel_dev)) { device_printf(GET_DEV(accel_dev), "Failed initialize accel_units\n"); @@ -343,7 +349,8 @@ adf_dev_init(struct adf_accel_dev *accel_dev) hw_data->enable_error_correction(accel_dev); - if (hw_data->enable_vf2pf_comms(accel_dev)) { + if (hw_data->enable_vf2pf_comms && + hw_data->enable_vf2pf_comms(accel_dev)) { device_printf(GET_DEV(accel_dev), "QAT: Failed to enable vf2pf comms\n"); return EFAULT; diff --git a/sys/dev/qat/qat_common/adf_isr.c b/sys/dev/qat/qat_common/adf_isr.c index 31e439ee60e6..1b6f232f2c4d 100644 --- a/sys/dev/qat/qat_common/adf_isr.c +++ b/sys/dev/qat/qat_common/adf_isr.c @@ -38,6 +38,9 @@ adf_enable_msix(struct adf_accel_dev *accel_dev) int num_vectors = 0; u_int *vectors; + if (hw_data->set_msix_rttable) + hw_data->set_msix_rttable(accel_dev); + /* If SR-IOV is disabled, add entries for each bank */ if (!accel_dev->u1.pf.vf_info) { msix_num_entries += hw_data->num_banks; @@ -90,8 +93,11 @@ adf_msix_isr_bundle(void *bank_ptr) { struct adf_etr_bank_data *bank = bank_ptr; struct adf_etr_data *priv_data = bank->accel_dev->transport; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, + bank->bank_number, + 0); adf_response_handler((uintptr_t)&priv_data->banks[bank->bank_number]); return; } diff --git a/sys/dev/qat/qat_common/adf_transport.c b/sys/dev/qat/qat_common/adf_transport.c index e896f1d70e08..a608cb1c217c 100644 --- a/sys/dev/qat/qat_common/adf_transport.c +++ b/sys/dev/qat/qat_common/adf_transport.c @@ -73,27 +73,36 @@ adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + mtx_lock(&bank->lock); bank->irq_mask |= (1 << ring); mtx_unlock(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); - WRITE_CSR_INT_COL_CTL(bank->csr_addr, - bank->bank_number, - bank->irq_coalesc_timer); + csr_ops->write_csr_int_col_en(bank->csr_addr, + bank->bank_number, + bank->irq_mask); + csr_ops->write_csr_int_col_ctl(bank->csr_addr, + bank->bank_number, + bank->irq_coalesc_timer); } static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + mtx_lock(&bank->lock); bank->irq_mask &= ~(1 << ring); mtx_unlock(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); + csr_ops->write_csr_int_col_en(bank->csr_addr, + bank->bank_number, + bank->irq_mask); } int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 msg_size = 0; if (atomic_add_return(1, ring->inflights) > ring->max_inflights) { @@ -110,10 +119,10 @@ adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) ring->tail = adf_modulo(ring->tail + msg_size, ADF_RING_SIZE_MODULO(ring->ring_size)); - WRITE_CSR_RING_TAIL(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, - ring->tail); + csr_ops->write_csr_ring_tail(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, + ring->tail); ring->csr_tail_offset = ring->tail; mtx_unlock(&ring->lock); return 0; @@ -122,6 +131,7 @@ adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) int adf_handle_response(struct adf_etr_ring_data *ring, u32 quota) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 msg_counter = 0; u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); @@ -139,10 +149,10 @@ adf_handle_response(struct adf_etr_ring_data *ring, u32 quota) msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); } if (msg_counter > 0) - WRITE_CSR_RING_HEAD(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, - ring->head); + csr_ops->write_csr_ring_head(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, + ring->head); return msg_counter; } @@ -154,6 +164,7 @@ adf_poll_bank(u32 accel_id, u32 bank_num, u32 quota) struct adf_etr_data *trans_data; struct adf_etr_bank_data *bank; struct adf_etr_ring_data *ring; + struct adf_hw_csr_ops *csr_ops; u32 rings_not_empty; u32 ring_num; u32 resp_total = 0; @@ -168,12 +179,14 @@ adf_poll_bank(u32 accel_id, u32 bank_num, u32 quota) return EINVAL; } + csr_ops = GET_CSR_OPS(accel_dev); trans_data = accel_dev->transport; bank = &trans_data->banks[bank_num]; mtx_lock(&bank->lock); /* Read the ring status CSR to determine which rings are empty. */ - rings_not_empty = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); + rings_not_empty = + csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number); /* Complement to find which rings have data to be processed. */ rings_not_empty = (~rings_not_empty) & bank->ring_mask; @@ -265,25 +278,27 @@ adf_poll_all_banks(u32 accel_id, u32 quota) static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, - ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, + ring_config); } static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size, ADF_RING_NEAR_WATERMARK_512, ADF_RING_NEAR_WATERMARK_0); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, - ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, + ring_config); } static int @@ -292,6 +307,7 @@ adf_init_ring(struct adf_etr_ring_data *ring) struct adf_etr_bank_data *bank = ring->bank; struct adf_accel_dev *accel_dev = bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u64 ring_base; u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); @@ -323,11 +339,12 @@ adf_init_ring(struct adf_etr_ring_data *ring) else adf_configure_rx_ring(ring); - ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); - WRITE_CSR_RING_BASE(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, - ring_base); + ring_base = + csr_ops->build_csr_ring_base_addr(ring->dma_addr, ring->ring_size); + csr_ops->write_csr_ring_base(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, + ring_base); mtx_init(&ring->lock, "adf bank", NULL, MTX_DEF); return 0; } @@ -443,19 +460,20 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) { struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Disable interrupts for the given ring */ adf_disable_ring_irq(bank, ring->ring_number); /* Clear PCI config space */ - WRITE_CSR_RING_CONFIG(bank->csr_addr, - bank->bank_number, - ring->ring_number, - 0); - WRITE_CSR_RING_BASE(bank->csr_addr, - bank->bank_number, - ring->ring_number, - 0); + csr_ops->write_csr_ring_config(bank->csr_addr, + bank->bank_number, + ring->ring_number, + 0); + csr_ops->write_csr_ring_base(bank->csr_addr, + bank->bank_number, + ring->ring_number, + 0); adf_ring_debugfs_rm(ring); adf_unreserve_ring(bank, ring->ring_number); /* Disable HW arbitration for the given ring */ @@ -468,10 +486,12 @@ adf_ring_response_handler(struct adf_etr_bank_data *bank) { struct adf_accel_dev *accel_dev = bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u8 num_rings_per_bank = hw_data->num_rings_per_bank; u32 empty_rings, i; - empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); + empty_rings = + csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number); empty_rings = ~empty_rings & bank->irq_mask; for (i = 0; i < num_rings_per_bank; ++i) { @@ -484,12 +504,13 @@ void adf_response_handler(uintptr_t bank_addr) { struct adf_etr_bank_data *bank = (void *)bank_addr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Handle all the responses and re-enable IRQs */ adf_ring_response_handler(bank); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, - bank->bank_number, - bank->irq_mask); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, + bank->bank_number, + bank->irq_mask); } static inline int @@ -548,10 +569,12 @@ adf_init_bank(struct adf_accel_dev *accel_dev, struct resource *csr_addr) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; u8 num_rings_per_bank = hw_data->num_rings_per_bank; + u32 irq_mask = BIT(num_rings_per_bank) - 1; u32 size = 0; explicit_bzero(bank, sizeof(*bank)); @@ -580,8 +603,8 @@ adf_init_bank(struct adf_accel_dev *accel_dev, bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; for (i = 0; i < num_rings_per_bank; i++) { - WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); - WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); ring = &bank->rings[i]; if (hw_data->tx_rings_mask & (1 << i)) { ring->inflights = @@ -605,8 +628,8 @@ adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } - WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK); - WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); + csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); + csr_ops->write_csr_int_srcsel(csr_addr, bank_num); return 0; err: for (i = 0; i < num_rings_per_bank; i++) { diff --git a/sys/dev/qat/qat_common/qat_hal.c b/sys/dev/qat/qat_common/qat_hal.c index 0d711f5997f8..2da0e1141d79 100644 --- a/sys/dev/qat/qat_common/qat_hal.c +++ b/sys/dev/qat/qat_common/qat_hal.c @@ -75,6 +75,29 @@ static const uint64_t inst[] = { 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, }; +static const uint64_t inst_CPM2X[] = { + 0x0F0000C0000ull, 0x0D802C00011ull, 0x0F0000C0001ull, 0x0FC066C0001ull, + 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F000500300ull, + 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0A0580C0000ull, + 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 0x0A0584C0000ull, + 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 0x0A0588C0000ull, + 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 0x0A058CC0000ull, + 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 0x0A05C0C0000ull, + 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 0x0A05C4C0000ull, + 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 0x0A05C8C0000ull, + 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 0x0A05CCC0000ull, + 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 0x0A0400C0000ull, + 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 0x0A0402C0000ull, + 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 0x0A0404C0000ull, + 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 0x0A0406C0000ull, + 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 0x0A0408C0000ull, + 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 0x0A040AC0000ull, + 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 0x0A040CC0000ull, + 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 0x0A040EC0000ull, + 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 0x0D81341C010ull, + 0x0E000000001ull, 0x0E000010000ull, +}; + void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, unsigned char ae, @@ -206,6 +229,11 @@ qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, { unsigned int csr, new_csr; + if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + pr_err("QAT: No next neigh for CPM2X\n"); + return EINVAL; + } + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); csr &= IGNORE_W1C_MASK; @@ -339,6 +367,21 @@ qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num) return reg_addr; } +static u32 +qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle *handle) +{ + u32 tg = 0, ae; + u32 valid_ae_mask = 0; + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (handle->hal_handle->ae_mask & (1 << ae)) { + tg = ae / 4; + valid_ae_mask |= (1 << (tg * 2)); + } + } + return valid_ae_mask; +} + void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) { @@ -353,15 +396,26 @@ qat_hal_reset(struct icp_qat_fw_loader_handle *handle) ae_reset_csr[1] = ICP_RESET_CPP1; if (handle->hal_handle->ae_mask > 0xffff) ++cpp_num; + } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + ae_reset_csr[0] = ICP_RESET_CPP0; } else { ae_reset_csr[0] = ICP_RESET; } for (i = 0; i < cpp_num; i++) { if (i == 0) { - valid_ae_mask = handle->hal_handle->ae_mask & 0xFFFF; - valid_slice_mask = - handle->hal_handle->slice_mask & 0x3F; + if (IS_QAT_GEN4( + pci_get_device(GET_DEV(handle->accel_dev)))) { + valid_ae_mask = + qat_hal_get_ae_mask_gen4(handle); + valid_slice_mask = + handle->hal_handle->slice_mask; + } else { + valid_ae_mask = + handle->hal_handle->ae_mask & 0xFFFF; + valid_slice_mask = + handle->hal_handle->slice_mask & 0x3F; + } } else { valid_ae_mask = (handle->hal_handle->ae_mask >> AES_PER_CPP) & @@ -509,7 +563,7 @@ qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) unsigned long ae_mask = handle->hal_handle->ae_mask; misc_ctl_csr = - (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) ? + (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) ? MISC_CONTROL_C4XXX : MISC_CONTROL; /* stop the timestamp timers */ @@ -586,6 +640,9 @@ qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) clk_csr[1] = ICP_GLOBAL_CLK_ENABLE_CPP1; if (handle->hal_handle->ae_mask > 0xffff) ++cpp_num; + } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + ae_reset_csr[0] = ICP_RESET_CPP0; + clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0; } else { ae_reset_csr[0] = ICP_RESET; clk_csr[0] = ICP_GLOBAL_CLK_ENABLE; @@ -593,9 +650,18 @@ qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) for (i = 0; i < cpp_num; i++) { if (i == 0) { - valid_ae_mask = handle->hal_handle->ae_mask & 0xFFFF; - valid_slice_mask = - handle->hal_handle->slice_mask & 0x3F; + if (IS_QAT_GEN4( + pci_get_device(GET_DEV(handle->accel_dev)))) { + valid_ae_mask = + qat_hal_get_ae_mask_gen4(handle); + valid_slice_mask = + handle->hal_handle->slice_mask; + } else { + valid_ae_mask = + handle->hal_handle->ae_mask & 0xFFFF; + valid_slice_mask = + handle->hal_handle->slice_mask & 0x3F; + } } else { valid_ae_mask = (handle->hal_handle->ae_mask >> AES_PER_CPP) & @@ -714,7 +780,24 @@ qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, const uint64_t *uword) { unsigned int ustore_addr; - unsigned int i; + unsigned int i, ae_in_group; + + if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + ae_in_group = ae / 4 * 4; + + for (i = 0; i < AE_TG_NUM_CPM2X; i++) { + if (ae_in_group + i == ae) + continue; + if (ae_in_group + i >= handle->hal_handle->ae_max_num) + break; + if (qat_hal_check_ae_active(handle, ae_in_group + i)) { + pr_err( + "ae%d in T_group is active, cannot write to ustore!\n", + ae_in_group + i); + return; + } + } + } qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); uaddr |= UA_ECS; @@ -826,10 +909,25 @@ qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val); csr_val &= IGNORE_W1C_MASK; - csr_val |= CE_NN_MODE; + if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + csr_val |= CE_NN_MODE; + } qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); - qat_hal_wr_uwords( - handle, ae, 0, ARRAY_SIZE(inst), (const uint64_t *)inst); + + if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + if (ae % 4 == 0) + qat_hal_wr_uwords(handle, + ae, + 0, + ARRAY_SIZE(inst_CPM2X), + (const uint64_t *)inst_CPM2X); + } else { + qat_hal_wr_uwords(handle, + ae, + 0, + ARRAY_SIZE(inst), + (const uint64_t *)inst); + } qat_hal_wr_indr_csr(handle, ae, ctx_mask, @@ -971,6 +1069,7 @@ qat_hal_init(struct adf_accel_dev *accel_dev) malloc(sizeof(*handle->hal_handle), M_QAT, M_WAITOK | M_ZERO); handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; + handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; handle->cfg_ae_mask = 0xFFFFFFFF; /* create AE objects */ @@ -1038,17 +1137,23 @@ qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) free(handle, M_QAT); } -void -qat_hal_start(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, - unsigned int ctx_mask) +int +qat_hal_start(struct icp_qat_fw_loader_handle *handle) { + unsigned char ae = 0; int retry = 0; unsigned int fcu_sts = 0; unsigned int fcu_ctl_csr, fcu_sts_csr; + unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 ae_ctr = 0; if (handle->fw_auth) { - if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) + { + ae_ctr++; + } + if (IS_QAT_GEN3_OR_GEN4( + pci_get_device(GET_DEV(handle->accel_dev)))) { fcu_ctl_csr = FCU_CONTROL_C4XXX; fcu_sts_csr = FCU_STATUS_C4XXX; @@ -1061,17 +1166,27 @@ qat_hal_start(struct icp_qat_fw_loader_handle *handle, pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) - return; + return ae_ctr; } while (retry++ < FW_AUTH_MAX_RETRY); pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae, fcu_sts); + return 0; } else { - qat_hal_put_wakeup_event(handle, - ae, - (~ctx_mask) & ICP_QAT_UCLO_AE_ALL_CTX, - 0x10000); - qat_hal_enable_ctx(handle, ae, ctx_mask); + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) + { + qat_hal_put_wakeup_event(handle, + ae, + 0, + IS_QAT_GEN4( + pci_get_device(GET_DEV( + handle->accel_dev))) ? + 0x80000000 : + 0x10000); + qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); + ae_ctr++; + } + return ae_ctr; } } @@ -1193,7 +1308,7 @@ qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &ind_lm_addr_byte0); qat_hal_rd_indr_csr( handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &ind_lm_addr_byte1); - if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { + if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { qat_hal_rd_indr_csr( handle, ae, ctx, LM_ADDR_2_INDIRECT, &ind_lm_addr2); qat_hal_rd_indr_csr( @@ -1286,7 +1401,7 @@ qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, (1 << ctx), INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); - if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { + if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { qat_hal_wr_indr_csr( handle, ae, (1 << ctx), LM_ADDR_2_INDIRECT, ind_lm_addr2); qat_hal_wr_indr_csr( @@ -1832,6 +1947,11 @@ qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, int stat = 0; unsigned char ctx; + if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + pr_err("QAT: No next neigh for CPM2X\n"); + return EINVAL; + } + if (ctx_mask == 0) return EINVAL; diff --git a/sys/dev/qat/qat_common/qat_uclo.c b/sys/dev/qat/qat_common/qat_uclo.c index 4232fb0ad095..12b5dafbf73f 100644 --- a/sys/dev/qat/qat_common/qat_uclo.c +++ b/sys/dev/qat/qat_common/qat_uclo.c @@ -46,9 +46,13 @@ qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, encap_image->img_ptr->ctx_assigned; ae_data->shareable_ustore = ICP_QAT_SHARED_USTORE_MODE(encap_image->img_ptr->ae_mode); - ae_data->eff_ustore_size = ae_data->shareable_ustore ? - (obj_handle->ustore_phy_size << 1) : - obj_handle->ustore_phy_size; + if (obj_handle->prod_type == ICP_QAT_AC_4XXX_A_DEV_TYPE) + ae_data->eff_ustore_size = obj_handle->ustore_phy_size; + else { + ae_data->eff_ustore_size = ae_data->shareable_ustore ? + (obj_handle->ustore_phy_size << 1) : + obj_handle->ustore_phy_size; + } } else { ae_slice->ctx_mask_assigned = 0; } @@ -324,9 +328,13 @@ qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae; + unsigned int lmem; - if (qat_uclo_fetch_initmem_ae( - handle, init_mem, ICP_QAT_UCLO_MAX_LMEM_REG, &ae)) + lmem = IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))) ? + ICP_QAT_UCLO_MAX_LMEM_REG_2X : + ICP_QAT_UCLO_MAX_LMEM_REG; + + if (qat_uclo_fetch_initmem_ae(handle, init_mem, lmem, &ae)) return EINVAL; if (qat_uclo_create_batch_init_list( handle, init_mem, ae, &obj_handle->lm_init_tab[ae])) @@ -411,6 +419,8 @@ qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, { unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned long ae_assigned = uof_image->ae_assigned; + const bool gen4 = + IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))); if (!test_bit(ae, &cfg_ae_mask)) continue; @@ -418,7 +428,8 @@ qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, if (!test_bit(ae, &ae_assigned)) continue; - if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1)) { + if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1) && + !gen4) { qat_hal_get_scs_neigh_ae(ae, &neigh_ae); if (test_bit(neigh_ae, &ae_assigned)) @@ -427,7 +438,7 @@ qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, ustore_size = obj_handle->ae_data[ae].eff_ustore_size; patt_pos = page->beg_addr_p + page->micro_words_num; - if (obj_handle->ae_data[ae].shareable_ustore) { + if (obj_handle->ae_data[ae].shareable_ustore && !gen4) { qat_hal_get_scs_neigh_ae(ae, &neigh_ae); if (init[ae] == 0 && page->beg_addr_p != 0) { qat_hal_wr_coalesce_uwords(handle, @@ -445,6 +456,9 @@ qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, init[ae] = 1; init[neigh_ae] = 1; } else { + if (gen4 && (ae % 4 != 0)) + continue; + qat_hal_wr_uwords(handle, (unsigned char)ae, 0, @@ -727,6 +741,61 @@ qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, return 0; } +static int +UcLo_checkTGroupList2X(struct icp_qat_fw_loader_handle *handle) +{ + int i; + unsigned int swAe = 0; + unsigned int ii, jj; + struct icp_qat_uclo_aedata *ae_data0, *ae_datax; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + + for (i = 0; i < obj_handle->uimage_num; i++) { + struct icp_qat_uof_image *image = + obj_handle->ae_uimage[i].img_ptr; + if (image->numpages > 1) { + pr_err( + "Only 1 page is allowed in a UOF for CPM2X; We found %d in %s\n", + image->numpages, + qat_uclo_get_string(&obj_handle->str_table, + image->img_name)); + return EINVAL; + } + } + + for (swAe = 0; + (swAe < obj_handle->ae_num) && (swAe < ICP_QAT_UCLO_MAX_AE); + swAe += AE_TG_NUM_CPM2X) { + if (!qat_hal_check_ae_active(handle, swAe)) { + continue; + } + + for (ii = swAe; ii < (swAe + AE_TG_NUM_CPM2X); ii++) { + ae_data0 = &obj_handle->ae_data[ii]; + if (ae_data0->slice_num != 1) // not assigned + continue; + + for (jj = ii + 1; jj < (swAe + AE_TG_NUM_CPM2X); jj++) { + ae_datax = &obj_handle->ae_data[jj]; + if (ae_datax->slice_num != 1) // not assigned + continue; + if (ae_data0->ae_slices[0] + .encap_image->img_ptr != + ae_datax->ae_slices[0] + .encap_image->img_ptr) { + pr_err("Only 1 list is allowed in a "); + pr_err("Tgroup for CPM2X;\n"); + pr_err("ME%d, %d is assigned", ii, jj); + pr_err(" different list files\n"); + return EINVAL; + } + } + } + } + + return 0; +} + static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) { @@ -752,6 +821,11 @@ qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) return EINVAL; } } + if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + if (UcLo_checkTGroupList2X(handle)) { + return EINVAL; + } + } if (!mflag) { pr_err("QAT: uimage uses AE not set"); return EINVAL; @@ -817,6 +891,9 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_200XX_DEV_TYPE; case ADF_C4XXX_PCI_DEVICE_ID: return ICP_QAT_AC_C4XXX_DEV_TYPE; + case ADF_4XXX_PCI_DEVICE_ID: + case ADF_401XX_PCI_DEVICE_ID: + return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", pci_get_device(GET_DEV(handle->accel_dev))); @@ -1001,11 +1078,13 @@ qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, ae_mode = (char)ICP_QAT_SHARED_USTORE_MODE(uof_image->ae_mode); qat_hal_set_ae_scs_mode(handle, ae, ae_mode); - nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return EFAULT; + if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return EFAULT; + } } ae_mode = (char)ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, ae_mode)) { @@ -1017,7 +1096,7 @@ qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); return EFAULT; } - if (obj_handle->prod_type == ICP_QAT_AC_C4XXX_DEV_TYPE) { + if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { ae_mode = (char)ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, ae_mode)) { pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); @@ -1168,12 +1247,14 @@ qat_uclo_map_suof_file_hdr(const struct icp_qat_fw_loader_handle *handle, } static void -qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, +qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr *suof_img_hdr, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; const struct icp_qat_simg_ae_mode *ae_mode; struct icp_qat_suof_objhdr *suof_objhdr; + unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); suof_img_hdr->simg_buf = (suof_handle->suof_buf + suof_chunk_hdr->offset + @@ -1187,9 +1268,10 @@ qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, suof_img_hdr->css_key = (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr)); suof_img_hdr->css_signature = suof_img_hdr->css_key + - ICP_QAT_CSS_FWSK_MODULUS_LEN + ICP_QAT_CSS_FWSK_EXPONENT_LEN; + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id); suof_img_hdr->css_simg = - suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN; + suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN(device_id); ae_mode = (const struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); suof_img_hdr->ae_mask = ae_mode->ae_mask; @@ -1277,7 +1359,8 @@ qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; - int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; + int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE, + aeMax_img = ICP_QAT_UCLO_MAX_AE; unsigned int i = 0; struct icp_qat_suof_img_hdr img_header; @@ -1305,7 +1388,7 @@ qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, } for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle->sobj_handle, + qat_uclo_map_simg(handle, &suof_img_hdr[i], &suof_chunk_hdr[1 + i]); ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]); @@ -1315,9 +1398,29 @@ qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, if ((suof_img_hdr[i].ae_mask & 0x1) != 0) ae0_img = i; } - qat_uclo_tail_img(suof_img_hdr, - ae0_img, - suof_handle->img_table.num_simgs); + + if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + qat_uclo_tail_img(suof_img_hdr, + ae0_img, + suof_handle->img_table.num_simgs); + } else { + if (suof_handle->img_table.num_simgs == 1) + return 0; + qat_uclo_tail_img(suof_img_hdr, + ae0_img, + suof_handle->img_table.num_simgs - 1); + for (i = 0; i < suof_handle->img_table.num_simgs; i++) { + if ((suof_img_hdr[i].ae_mask & + (0x1 << (handle->hal_handle->ae_max_num - 1))) != + 0) { + aeMax_img = i; + break; + } + } + qat_uclo_tail_img(suof_img_hdr, + aeMax_img, + suof_handle->img_table.num_simgs); + } return 0; } @@ -1335,7 +1438,7 @@ qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - sizeof(struct icp_qat_auth_chunk); - if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { + if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { fcu_ctl_csr = FCU_CONTROL_C4XXX; fcu_sts_csr = FCU_STATUS_C4XXX; fcu_dram_hi_csr = FCU_DRAM_ADDR_HI_C4XXX; @@ -1372,6 +1475,103 @@ qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, return EINVAL; } +static int +qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, int imgid) +{ + struct icp_qat_suof_handle *sobj_handle; + + if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) + return 0; + + sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; + if (handle->hal_handle->admin_ae_mask & + sobj_handle->img_table.simg_hdr[imgid].ae_mask) + return 0; + + return 1; +} + +static int +qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned int i = 0; + unsigned int fcuSts = 0, fcuAeBroadcastMask = 0; + unsigned int retry = 0; + unsigned int fcuStsCsr = 0; + unsigned int fcuCtlCsr = 0; + unsigned int loadedAes = 0; + unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); + + if (IS_QAT_GEN4(device_id)) { + fcuCtlCsr = FCU_CONTROL_4XXX; + fcuStsCsr = FCU_STATUS_4XXX; + } else { + pr_err("Uclo_BroadcastLoadFW only applicable for CPM20\n"); + return EINVAL; + } + + for (i = 0; i < ICP_QAT_UCLO_MAX_AE; i++) { + if (!test_bit(i, (unsigned long *)&handle->hal_handle->ae_mask)) + continue; + + if (qat_hal_check_ae_active(handle, (unsigned char)i)) { + pr_err( + "Uclo_BroadcastLoadFW error (invalid AE status)\n"); + return EINVAL; + } + + if ((desc->ae_mask >> i) & 0x1) { + fcuAeBroadcastMask |= 1 << i; + } + } + + if (fcuAeBroadcastMask) { + retry = 0; + SET_FCU_CSR(handle, + FCU_ME_BROADCAST_MASK_TYPE, + fcuAeBroadcastMask); + SET_FCU_CSR(handle, fcuCtlCsr, FCU_CTRL_CMD_LOAD); + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcuSts = GET_FCU_CSR(handle, fcuStsCsr); + + if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_FAIL) { + pr_err( + "Uclo_BroadcastLoadFW fail (fcu_status = 0x%x)\n", + fcuSts & FCU_AUTH_STS_MASK); + return EINVAL; + } else if ((fcuSts & FCU_AUTH_STS_MASK) == + FCU_STS_LOAD_DONE) { + if (IS_QAT_GEN4(device_id)) + loadedAes = + GET_FCU_CSR(handle, + FCU_AE_LOADED_4XXX); + else + loadedAes = + (fcuSts >> FCU_LOADED_AE_POS); + + if ((loadedAes & fcuAeBroadcastMask) == + fcuAeBroadcastMask) + break; + } else if ((fcuSts & FCU_AUTH_STS_MASK) == + FCU_STS_VERI_DONE) { + SET_FCU_CSR(handle, + fcuCtlCsr, + FCU_CTRL_CMD_LOAD); + } + } while (retry++ < FW_BROADCAST_MAX_RETRY); + if (retry > FW_BROADCAST_MAX_RETRY) { + pr_err( + "Uclo_BroadcastLoadFW fail(fcu_status = 0x%x),retry = %d\n", + fcuSts & FCU_AUTH_STS_MASK, + retry); + return EINVAL; + } + } + return 0; +} + static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc, @@ -1417,14 +1617,16 @@ qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_auth_chunk *auth_chunk; u64 virt_addr, bus_addr, virt_base; unsigned int length, simg_offset = sizeof(*auth_chunk); + unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); - if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + if (size > + (ICP_QAT_AE_IMG_OFFSET(device_id) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return EINVAL; } length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? - ICP_QAT_CSS_AE_SIMG_LEN + simg_offset : - size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset; + ICP_QAT_CSS_AE_SIMG_LEN(device_id) + simg_offset : + size + ICP_QAT_CSS_FWSK_PAD_LEN(device_id) + simg_offset; if (qat_uclo_simg_alloc(handle, img_desc, length)) { pr_err("QAT: error, allocate continuous dram fail\n"); return -ENOMEM; @@ -1451,42 +1653,43 @@ qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, memcpy((void *)(uintptr_t)virt_addr, (const void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)); /* padding */ - explicit_bzero((void *)(uintptr_t)(virt_addr + - ICP_QAT_CSS_FWSK_MODULUS_LEN), - ICP_QAT_CSS_FWSK_PAD_LEN); + explicit_bzero((void *)(uintptr_t)( + virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)), + ICP_QAT_CSS_FWSK_PAD_LEN(device_id)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_PAD_LEN), + memcpy((void *)(uintptr_t)(virt_addr + + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + + ICP_QAT_CSS_FWSK_PAD_LEN(device_id)), (const void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN), + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + - ICP_QAT_CSS_FWSK_PUB_LEN; - virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN; + ICP_QAT_CSS_FWSK_PUB_LEN(device_id); + virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(device_id); auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, (const void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN), - ICP_QAT_CSS_SIGNATURE_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id)), + ICP_QAT_CSS_SIGNATURE_LEN(device_id)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + - ICP_QAT_CSS_SIGNATURE_LEN; - virt_addr += ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(device_id); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(device_id); auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET; + auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(device_id); memcpy((void *)(uintptr_t)virt_addr, - (const void *)(image + ICP_QAT_AE_IMG_OFFSET), + (const void *)(image + ICP_QAT_AE_IMG_OFFSET(device_id)), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ @@ -1506,7 +1709,8 @@ qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; virt_addr += sizeof(struct icp_qat_css_hdr) + - ICP_QAT_CSS_FWSK_PUB_LEN + ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_FWSK_PUB_LEN(device_id) + + ICP_QAT_CSS_SIGNATURE_LEN(device_id); auth_desc->ae_mask = ((struct icp_qat_simg_ae_mode *)virt_addr)->ae_mask & handle->cfg_ae_mask; @@ -1528,7 +1732,7 @@ qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, unsigned int loaded_aes = FCU_LOADED_AE_POS; unsigned long ae_mask = handle->hal_handle->ae_mask; - if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { + if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { fcu_ctl_csr = FCU_CONTROL_C4XXX; fcu_sts_csr = FCU_STATUS_C4XXX; @@ -1549,14 +1753,19 @@ qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, } SET_FCU_CSR(handle, fcu_ctl_csr, - (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); + (FCU_CTRL_CMD_LOAD | + (IS_QAT_GEN4( + pci_get_device(GET_DEV(handle->accel_dev))) ? + (1 << FCU_CTRL_BROADCAST_POS) : + 0) | + (i << FCU_CTRL_AE_POS))); do { pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_DONE) { - loaded_aes = IS_QAT_GEN3(pci_get_device( + loaded_aes = IS_QAT_GEN3_OR_GEN4(pci_get_device( GET_DEV(handle->accel_dev))) ? GET_FCU_CSR(handle, FCU_AE_LOADED_C4XXX) : (fcu_sts >> FCU_LOADED_AE_POS); @@ -1606,6 +1815,16 @@ qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, qat_uclo_simg_free(handle, &img_desc); } else { + if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { + device_printf( + NULL, "QAT: PKE service is not allowed because "); + device_printf(NULL, "MMP fw will not be loaded for "); + device_printf(NULL, + "device 0x%x", + pci_get_device( + GET_DEV(handle->accel_dev))); + return status; + } if (pci_get_device(GET_DEV(handle->accel_dev)) == ADF_C3XXX_PCI_DEVICE_ID) { pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); @@ -2044,7 +2263,8 @@ qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, uw_relative_addr + i, fill_pat); - if (obj_handle->ae_data[ae].shareable_ustore) + if (obj_handle->ae_data[ae].shareable_ustore && + !IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) /* copy the buffer to ustore */ qat_hal_wr_coalesce_uwords(handle, (unsigned char)ae, @@ -2140,10 +2360,16 @@ qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) goto wr_err; - if (qat_uclo_load_fw(handle, desc)) - goto wr_err; + if (qat_uclo_is_broadcast(handle, i)) { + if (qat_uclo_broadcast_load_fw(handle, desc)) + goto wr_err; + } else { + if (qat_uclo_load_fw(handle, desc)) + goto wr_err; + } qat_uclo_simg_free(handle, &img_desc); } + return 0; wr_err: qat_uclo_simg_free(handle, &img_desc); diff --git a/sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c b/sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c index 22e1464cff12..5bfe1c7f40b3 100644 --- a/sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c +++ b/sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "adf_200xx_hw_data.h" #include "icp_qat_hw.h" #include "adf_heartbeat.h" @@ -495,6 +496,7 @@ adf_init_hw_data_200xx(struct adf_hw_device_data *hw_data) hw_data->get_errsou_offset = get_errsou_offset; hw_data->get_clock_speed = get_clock_speed; hw_data->get_sku = get_sku; + hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE; hw_data->fw_name = ADF_200XX_FW; hw_data->fw_mmp_name = ADF_200XX_MMP; hw_data->init_admin_comms = adf_init_admin_comms; @@ -532,6 +534,8 @@ adf_init_hw_data_200xx(struct adf_hw_device_data *hw_data) hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP; hw_data->pre_reset = adf_dev_pre_reset; hw_data->post_reset = adf_dev_post_reset; + + adf_gen2_init_hw_csr_info(&hw_data->csr_info); } void diff --git a/sys/dev/qat/qat_hw/qat_200xx/adf_drv.c b/sys/dev/qat/qat_hw/qat_200xx/adf_drv.c index 65626ac4b56f..75b9778ab84f 100644 --- a/sys/dev/qat/qat_hw/qat_200xx/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_200xx/adf_drv.c @@ -145,6 +145,7 @@ adf_attach(device_t dev) /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); + hw_data->admin_ae_mask = hw_data->ae_mask; accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c new file mode 100644 index 000000000000..1c8c9a2fda4c --- /dev/null +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c @@ -0,0 +1,973 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2022 Intel Corporation */ +/* $FreeBSD$ */ +#include +#include +#include +#include +#include +#include +#include +#include "adf_4xxx_hw_data.h" +#include "adf_heartbeat.h" +#include "icp_qat_fw_init_admin.h" +#include "icp_qat_hw.h" + +#define ADF_CONST_TABLE_SIZE 1024 + +struct adf_fw_config { + u32 ae_mask; + char *obj_name; +}; + +/* Accel unit information */ +static const struct adf_accel_unit adf_4xxx_au_a_ae[] = { + { 0x1, 0x1, 0xF, 0x1B, 4, ADF_ACCEL_SERVICE_NULL }, + { 0x2, 0x1, 0xF0, 0x6C0, 4, ADF_ACCEL_SERVICE_NULL }, + { 0x4, 0x1, 0x100, 0xF000, 1, ADF_ACCEL_ADMIN }, +}; + +/* Worker thread to service arbiter mappings */ +static u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555, + 0x5555555, 0x5555555, + 0xAAAAAAA, 0xAAAAAAA, + 0xAAAAAAA, 0xAAAAAAA, + 0x0 }; + +/* Masks representing ME thread-service mappings. + * Thread 7 carries out Admin work and is thus + * left out. + */ +static u8 default_active_thd_mask = 0x7F; +static u8 dc_me_active_thd_mask = 0x03; + +static u32 thrd_to_arb_map_gen[ADF_4XXX_MAX_ACCELENGINES] = { 0 }; + +#define ADF_4XXX_ASYM_SYM \ + (ASYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_DC \ + (COMP | COMP << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_SYM \ + (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_ASYM \ + (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + ASYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_ASYM_DC \ + (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_SYM_DC \ + (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_NA \ + (NA | NA << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ + NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ + NA << ADF_CFG_SERV_RING_PAIR_3_SHIFT) + +#define ADF_4XXX_DEFAULT_RING_TO_SRV_MAP ADF_4XXX_ASYM_SYM + +struct adf_enabled_services { + const char svcs_enabled[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + u16 rng_to_svc_msk; +}; + +static struct adf_enabled_services adf_4xxx_svcs[] = { + { "dc", ADF_4XXX_DC }, + { "sym", ADF_4XXX_SYM }, + { "asym", ADF_4XXX_ASYM }, + { "dc;asym", ADF_4XXX_ASYM_DC }, + { "asym;dc", ADF_4XXX_ASYM_DC }, + { "sym;dc", ADF_4XXX_SYM_DC }, + { "dc;sym", ADF_4XXX_SYM_DC }, + { "asym;sym", ADF_4XXX_ASYM_SYM }, + { "sym;asym", ADF_4XXX_ASYM_SYM }, +}; + +static struct adf_hw_device_class adf_4xxx_class = { + .name = ADF_4XXX_DEVICE_NAME, + .type = DEV_4XXX, + .instances = 0, +}; + +static u32 +get_accel_mask(struct adf_accel_dev *accel_dev) +{ + return ADF_4XXX_ACCELERATORS_MASK; +} + +static u32 +get_ae_mask(struct adf_accel_dev *accel_dev) +{ + u32 fusectl4 = accel_dev->hw_device->fuses; + + return ~fusectl4 & ADF_4XXX_ACCELENGINES_MASK; +} + +static int +get_ring_to_svc_map(struct adf_accel_dev *accel_dev, u16 *ring_to_svc_map) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + u32 i = 0; + + *ring_to_svc_map = 0; + /* Get the services enabled by user */ + snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); + if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) + return EFAULT; + + for (i = 0; i < ARRAY_SIZE(adf_4xxx_svcs); i++) { + if (!strncmp(val, + adf_4xxx_svcs[i].svcs_enabled, + ADF_CFG_MAX_KEY_LEN_IN_BYTES)) { + *ring_to_svc_map = adf_4xxx_svcs[i].rng_to_svc_msk; + return 0; + } + } + + device_printf(GET_DEV(accel_dev), + "Invalid services enabled: %s\n", + val); + return EFAULT; +} + +static u32 +get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_4XXX_MAX_ACCELERATORS; +} + +static u32 +get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} + +static u32 +get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_PMISC_BAR; +} + +static u32 +get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ETR_BAR; +} + +static u32 +get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_SRAM_BAR; +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void +set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + struct resource *csr; + int i; + + csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); +} + +static u32 +adf_4xxx_get_hw_cap(struct adf_accel_dev *accel_dev) +{ + device_t pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fusectl1; + u32 capabilities; + + /* Read accelerator capabilities mask */ + fusectl1 = pci_read_config(pdev, ADF_4XXX_FUSECTL1_OFFSET, 4); + capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_AES_V2 | ICP_ACCEL_CAPABILITIES_RL; + + if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + if (fusectl1 & ICP_ACCEL_4XXX_MASK_SMX_SLICE) { + capabilities &= ~ICP_ACCEL_CAPABILITIES_SM3; + capabilities &= ~ICP_ACCEL_CAPABILITIES_SM4; + } + return capabilities; +} + +static u32 +get_hb_clock(struct adf_hw_device_data *self) +{ + /* + * 4XXX uses KPT counter for HB + */ + return ADF_4XXX_KPT_COUNTER_FREQ; +} + +static u32 +get_ae_clock(struct adf_hw_device_data *self) +{ + /* + * Clock update interval is <16> ticks for qat_4xxx. + */ + return self->clock_frequency / 16; +} + +static int +measure_clock(struct adf_accel_dev *accel_dev) +{ + u32 frequency; + int ret = 0; + + ret = adf_dev_measure_clock(accel_dev, + &frequency, + ADF_4XXX_MIN_AE_FREQ, + ADF_4XXX_MAX_AE_FREQ); + if (ret) + return ret; + + accel_dev->hw_device->clock_frequency = frequency; + return 0; +} + +static int +adf_4xxx_configure_accel_units(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = { 0 }; + char val_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; + + if (adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC)) + goto err; + + snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); + snprintf(val_str, + sizeof(val_str), + ADF_CFG_ASYM ADF_SERVICES_SEPARATOR ADF_CFG_SYM); + + if (adf_cfg_add_key_value_param( + accel_dev, ADF_GENERAL_SEC, key, (void *)val_str, ADF_STR)) + goto err; + + return 0; +err: + device_printf(GET_DEV(accel_dev), "Failed to configure accel units\n"); + return EINVAL; +} + +static u32 +get_num_accel_units(struct adf_hw_device_data *self) +{ + return ADF_4XXX_MAX_ACCELUNITS; +} + +static void +get_accel_unit(struct adf_hw_device_data *self, + struct adf_accel_unit **accel_unit) +{ + memcpy(*accel_unit, adf_4xxx_au_a_ae, sizeof(adf_4xxx_au_a_ae)); +} + +static void +adf_exit_accel_unit_services(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->au_info) { + kfree(accel_dev->au_info->au); + accel_dev->au_info->au = NULL; + kfree(accel_dev->au_info); + accel_dev->au_info = NULL; + } +} + +static int +get_accel_unit_config(struct adf_accel_dev *accel_dev, + u8 *num_sym_au, + u8 *num_dc_au, + u8 *num_asym_au) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + u32 num_au = hw_data->get_num_accel_units(hw_data); + /* One AU will be allocated by default if a service enabled */ + u32 alloc_au = 1; + /* There's always one AU that is used for Admin AE */ + u32 service_mask = ADF_ACCEL_ADMIN; + char *token, *cur_str; + u32 disabled_caps = 0; + + /* Get the services enabled by user */ + snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); + if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) + return EFAULT; + cur_str = val; + token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); + while (token) { + if (!strncmp(token, ADF_CFG_SYM, strlen(ADF_CFG_SYM))) + service_mask |= ADF_ACCEL_CRYPTO; + if (!strncmp(token, ADF_CFG_ASYM, strlen(ADF_CFG_ASYM))) + service_mask |= ADF_ACCEL_ASYM; + + /* cy means both asym & crypto should be enabled + * Hardware resources allocation check will be done later + */ + if (!strncmp(token, ADF_CFG_CY, strlen(ADF_CFG_CY))) + service_mask |= ADF_ACCEL_ASYM | ADF_ACCEL_CRYPTO; + if (!strncmp(token, ADF_SERVICE_DC, strlen(ADF_SERVICE_DC))) + service_mask |= ADF_ACCEL_COMPRESSION; + + token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); + } + + /* Ensure the user won't enable more services than it can support */ + if (hweight32(service_mask) > num_au) { + device_printf(GET_DEV(accel_dev), + "Can't enable more services than "); + device_printf(GET_DEV(accel_dev), "%d!\n", num_au); + return EFAULT; + } else if (hweight32(service_mask) == 2) { + /* Due to limitation, besides AU for Admin AE + * only 2 more AUs can be allocated + */ + alloc_au = 2; + } + + if (service_mask & ADF_ACCEL_CRYPTO) + *num_sym_au = alloc_au; + if (service_mask & ADF_ACCEL_ASYM) + *num_asym_au = alloc_au; + if (service_mask & ADF_ACCEL_COMPRESSION) + *num_dc_au = alloc_au; + + /*update capability*/ + if (!*num_sym_au || !(service_mask & ADF_ACCEL_CRYPTO)) { + disabled_caps = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_AES_V2; + } + if (!*num_asym_au || !(service_mask & ADF_ACCEL_ASYM)) { + disabled_caps |= ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + } + if (!*num_dc_au || !(service_mask & ADF_ACCEL_COMPRESSION)) { + disabled_caps |= ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + accel_dev->hw_device->extended_dc_capabilities = 0; + } + accel_dev->hw_device->accel_capabilities_mask = + adf_4xxx_get_hw_cap(accel_dev) & ~disabled_caps; + + hw_data->service_mask = service_mask; + hw_data->service_to_load_mask = service_mask; + + return 0; +} + +static int +adf_init_accel_unit_services(struct adf_accel_dev *accel_dev) +{ + u8 num_sym_au = 0, num_dc_au = 0, num_asym_au = 0; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 num_au = hw_data->get_num_accel_units(hw_data); + u32 au_size = num_au * sizeof(struct adf_accel_unit); + u8 i; + + if (get_accel_unit_config( + accel_dev, &num_sym_au, &num_dc_au, &num_asym_au)) + return EFAULT; + + accel_dev->au_info = kzalloc(sizeof(*accel_dev->au_info), GFP_KERNEL); + if (!accel_dev->au_info) + return ENOMEM; + + accel_dev->au_info->au = kzalloc(au_size, GFP_KERNEL); + if (!accel_dev->au_info->au) { + kfree(accel_dev->au_info); + accel_dev->au_info = NULL; + return ENOMEM; + } + + accel_dev->au_info->num_cy_au = num_sym_au; + accel_dev->au_info->num_dc_au = num_dc_au; + accel_dev->au_info->num_asym_au = num_asym_au; + + get_accel_unit(hw_data, &accel_dev->au_info->au); + + /* Enable ASYM accel units */ + for (i = 0; i < num_au && num_asym_au > 0; i++) { + if (accel_dev->au_info->au[i].services == + ADF_ACCEL_SERVICE_NULL) { + accel_dev->au_info->au[i].services = ADF_ACCEL_ASYM; + num_asym_au--; + } + } + /* Enable SYM accel units */ + for (i = 0; i < num_au && num_sym_au > 0; i++) { + if (accel_dev->au_info->au[i].services == + ADF_ACCEL_SERVICE_NULL) { + accel_dev->au_info->au[i].services = ADF_ACCEL_CRYPTO; + num_sym_au--; + } + } + /* Enable compression accel units */ + for (i = 0; i < num_au && num_dc_au > 0; i++) { + if (accel_dev->au_info->au[i].services == + ADF_ACCEL_SERVICE_NULL) { + accel_dev->au_info->au[i].services = + ADF_ACCEL_COMPRESSION; + num_dc_au--; + } + } + accel_dev->au_info->dc_ae_msk |= + hw_data->get_obj_cfg_ae_mask(accel_dev, ADF_ACCEL_COMPRESSION); + + return 0; +} + +static int +adf_init_accel_units(struct adf_accel_dev *accel_dev) +{ + return adf_init_accel_unit_services(accel_dev); +} + +static void +adf_exit_accel_units(struct adf_accel_dev *accel_dev) +{ + /* reset the AU service */ + adf_exit_accel_unit_services(accel_dev); +} + +static const char * +get_obj_name(struct adf_accel_dev *accel_dev, + enum adf_accel_unit_services service) +{ + switch (service) { + case ADF_ACCEL_ASYM: + return ADF_4XXX_ASYM_OBJ; + case ADF_ACCEL_CRYPTO: + return ADF_4XXX_SYM_OBJ; + case ADF_ACCEL_COMPRESSION: + return ADF_4XXX_DC_OBJ; + case ADF_ACCEL_ADMIN: + return ADF_4XXX_ADMIN_OBJ; + default: + return NULL; + } +} + +static uint32_t +get_objs_num(struct adf_accel_dev *accel_dev) +{ + return ADF_4XXX_MAX_OBJ; +} + +static uint32_t +get_obj_cfg_ae_mask(struct adf_accel_dev *accel_dev, + enum adf_accel_unit_services service) +{ + u32 ae_mask = 0; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 num_au = hw_data->get_num_accel_units(hw_data); + struct adf_accel_unit *accel_unit = accel_dev->au_info->au; + u32 i = 0; + + if (service == ADF_ACCEL_SERVICE_NULL) + return 0; + + for (i = 0; i < num_au; i++) { + if (accel_unit[i].services == service) + ae_mask |= accel_unit[i].ae_mask; + } + + return ae_mask; +} + +static enum adf_accel_unit_services +adf_4xxx_get_service_type(struct adf_accel_dev *accel_dev, s32 obj_num) +{ + struct adf_accel_unit *accel_unit; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_au = hw_data->get_num_accel_units(hw_data); + int i; + + if (!hw_data->service_to_load_mask) + return ADF_ACCEL_SERVICE_NULL; + + if (accel_dev->au_info && accel_dev->au_info->au) + accel_unit = accel_dev->au_info->au; + else + return ADF_ACCEL_SERVICE_NULL; + + for (i = num_au - 2; i >= 0; i--) { + if (hw_data->service_to_load_mask & accel_unit[i].services) { + hw_data->service_to_load_mask &= + ~accel_unit[i].services; + return accel_unit[i].services; + } + } + + /* admin AE should be loaded last */ + if (hw_data->service_to_load_mask & accel_unit[num_au - 1].services) { + hw_data->service_to_load_mask &= + ~accel_unit[num_au - 1].services; + return accel_unit[num_au - 1].services; + } + + return ADF_ACCEL_SERVICE_NULL; +} + +static void +get_ring_svc_map_data(int ring_pair_index, + u16 ring_to_svc_map, + u8 *serv_type, + int *ring_index, + int *num_rings_per_srv, + int bundle_num) +{ + *serv_type = + GET_SRV_TYPE(ring_to_svc_map, bundle_num % ADF_CFG_NUM_SERVICES); + *ring_index = 0; + *num_rings_per_srv = ADF_4XXX_NUM_RINGS_PER_BANK / 2; +} + +static int +adf_get_dc_extcapabilities(struct adf_accel_dev *accel_dev, u32 *capabilities) +{ + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + u8 i; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_au = hw_data->get_num_accel_units(hw_data); + u32 first_dc_ae = 0; + + for (i = 0; i < num_au; i++) { + if (accel_dev->au_info->au[i].services & + ADF_ACCEL_COMPRESSION) { + first_dc_ae = accel_dev->au_info->au[i].ae_mask; + first_dc_ae &= ~(first_dc_ae - 1); + } + } + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; + + if (likely(first_dc_ae)) { + if (adf_send_admin(accel_dev, &req, &resp, first_dc_ae) || + resp.status) { + *capabilities = 0; + return EFAULT; + } + + *capabilities = resp.extended_features; + } + + return 0; +} + +static int +adf_get_fw_status(struct adf_accel_dev *accel_dev, + u8 *major, + u8 *minor, + u8 *patch) +{ + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + u32 ae_mask = 1; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_STATUS_GET; + + if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) + return EFAULT; + + *major = resp.version_major_num; + *minor = resp.version_minor_num; + *patch = resp.version_patch_num; + + return 0; +} + +static int +adf_4xxx_send_admin_init(struct adf_accel_dev *accel_dev) +{ + int ret = 0; + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 ae_mask = hw_data->ae_mask; + u32 admin_ae_mask = hw_data->admin_ae_mask; + u8 num_au = hw_data->get_num_accel_units(hw_data); + u8 i; + u32 dc_capabilities = 0; + + for (i = 0; i < num_au; i++) { + if (accel_dev->au_info->au[i].services == + ADF_ACCEL_SERVICE_NULL) + ae_mask &= ~accel_dev->au_info->au[i].ae_mask; + + if (accel_dev->au_info->au[i].services != ADF_ACCEL_ADMIN) + admin_ae_mask &= ~accel_dev->au_info->au[i].ae_mask; + } + + if (!accel_dev->admin) { + device_printf(GET_DEV(accel_dev), "adf_admin not available\n"); + return EFAULT; + } + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + + req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; + req.init_cfg_sz = ADF_CONST_TABLE_SIZE; + req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; + if (adf_send_admin(accel_dev, &req, &resp, admin_ae_mask)) { + device_printf(GET_DEV(accel_dev), + "Error sending constants config message\n"); + return EFAULT; + } + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_INIT_ME; + if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) { + device_printf(GET_DEV(accel_dev), + "Error sending init message\n"); + return EFAULT; + } + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; + req.init_cfg_ptr = accel_dev->admin->phy_hb_addr; + if (adf_get_hb_timer(accel_dev, &req.heartbeat_ticks)) + return EINVAL; + + if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) + device_printf(GET_DEV(accel_dev), + "Heartbeat is not supported\n"); + + ret = adf_get_dc_extcapabilities(accel_dev, &dc_capabilities); + if (unlikely(ret)) { + device_printf(GET_DEV(accel_dev), + "Could not get FW ext. capabilities\n"); + } + + accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; + + adf_get_fw_status(accel_dev, + &accel_dev->fw_versions.fw_version_major, + &accel_dev->fw_versions.fw_version_minor, + &accel_dev->fw_versions.fw_version_patch); + + device_printf(GET_DEV(accel_dev), + "FW version: %d.%d.%d\n", + accel_dev->fw_versions.fw_version_major, + accel_dev->fw_versions.fw_version_minor, + accel_dev->fw_versions.fw_version_patch); + + return ret; +} + +static enum dev_sku_info +get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static struct adf_accel_unit * +get_au_by_ae(struct adf_accel_dev *accel_dev, int ae_num) +{ + int i = 0; + struct adf_accel_unit *accel_unit = accel_dev->au_info->au; + + if (!accel_unit) + return NULL; + + for (i = 0; i < ADF_4XXX_MAX_ACCELUNITS; i++) + if (accel_unit[i].ae_mask & BIT(ae_num)) + return &accel_unit[i]; + + return NULL; +} + +static bool +check_accel_unit_service(enum adf_accel_unit_services au_srv, + enum adf_cfg_service_type ring_srv) +{ + if ((au_srv & ADF_ACCEL_SERVICE_NULL) && ring_srv == NA) + return true; + if ((au_srv & ADF_ACCEL_COMPRESSION) && ring_srv == COMP) + return true; + if ((au_srv & ADF_ACCEL_ASYM) && ring_srv == ASYM) + return true; + if ((au_srv & ADF_ACCEL_CRYPTO) && ring_srv == SYM) + return true; + + return false; +} + +static void +adf_4xxx_cfg_gen_dispatch_arbiter(struct adf_accel_dev *accel_dev, + u32 *thrd_to_arb_map_gen) +{ + struct adf_accel_unit *au = NULL; + int engine = 0; + int thread = 0; + int service; + u16 ena_srv_mask; + u16 service_type; + u32 service_mask; + unsigned long thd_srv_mask = default_active_thd_mask; + + ena_srv_mask = accel_dev->hw_device->ring_to_svc_map; + /* If ring_to_svc_map is not changed, return default arbiter value */ + if (ena_srv_mask == ADF_4XXX_DEFAULT_RING_TO_SRV_MAP) { + memcpy(thrd_to_arb_map_gen, + thrd_to_arb_map, + sizeof(thrd_to_arb_map_gen[0]) * + ADF_4XXX_MAX_ACCELENGINES); + return; + } + + for (engine = 0; engine < ADF_4XXX_MAX_ACCELENGINES - 1; engine++) { + thrd_to_arb_map_gen[engine] = 0; + service_mask = 0; + au = get_au_by_ae(accel_dev, engine); + if (!au) + continue; + + for (service = 0; service < ADF_CFG_MAX_SERVICES; service++) { + service_type = GET_SRV_TYPE(ena_srv_mask, service); + if (check_accel_unit_service(au->services, + service_type)) + service_mask |= BIT(service); + } + + if (au->services == ADF_ACCEL_COMPRESSION) + thd_srv_mask = dc_me_active_thd_mask; + else + thd_srv_mask = default_active_thd_mask; + + for_each_set_bit(thread, &thd_srv_mask, 8) + { + thrd_to_arb_map_gen[engine] |= + (service_mask << (ADF_CFG_MAX_SERVICES * thread)); + } + } +} + +static void +adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) +{ + int i; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + for (i = 1; i < ADF_4XXX_MAX_ACCELENGINES; i++) { + if (~hw_device->ae_mask & (1 << i)) + thrd_to_arb_map[i] = 0; + } + adf_4xxx_cfg_gen_dispatch_arbiter(accel_dev, thrd_to_arb_map_gen); + *arb_map_config = thrd_to_arb_map_gen; +} + +static void +get_arb_info(struct arb_info *arb_info) +{ + arb_info->wrk_cfg_offset = ADF_4XXX_ARB_CONFIG; + arb_info->arbiter_offset = ADF_4XXX_ARB_OFFSET; + arb_info->wrk_thd_2_srv_arb_map = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void +get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; +} + +static void +adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; + struct resource *csr = misc_bar->virt_addr; + + /* Enable all in errsou3 except VFLR notification on host */ + ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY); +} + +static void +adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + struct resource *addr; + + addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); +} + +static int +adf_init_device(struct adf_accel_dev *accel_dev) +{ + struct resource *addr; + u32 status; + u32 csr; + int ret; + + addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_4XXX_ERRMSK2); + csr |= ADF_4XXX_PM_SOU; + ADF_CSR_WR(addr, ADF_4XXX_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_4XXX_PM_INTERRUPT, ADF_4XXX_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + status = 0; + ret = read_poll_timeout(ADF_CSR_RD, + status, + status & ADF_4XXX_PM_INIT_STATE, + ADF_4XXX_PM_POLL_DELAY_US, + ADF_4XXX_PM_POLL_TIMEOUT_US, + true, + addr, + ADF_4XXX_PM_STATUS); + if (ret) + device_printf(GET_DEV(accel_dev), + "Failed to power up the device\n"); + + return ret; +} + +void +adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_4xxx_class; + hw_data->instance_id = adf_4xxx_class.instances++; + hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; + hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = adf_4xxx_get_hw_cap; + hw_data->clock_frequency = ADF_4XXX_AE_FREQ; + hw_data->get_sku = get_sku; + hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE; + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_4xxx_send_admin_init; + hw_data->init_arb = adf_init_gen2_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_enable_ints; + hw_data->init_device = adf_init_device; + hw_data->reset_device = adf_reset_flr; + hw_data->restore_device = adf_dev_restore; + hw_data->init_accel_units = adf_init_accel_units; + hw_data->exit_accel_units = adf_exit_accel_units; + hw_data->get_num_accel_units = get_num_accel_units; + hw_data->configure_accel_units = adf_4xxx_configure_accel_units; + hw_data->get_ring_to_svc_map = get_ring_to_svc_map; + hw_data->get_ring_svc_map_data = get_ring_svc_map_data; + hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; + hw_data->get_objs_num = get_objs_num; + hw_data->get_obj_name = get_obj_name; + hw_data->get_obj_cfg_ae_mask = get_obj_cfg_ae_mask; + hw_data->get_service_type = adf_4xxx_get_service_type; + hw_data->set_msix_rttable = set_msix_default_rttable; + hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; + hw_data->disable_iov = adf_disable_sriov; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->config_device = adf_config_device; + hw_data->set_asym_rings_mask = adf_cfg_set_asym_rings_mask; + hw_data->get_hb_clock = get_hb_clock; + hw_data->get_heartbeat_status = adf_get_heartbeat_status; + hw_data->get_ae_clock = get_ae_clock; + hw_data->measure_clock = measure_clock; + hw_data->query_storage_cap = 1; + + adf_gen4_init_hw_csr_info(&hw_data->csr_info); +} + +void +adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h new file mode 100644 index 000000000000..c3e9750e2b17 --- /dev/null +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007 - 2022 Intel Corporation */ +/* $FreeBSD$ */ +#ifndef ADF_4XXX_HW_DATA_H_ +#define ADF_4XXX_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_4XXX_SRAM_BAR 0 +#define ADF_4XXX_PMISC_BAR 1 +#define ADF_4XXX_ETR_BAR 2 +#define ADF_4XXX_RX_RINGS_OFFSET 1 +#define ADF_4XXX_TX_RINGS_MASK 0x1 +#define ADF_4XXX_MAX_ACCELERATORS 1 +#define ADF_4XXX_MAX_ACCELENGINES 9 +#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) + +/* 2 Accel units dedicated to services and */ +/* 1 Accel unit dedicated to Admin AE */ +#define ADF_4XXX_MAX_ACCELUNITS 3 + +/* Physical function fuses */ +#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) +#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) +#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) +#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) +#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) +#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) + +#define ADF_4XXX_ACCELERATORS_MASK (0x1) +#define ADF_4XXX_ACCELENGINES_MASK (0x1FF) +#define ADF_4XXX_ADMIN_AE_MASK (0x100) + +#define ADF_4XXX_ETR_MAX_BANKS 64 + +/* MSIX interrupt */ +#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) +#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) +#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) +#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i)*0x04)) + +/* Bank and ring configuration */ +#define ADF_4XXX_NUM_RINGS_PER_BANK 2 + +/* Error source registers */ +#define ADF_4XXX_ERRSOU0 (0x41A200) +#define ADF_4XXX_ERRSOU1 (0x41A204) +#define ADF_4XXX_ERRSOU2 (0x41A208) +#define ADF_4XXX_ERRSOU3 (0x41A20C) + +/* Error source mask registers */ +#define ADF_4XXX_ERRMSK0 (0x41A210) +#define ADF_4XXX_ERRMSK1 (0x41A214) +#define ADF_4XXX_ERRMSK2 (0x41A218) +#define ADF_4XXX_ERRMSK3 (0x41A21C) + +#define ADF_4XXX_VFLNOTIFY BIT(7) + +/* Arbiter configuration */ +#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_4XXX_ARB_OFFSET (0x0) +#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) + +/* Admin Interface Reg Offset */ +#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) +#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) +#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) + +/* Power management */ +#define ADF_4XXX_PM_POLL_DELAY_US 20 +#define ADF_4XXX_PM_POLL_TIMEOUT_US USEC_PER_SEC +#define ADF_4XXX_PM_STATUS (0x50A00C) +#define ADF_4XXX_PM_INTERRUPT (0x50A028) +#define ADF_4XXX_PM_DRV_ACTIVE BIT(20) +#define ADF_4XXX_PM_INIT_STATE BIT(21) +/* Power management source in ERRSOU2 and ERRMSK2 */ +#define ADF_4XXX_PM_SOU BIT(18) + +/* Firmware Binaries */ +#define ADF_4XXX_FW "qat_4xxx_fw" +#define ADF_4XXX_MMP "qat_4xxx_mmp_fw" +#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin" +#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" +#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" +#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" + +/* Only 3 types of images can be loaded including the admin image */ +#define ADF_4XXX_MAX_OBJ 3 + +#define ADF_4XXX_AE_FREQ (1000 * 1000000) +#define ADF_4XXX_KPT_COUNTER_FREQ (100 * 1000000) + +#define ADF_4XXX_MIN_AE_FREQ (9 * 1000000) +#define ADF_4XXX_MAX_AE_FREQ (1100 * 1000000) + +/* qat_4xxx fuse bits are different from old GENs, redefine them */ +enum icp_qat_4xxx_slice_mask { + ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), + ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6), +}; + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c b/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c new file mode 100644 index 000000000000..76dcf7b37dee --- /dev/null +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007 - 2022 Intel Corporation */ +/* $FreeBSD$ */ +#include "qat_freebsd.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" +#include "adf_accel_devices.h" +#include "adf_4xxx_hw_data.h" +#include "adf_gen4_hw_data.h" +#include "adf_fw_counters.h" +#include "adf_cfg_device.h" +#include +#include +#include +#include +#include +#include "adf_heartbeat_dbg.h" +#include "adf_cnvnr_freq_counters.h" + +static MALLOC_DEFINE(M_QAT_4XXX, "qat_4xxx", "qat_4xxx"); + +#define ADF_SYSTEM_DEVICE(device_id) \ + { \ + PCI_VENDOR_ID_INTEL, device_id \ + } + +static const struct pci_device_id adf_pci_tbl[] = + { ADF_SYSTEM_DEVICE(ADF_4XXX_PCI_DEVICE_ID), + ADF_SYSTEM_DEVICE(ADF_401XX_PCI_DEVICE_ID), + { + 0, + } }; + +static int +adf_probe(device_t dev) +{ + const struct pci_device_id *id; + + for (id = adf_pci_tbl; id->vendor != 0; id++) { + if (pci_get_vendor(dev) == id->vendor && + pci_get_device(dev) == id->device) { + device_set_desc(dev, + "Intel " ADF_4XXX_DEVICE_NAME + " QuickAssist"); + return BUS_PROBE_GENERIC; + } + } + return ENXIO; +} + +static void +adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + if (accel_dev->dma_tag) + bus_dma_tag_destroy(accel_dev->dma_tag); + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + bus_free_resource(accel_pci_dev->pci_dev, + SYS_RES_MEMORY, + bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (pci_get_device(accel_pci_dev->pci_dev)) { + case ADF_4XXX_PCI_DEVICE_ID: + case ADF_401XX_PCI_DEVICE_ID: + adf_clean_hw_data_4xxx(accel_dev->hw_device); + break; + default: + break; + } + free(accel_dev->hw_device, M_QAT_4XXX); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int +adf_attach(device_t dev) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + int ret, rid; + struct adf_cfg_device *cfg_dev = NULL; + + /* Set pci MaxPayLoad to 256. Implemented to avoid the issue of + * Pci-passthrough causing Maxpayload to be reset to 128 bytes + * when the device is reset. + */ + if (pci_get_max_payload(dev) != 256) + pci_set_max_payload(dev, 256); + + accel_dev = device_get_softc(dev); + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = dev; + + if (bus_get_domain(dev, &accel_pci_dev->node) != 0) + accel_pci_dev->node = 0; + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + device_printf(dev, "Failed to add new accelerator device.\n"); + return ENXIO; + } + + /* Allocate and configure device configuration structure */ + hw_data = malloc(sizeof(*hw_data), M_QAT_4XXX, M_WAITOK | M_ZERO); + + accel_dev->hw_device = hw_data; + adf_init_hw_data_4xxx(accel_dev->hw_device); + accel_pci_dev->revid = pci_get_revid(dev); + hw_data->fuses = pci_read_config(dev, ADF_4XXX_FUSECTL4_OFFSET, 4); + if (accel_pci_dev->revid == 0x00) { + device_printf(dev, "A0 stepping is not supported.\n"); + ret = ENODEV; + goto out_err; + } + + /* Get PPAERUCM values and store */ + ret = adf_aer_store_ppaerucm_reg(dev, hw_data); + if (ret) + goto out_err; + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); + hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); + + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + device_printf(dev, "No acceleration units found\n"); + ret = ENXIO; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + ret = adf_clock_debugfs_add(accel_dev); + if (ret) + goto out_err; + + pci_set_max_read_req(dev, 1024); + + ret = bus_dma_tag_create(bus_get_dma_tag(dev), + 1, + 0, + BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, + NULL, + NULL, + BUS_SPACE_MAXSIZE, + /* BUS_SPACE_UNRESTRICTED */ 1, + BUS_SPACE_MAXSIZE, + 0, + NULL, + NULL, + &accel_dev->dma_tag); + if (ret) + goto out_err; + + if (hw_data->get_accel_cap) { + hw_data->accel_capabilities_mask = + hw_data->get_accel_cap(accel_dev); + } + + /* Find and map all the device's BARS */ + i = 0; + for (bar_nr = 0; i < ADF_PCI_MAX_BARS && bar_nr < PCIR_MAX_BAR_0; + bar_nr++) { + struct adf_bar *bar; + + rid = PCIR_BAR(bar_nr); + if (bus_get_resource(dev, SYS_RES_MEMORY, rid, NULL, NULL) != 0) + continue; + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, + &rid, + RF_ACTIVE); + if (!bar->virt_addr) { + device_printf(dev, "Failed to map BAR %d\n", bar_nr); + ret = ENXIO; + goto out_err; + } + bar->base_addr = rman_get_start(bar->virt_addr); + bar->size = rman_get_size(bar->virt_addr); + } + pci_enable_busmaster(dev); + + if (!accel_dev->hw_device->config_device) { + ret = EFAULT; + goto out_err; + } + + ret = accel_dev->hw_device->config_device(accel_dev); + if (ret) + goto out_err; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_dev_stop; + + cfg_dev = accel_dev->cfg->dev; + adf_cfg_device_clear(cfg_dev, accel_dev); + free(cfg_dev, M_QAT); + accel_dev->cfg->dev = NULL; + return ret; +out_dev_stop: + adf_dev_stop(accel_dev); +out_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static int +adf_detach(device_t dev) +{ + struct adf_accel_dev *accel_dev = device_get_softc(dev); + + if (adf_dev_stop(accel_dev)) { + device_printf(dev, "Failed to stop QAT accel dev\n"); + return EBUSY; + } + + adf_dev_shutdown(accel_dev); + + adf_cleanup_accel(accel_dev); + + return 0; +} + +static device_method_t adf_methods[] = { DEVMETHOD(device_probe, adf_probe), + DEVMETHOD(device_attach, adf_attach), + DEVMETHOD(device_detach, adf_detach), + + DEVMETHOD_END }; + +static driver_t adf_driver = { "qat", + adf_methods, + sizeof(struct adf_accel_dev) }; + +DRIVER_MODULE_ORDERED(qat_4xxx, pci, adf_driver, NULL, NULL, SI_ORDER_THIRD); +MODULE_VERSION(qat_4xxx, 1); +MODULE_DEPEND(qat_4xxx, qat_common, 1, 1, 1); +MODULE_DEPEND(qat_4xxx, qat_api, 1, 1, 1); +MODULE_DEPEND(qat_4xxx, linuxkpi, 1, 1, 1); diff --git a/sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c b/sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c index 5f7fe3249353..a13683800c8e 100644 --- a/sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "adf_c3xxx_hw_data.h" #include "icp_qat_hw.h" #include "adf_heartbeat.h" @@ -368,6 +369,7 @@ adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_errsou_offset = get_errsou_offset; hw_data->get_clock_speed = get_clock_speed; hw_data->get_sku = get_sku; + hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE; hw_data->fw_name = ADF_C3XXX_FW; hw_data->fw_mmp_name = ADF_C3XXX_MMP; hw_data->init_admin_comms = adf_init_admin_comms; @@ -406,6 +408,8 @@ adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP; hw_data->pre_reset = adf_dev_pre_reset; hw_data->post_reset = adf_dev_post_reset; + + adf_gen2_init_hw_csr_info(&hw_data->csr_info); } void diff --git a/sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c b/sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c index 307be8d31879..6d17b3216a29 100644 --- a/sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c @@ -133,6 +133,7 @@ adf_attach(device_t dev) /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); + hw_data->admin_ae_mask = hw_data->ae_mask; accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ diff --git a/sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c b/sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c index c8ff19d00bde..28dafa68a357 100644 --- a/sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c +++ b/sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "adf_c4xxx_hw_data.h" #include "adf_c4xxx_reset.h" #include "adf_c4xxx_inline.h" @@ -754,7 +755,6 @@ c4xxx_get_hw_cap(struct adf_accel_dev *accel_dev) ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 | ICP_ACCEL_CAPABILITIES_CHACHA_POLY | ICP_ACCEL_CAPABILITIES_AESGCM_SPC | - ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY | ICP_ACCEL_CAPABILITIES_ECEDMONT; if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { @@ -2128,74 +2128,6 @@ configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) } } -static int -adf_get_heartbeat_status_c4xxx(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct icp_qat_fw_init_c4xxx_admin_hb_stats *live_s = - (struct icp_qat_fw_init_c4xxx_admin_hb_stats *) - accel_dev->admin->virt_hb_addr; - const size_t max_aes = hw_device->get_num_aes(hw_device); - const size_t stats_size = - max_aes * sizeof(struct icp_qat_fw_init_c4xxx_admin_hb_stats); - int ret = 0; - size_t ae = 0, thr; - unsigned long ae_mask = 0; - int num_threads_per_ae = ADF_NUM_THREADS_PER_AE; - - /* - * Memory layout of Heartbeat - * - * +----------------+----------------+---------+ - * | Live value | Last value | Count | - * +----------------+----------------+---------+ - * \_______________/\_______________/\________/ - * ^ ^ ^ - * | | | - * | | max_aes * sizeof(adf_hb_count) - * | max_aes * - * sizeof(icp_qat_fw_init_c4xxx_admin_hb_stats) - * max_aes * sizeof(icp_qat_fw_init_c4xxx_admin_hb_stats) - */ - struct icp_qat_fw_init_c4xxx_admin_hb_stats *curr_s; - struct icp_qat_fw_init_c4xxx_admin_hb_stats *last_s = live_s + max_aes; - struct adf_hb_count *count = (struct adf_hb_count *)(last_s + max_aes); - - curr_s = malloc(stats_size, M_QAT, M_WAITOK | M_ZERO); - - memcpy(curr_s, live_s, stats_size); - ae_mask = hw_device->ae_mask; - - for_each_set_bit(ae, &ae_mask, max_aes) - { - for (thr = 0; thr < num_threads_per_ae; ++thr) { - struct icp_qat_fw_init_admin_hb_cnt *curr = - &curr_s[ae].stats[thr]; - struct icp_qat_fw_init_admin_hb_cnt *prev = - &last_s[ae].stats[thr]; - u16 req = curr->req_heartbeat_cnt; - u16 resp = curr->resp_heartbeat_cnt; - u16 last = prev->resp_heartbeat_cnt; - - if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && - resp == last) { - u16 retry = ++count[ae].ae_thread[thr]; - - if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) - ret = EIO; - } else { - count[ae].ae_thread[thr] = 0; - } - } - } - - /* Copy current stats for the next iteration */ - memcpy(last_s, curr_s, stats_size); - free(curr_s, M_QAT); - - return ret; -} - void adf_init_hw_data_c4xxx(struct adf_hw_device_data *hw_data) { @@ -2230,6 +2162,7 @@ adf_init_hw_data_c4xxx(struct adf_hw_device_data *hw_data) hw_data->get_clock_speed = get_clock_speed; hw_data->get_eth_doorbell_msg = get_eth_doorbell_msg; hw_data->get_sku = get_sku; + hw_data->heartbeat_ctr_num = ADF_NUM_THREADS_PER_AE; hw_data->check_prod_sku = c4xxx_check_prod_sku; hw_data->fw_name = ADF_C4XXX_FW; hw_data->fw_mmp_name = ADF_C4XXX_MMP; @@ -2256,7 +2189,7 @@ adf_init_hw_data_c4xxx(struct adf_hw_device_data *hw_data) hw_data->reset_hw_units = adf_c4xxx_reset_hw_units; hw_data->exit_accel_units = adf_exit_accel_units; hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP; - hw_data->get_heartbeat_status = adf_get_heartbeat_status_c4xxx; + hw_data->get_heartbeat_status = adf_get_heartbeat_status; hw_data->get_ae_clock = get_ae_clock; hw_data->clock_frequency = ADF_C4XXX_AE_FREQ; hw_data->measure_clock = measure_clock; @@ -2275,6 +2208,9 @@ adf_init_hw_data_c4xxx(struct adf_hw_device_data *hw_data) hw_data->count_ras_event = adf_fw_count_ras_event; hw_data->config_device = adf_config_device; hw_data->set_asym_rings_mask = adf_cfg_set_asym_rings_mask; + + adf_gen2_init_hw_csr_info(&hw_data->csr_info); + hw_data->csr_info.arb_enable_mask = 0xF; } void diff --git a/sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c b/sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c index 3470e4e8a8a0..1a116ef4acb0 100644 --- a/sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c @@ -130,6 +130,7 @@ adf_attach(device_t dev) /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); + hw_data->admin_ae_mask = hw_data->ae_mask; /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c b/sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c index 2ff72b94fa53..bf73b60adc39 100644 --- a/sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c +++ b/sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "adf_c62x_hw_data.h" #include "icp_qat_hw.h" #include "adf_cfg.h" @@ -373,6 +374,7 @@ adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_errsou_offset = get_errsou_offset; hw_data->get_clock_speed = get_clock_speed; hw_data->get_sku = get_sku; + hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE; hw_data->fw_name = ADF_C62X_FW; hw_data->fw_mmp_name = ADF_C62X_MMP; hw_data->init_admin_comms = adf_init_admin_comms; @@ -411,6 +413,8 @@ adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP; hw_data->pre_reset = adf_dev_pre_reset; hw_data->post_reset = adf_dev_post_reset; + + adf_gen2_init_hw_csr_info(&hw_data->csr_info); } void diff --git a/sys/dev/qat/qat_hw/qat_c62x/adf_drv.c b/sys/dev/qat/qat_hw/qat_c62x/adf_drv.c index ebeb4949a8c2..198bd1e0a78d 100644 --- a/sys/dev/qat/qat_hw/qat_c62x/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_c62x/adf_drv.c @@ -133,6 +133,7 @@ adf_attach(device_t dev) /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); + hw_data->admin_ae_mask = hw_data->ae_mask; accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c b/sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c index 44dde0750792..59c53db3947a 100644 --- a/sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "adf_dh895xcc_hw_data.h" #include "icp_qat_hw.h" #include "adf_heartbeat.h" @@ -361,6 +362,7 @@ adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_clock_speed = get_clock_speed; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; + hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE; hw_data->fw_name = ADF_DH895XCC_FW; hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; @@ -396,6 +398,8 @@ adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP; hw_data->pre_reset = adf_dev_pre_reset; hw_data->post_reset = adf_dev_post_reset; + + adf_gen2_init_hw_csr_info(&hw_data->csr_info); } void diff --git a/sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c b/sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c index 2ddff279e4e2..0ab733bab18f 100644 --- a/sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c @@ -126,6 +126,7 @@ adf_attach(device_t dev) /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); + hw_data->admin_ae_mask = hw_data->ae_mask; accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/sys/modules/qat/qat_api/Makefile b/sys/modules/qat/qat_api/Makefile index 65da6fa52fad..70886968ab56 100644 --- a/sys/modules/qat/qat_api/Makefile +++ b/sys/modules/qat/qat_api/Makefile @@ -44,6 +44,7 @@ SRCS+= common/crypto/sym/qat/lac_sym_qat.c SRCS+= common/crypto/sym/qat/lac_sym_qat_hash.c SRCS+= common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c SRCS+= common/crypto/sym/qat/lac_sym_qat_cipher.c +SRCS+= common/crypto/sym/qat/lac_sym_qat_constants_table.c SRCS+= common/crypto/sym/qat/lac_sym_qat_key.c SRCS+= common/crypto/sym/key/lac_sym_key.c SRCS+= common/stubs/lac_stubs.c diff --git a/sys/modules/qat/qat_common/Makefile b/sys/modules/qat/qat_common/Makefile index 9645d3b765db..22ea235fdce6 100644 --- a/sys/modules/qat/qat_common/Makefile +++ b/sys/modules/qat/qat_common/Makefile @@ -9,6 +9,8 @@ SRCS+= adf_accel_engine.c adf_freebsd_admin.c adf_aer.c adf_cfg.c qat_common_mod SRCS+= adf_heartbeat.c adf_freebsd_heartbeat_dbg.c SRCS+= adf_dev_mgr.c adf_hw_arbiter.c SRCS+= adf_init.c adf_transport.c adf_isr.c adf_fw_counters.c adf_dev_err.c +SRCS+= adf_gen2_hw_data.c +SRCS+= adf_gen4_hw_data.c SRCS+= qat_freebsd.c SRCS+= adf_freebsd_cfg_dev_dbg.c adf_freebsd_ver_dbg.c SRCS+= adf_cfg_device.c adf_cfg_section.c adf_cfg_instance.c adf_cfg_bundle.c diff --git a/sys/modules/qat/qat_hw/Makefile b/sys/modules/qat/qat_hw/Makefile index 820af989b536..40c1d26b4687 100644 --- a/sys/modules/qat/qat_hw/Makefile +++ b/sys/modules/qat/qat_hw/Makefile @@ -6,6 +6,7 @@ KMOD= qat_hw SRCS+= qat_c62x/adf_c62x_hw_data.c qat_c62x/adf_drv.c SRCS+= qat_200xx/adf_200xx_hw_data.c qat_200xx/adf_drv.c +SRCS+= qat_4xxx/adf_4xxx_hw_data.c qat_4xxx/adf_drv.c SRCS+= qat_c3xxx/adf_c3xxx_hw_data.c qat_c3xxx/adf_drv.c SRCS+= qat_dh895xcc/adf_dh895xcc_hw_data.c qat_dh895xcc/adf_drv.c SRCS+= qat_c4xxx/adf_c4xxx_hw_data.c qat_c4xxx/adf_drv.c qat_c4xxx/adf_c4xxx_ae_config.c qat_c4xxx/adf_c4xxx_misc_error_stats.c diff --git a/sys/modules/qatfw/Makefile b/sys/modules/qatfw/Makefile index 6c87a28c6527..ac7bddd18858 100644 --- a/sys/modules/qatfw/Makefile +++ b/sys/modules/qatfw/Makefile @@ -5,6 +5,7 @@ SUBDIR= qat_c62x \ qat_200xx \ qat_c3xxx \ qat_c4xxx \ - qat_dh895xcc + qat_dh895xcc \ + qat_4xxx .include diff --git a/sys/modules/qatfw/qat_4xxx/Makefile b/sys/modules/qatfw/qat_4xxx/Makefile new file mode 100644 index 000000000000..895bb4d0cb9f --- /dev/null +++ b/sys/modules/qatfw/qat_4xxx/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2007-2022 Intel Corporation +# $FreeBSD$ +.PATH: ${SRCTOP}/sys/contrib/dev/qat + +KMOD= qat_4xxx_fw + +FIRMWS= ${SRCTOP}/sys/contrib/dev/qat/qat_4xxx.bin:qat_4xxx_fw:111 ${SRCTOP}/sys/contrib/dev/qat/qat_4xxx_mmp.bin:qat_4xxx_mmp_fw:111 + +.include