From a81dead0b275bb5fc8bd4425d9d9265c54fd5860 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Mon, 13 May 2024 16:53:31 +0300 Subject: [PATCH] [controller] Add NFSStorageClass controller and validation webhook for StorageClasses with provisioner nfs.csi.k8s.io (#4) Signed-off-by: Aleksandr Zimin --- .github/workflows/build_dev.yml | 1 - .github/workflows/build_prod.yml | 1 - .gitignore | 3 + .werf/bundle.yaml | 1 + charts/deckhouse_lib_helm-1.21.0.tgz | Bin 24064 -> 0 bytes charts/deckhouse_lib_helm-1.22.0.tgz | Bin 0 -> 24121 bytes crds/doc-ru-nfsstorageclass.yaml | 14 +- crds/nfsstorageclass.yaml | 37 +- hooks/common.py | 34 + hooks/generate_webhook_certs.py | 42 + hooks/lib/__init__.py | 0 hooks/lib/certificate/__init__.py | 0 hooks/lib/certificate/certificate.py | 265 +++++++ hooks/lib/certificate/parse.py | 31 + hooks/lib/hooks/__init__.py | 0 hooks/lib/hooks/copy_custom_certificate.py | 84 ++ hooks/lib/hooks/hook.py | 56 ++ hooks/lib/hooks/internal_tls.py | 434 +++++++++++ hooks/lib/hooks/manage_tenant_secrets.py | 140 ++++ hooks/lib/module/__init__.py | 0 hooks/lib/module/module.py | 59 ++ hooks/lib/module/values.py | 60 ++ hooks/lib/password_generator/__init__.py | 0 .../password_generator/password_generator.py | 77 ++ hooks/lib/tests/__init__.py | 0 .../lib/tests/test_copy_custom_certificate.py | 110 +++ hooks/lib/tests/test_internal_tls_test.py | 159 ++++ hooks/lib/tests/test_manage_tenant_secrets.py | 102 +++ hooks/lib/tests/testing.py | 57 ++ hooks/lib/utils.py | 54 ++ images/controller/Dockerfile | 17 + .../api/v1alpha1/nfs_storage_class.go | 60 ++ images/controller/api/v1alpha1/register.go | 49 ++ .../api/v1alpha1/zz_generated.deepcopy.go | 77 ++ images/controller/cmd/main.go | 131 ++++ images/controller/go.mod | 71 ++ images/controller/go.sum | 204 +++++ images/controller/pkg/config/config.go | 72 ++ .../pkg/controller/controller_suite_test.go | 66 ++ .../controller/nfs_storage_class_watcher.go | 253 ++++++ .../nfs_storage_class_watcher_func.go | 720 ++++++++++++++++++ .../nfs_storage_class_watcher_test.go | 452 +++++++++++ images/controller/pkg/kubutils/kubernetes.go | 35 + images/controller/pkg/logger/logger.go | 84 ++ images/csi-nfs/werf.inc.yaml | 2 +- images/webhooks/src/go.mod | 35 + images/webhooks/src/go.sum | 110 +++ images/webhooks/src/handlers/func.go | 72 ++ images/webhooks/src/handlers/scValidator.go | 58 ++ images/webhooks/src/main.go | 82 ++ .../werf.inc.yaml | 17 +- openapi/config-values.yaml | 15 +- openapi/doc-ru-config-values.yaml | 4 + openapi/values.yaml | 17 + templates/controller/deployment.yaml | 98 +++ templates/controller/rbac-for-us.yaml | 109 +++ templates/csi/controller.yaml | 1 + templates/webhooks/deployment.yaml | 96 +++ templates/webhooks/rbac-for-us.yaml | 7 + templates/webhooks/secret.yaml | 12 + templates/webhooks/service.yaml | 16 + templates/webhooks/webhook.yaml | 23 + werf-giterminism.yaml | 2 +- 63 files changed, 4861 insertions(+), 27 deletions(-) delete mode 100644 charts/deckhouse_lib_helm-1.21.0.tgz create mode 100644 charts/deckhouse_lib_helm-1.22.0.tgz create mode 100644 hooks/common.py create mode 100755 hooks/generate_webhook_certs.py create mode 100644 hooks/lib/__init__.py create mode 100644 hooks/lib/certificate/__init__.py create mode 100644 hooks/lib/certificate/certificate.py create mode 100644 hooks/lib/certificate/parse.py create mode 100644 hooks/lib/hooks/__init__.py create mode 100644 hooks/lib/hooks/copy_custom_certificate.py create mode 100644 hooks/lib/hooks/hook.py create mode 100644 hooks/lib/hooks/internal_tls.py create mode 100644 hooks/lib/hooks/manage_tenant_secrets.py create mode 100644 hooks/lib/module/__init__.py create mode 100644 hooks/lib/module/module.py create mode 100644 hooks/lib/module/values.py create mode 100644 hooks/lib/password_generator/__init__.py create mode 100644 hooks/lib/password_generator/password_generator.py create mode 100644 hooks/lib/tests/__init__.py create mode 100644 hooks/lib/tests/test_copy_custom_certificate.py create mode 100644 hooks/lib/tests/test_internal_tls_test.py create mode 100644 hooks/lib/tests/test_manage_tenant_secrets.py create mode 100644 hooks/lib/tests/testing.py create mode 100644 hooks/lib/utils.py create mode 100644 images/controller/Dockerfile create mode 100644 images/controller/api/v1alpha1/nfs_storage_class.go create mode 100644 images/controller/api/v1alpha1/register.go create mode 100644 images/controller/api/v1alpha1/zz_generated.deepcopy.go create mode 100644 images/controller/cmd/main.go create mode 100644 images/controller/go.mod create mode 100644 images/controller/go.sum create mode 100644 images/controller/pkg/config/config.go create mode 100644 images/controller/pkg/controller/controller_suite_test.go create mode 100644 images/controller/pkg/controller/nfs_storage_class_watcher.go create mode 100644 images/controller/pkg/controller/nfs_storage_class_watcher_func.go create mode 100644 images/controller/pkg/controller/nfs_storage_class_watcher_test.go create mode 100644 images/controller/pkg/kubutils/kubernetes.go create mode 100644 images/controller/pkg/logger/logger.go create mode 100644 images/webhooks/src/go.mod create mode 100644 images/webhooks/src/go.sum create mode 100644 images/webhooks/src/handlers/func.go create mode 100644 images/webhooks/src/handlers/scValidator.go create mode 100644 images/webhooks/src/main.go rename images/{csi-nfs-controller => webhooks}/werf.inc.yaml (51%) create mode 100644 openapi/doc-ru-config-values.yaml create mode 100644 templates/controller/deployment.yaml create mode 100644 templates/controller/rbac-for-us.yaml create mode 100644 templates/webhooks/deployment.yaml create mode 100644 templates/webhooks/rbac-for-us.yaml create mode 100644 templates/webhooks/secret.yaml create mode 100644 templates/webhooks/service.yaml create mode 100644 templates/webhooks/webhook.yaml diff --git a/.github/workflows/build_dev.yml b/.github/workflows/build_dev.yml index 23366ab..0930de9 100644 --- a/.github/workflows/build_dev.yml +++ b/.github/workflows/build_dev.yml @@ -10,7 +10,6 @@ env: GOLANG_VERSION: ${{ vars.GOLANG_VERSION }} GOPROXY: ${{ secrets.GOPROXY }} SOURCE_REPO: ${{ secrets.SOURCE_REPO }} - SOURCE_REPO_TAG: ${{ vars.SOURCE_REPO_TAG }} on: pull_request: diff --git a/.github/workflows/build_prod.yml b/.github/workflows/build_prod.yml index 561e19b..fb19f1e 100644 --- a/.github/workflows/build_prod.yml +++ b/.github/workflows/build_prod.yml @@ -11,7 +11,6 @@ env: GOLANG_VERSION: ${{ vars.GOLANG_VERSION }} GOPROXY: ${{ secrets.GOPROXY }} SOURCE_REPO: ${{ secrets.SOURCE_REPO }} - SOURCE_REPO_TAG: ${{ vars.SOURCE_REPO_TAG }} on: push: diff --git a/.gitignore b/.gitignore index 4fd25ed..dcabfa3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ __pycache__/ *.py[cod] *$py.class .pytest_cache/ + +# dev +images/controller/Makefile diff --git a/.werf/bundle.yaml b/.werf/bundle.yaml index e876254..a2242b0 100644 --- a/.werf/bundle.yaml +++ b/.werf/bundle.yaml @@ -2,6 +2,7 @@ --- image: bundle from: registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc +fromCacheVersion: "2024-05-12.1" import: # Rendering .werf/images-digests.yaml is required! - image: images-digests diff --git a/charts/deckhouse_lib_helm-1.21.0.tgz b/charts/deckhouse_lib_helm-1.21.0.tgz deleted file mode 100644 index d00bd9f9445d4794ba4eae22cdf655de876d8626..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24064 zcmV)gK%~DPiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ{b{jX6IJ|%BDKPEK87ZGgU3{C-j&nXmQWE`Q%lafI**Tef z9k3fDF>V4K04D7{Wan5{AE&M@gMtl##LqRck;t2{gDVsIXgl(hg&g`k)Wx9zrQ2N3}uQY z5-BoE6RZe|$(S;#-%B*+0-d`4gJwL=5+a9N4A00Bs&}j{HBZSAO6W*nG2gmzXb*;m z2gCiXPc4am&tJp%ce<8)VI*l7ai#=M5+a5wUCAVLkN>C7pFe+E9{-2?&!0bj82|V2 zd;fl5#xy|zRtiT`B0@nV&qPFIfIfcQiqh-|9qiAxW@N_2{0JRBeMPtQhlItt!Ilps z75s+kfqh3ff=K$$JHn6*r*g`bx+9!~-VjElOa&j2J3%T2(;_fW_Zz_MeTjaMctVcQ z`}c^lD9K`i0zGPB_?CxjIKzyNiBvK`+XBN71;C(&Ll0@m)iBZtjhRVE=FDY+KzM(V1Gvy;{c>&{3t0II&~fMUhl z0HpG!1BR#M1~?6GJ79S-_=$_RM93BixwjsygZ(D?G=QnTYK38C&;X+HrWJz6eFKz+ zx9$AIF@$?KImtWQ8M`Sn*4CyQ%(0j(1Osil!TiXRY(^GBa?Sbxd&M)RdP1~q+rVPV zs@79XJ>EwD=-ALpR9W<=o_D}1zt zkXb2t|K5v+fqMNfnc|2Xq4>`O{oXNQ*5)qqmcV73m=ioAiL42BJEoCB0Z!AP9$p8x zT)a&86Z^{23gLWE|N9VZFDCGnAcQjvYjv^N)eA*4UM9*WQmzpFuoiQwV>EPqg% z5H=I+%zJNYn`Zi(Ak_`h$|TD|tutcV_2#2tkziA7*lD=WRl@|dl{2qxZ|+)_FYbZE zI{hVM+?#{eg_HXk7oK73OL6l8%^Jx)*l&<|)vXwlTey--9-mSvGFX`ZI*TVneddm2 zX9u-8h8ceM1H(6%CVI>qp@S`iBuPl5xG(~oVHHhZ0#YYYoThaKwb*LjqvW2ApZWY{ z)gmfE(u77>8X*Z{T+DY=DlX

5Qr)bhxtk5Ryt@MJCokn*Et85($D8(JtKX1X!@1 z5cMx9VQ9sX+si4Imx7GxI}}6-&*B02%$SHl&@O8L%otY~_ME%{1|ZC|co0oV^j2oG z(z@t&Dkv*dFnnXV8Gel=xx{LUg1sDy)x}_j8J-ZaC$$vJE@-ghkAFGEhfkl$Z02+4 znNFuV{hM;BE{M`0)e)=!+UGHoHZVaaNv7kdZ>W&!C#t63nNMs(h>$O{B)JB1a`p$P zvdK0OWJ0A9^Q{J(mUq!qP%fyNhf#v1?xQM86H;&&$hlNxHi!&SgGf+_U<1E{J}VT{ zjD_)-gsw+6AaRl7GB6cSh%jBC0U~DefyS!CSE{CijKLlyPUt^LTu0>m_cAjBJ!IL~ ztr@lxYOO%Z<4}?)GtxGALXj~=XC$->HUSzoQAQF;?uaGCGR!0qVctL5a25Sl8xe6s zBXS%?JY&kZ&`47IHutTyPUwZc%|5k0{j;DNh$lD>IZNiD;9P}cnh?_`%M5tV*A5b6 zd1{lv59qgyE8;iDV&a*R0SX2KWfq8ER6w|OztC2rf=envBQ4rMSi6r@l ztWJ&XqJy#2PT>W4r=C=(NC?)Z8-_XMtclP{lR@rbUwpBBas2A+`ttbXY^U;(8GeHZ zr#PZ&UX`j$Fw1Ob94BZ+qh{lh;M81xVB&rosb0(~S+_y}|?)(|YNXDadabkxG_H@3cT9 zost<5I0x)-=9Q}kd0{sl}x&*i$-@$Z3bU5qQ# zVc&mhBT?ETTHZ!f@(6w(CPMn8KbVFG)3DGq^jiV%VHr9MM*BLqr>LUMr#2k5ZP(?E zM?><^@56wUVvL3~p9fpAep~V{WKH_-I4x^YoDxbqLZ8}FG^8!`vlYEY`c$cOKvI2gods1PQ$>EeskXs}_Pk*MckcUTA~^X^pE zg0=RBACTgI!?UExjJ4z=bf@7qSeEVt-z6x}zmi4shpM)N-x5_Gy{Q@dQ-TvU{b%_V z7piQZeIvOEt7DH=e8UC83}x&sqD3+D>1?pdzzW%}=Q~LhpSVHb0u~I+mJ7 zQVTOn4WVvjlw0=JiT8(pBy5EI(j{kgAlj>0<=}{q> zqb7`Z-87*HRIttObH5H$;r!^@ONkR|m`z z{`VHEXsZ8V5|;G8Vv-O=f?t+L6(yL?dg3!F$9=GvG@?FWYrw2av)XO|ZQesFPcK3B zAeY<_R;9K>+Exy`C`em)uDe;596 z;r{jG>z6;gI`gKLD8eZo(S)k`aO)8|*U+#+5td{ZHCYyPOOpfvv32sc9%-s$qKKGj z4+P{3z@>Ayqzo&xXcQv1>LNh2HUUmu~@dZ=?o5Ff*U9`hvOx0Fo~G=Y*WZb?!i zE>J?SB+@@Pjz(CL7;(1s2;J(Tt;Zr!@aeDh%?UM6$*)6nmIK9tpqQkBL|Bm+QHI9& zh6_#6OF&i|!3}=ML2DeVxhHai)_VPtN3ve3W@#hNMNFBm_Bwtel{FxLgf6vZlQJ)u zb8>xtZ7kM_&1D0L`8FD+o?2&GgK0KOsGMrE^GM6knmgn&;SgnGE@p0*t}Vk_F1I*&t9i~+U>oxL>{bP4LpfSSyrD6it;JkhZmT=Ra ztmMMh=dHZy=hmYlPp|ZJ`5CE$^E;W2dg+P3-?`}GInf5E5u1pG*TGW_@1EI7ETd09 zr`Y>`I^$;_cza^_Jfd!opwA)X{-n|6y*c>LMd+V}cVBWy6i$5WQJ!1tKU?0fTPVL1 zw_fBE|9-uCi{D%0vT`5TjdO*SAT?6i?UXmacKkWG{d?)GpS{;Fr0{t-{eqqcpMPII z|1**KXW{af8xN&fyPp_;$LYKWYJuz5QuSMFS_hSfpF?O+J~i+ega_;109|26cx^kJ zt8Waibx*aB*5c)b!t$+mL)lT*7~y=Tk;uJ`jn6(BDNEw>h)BvbJwzq<6_tEOa{nwM zljX;Tmq6I-{s+cmvg*wb=GT4y!@<*M&mULre|i4&@X5pd5BKr&vp=D5lgSARZ9OU6 zd{4n)&+85V;wk`oB((zUJ6>^6g;6fde?ONSn8H=XxBBneWdOy)rfUF-w=EX{7^Qym z|D9g=K)t6;P`%eJV9OPwN)_hIZ#Fv+FnUMf7|wHXZ3i$K{tS| zit)8p1t`j>%9Hd@_-uw*>;q|h-|~5_(=@!@WO?{fU%PVlV?hmSEt}Kw7k8>e-GqG& zwW!;e+9@3g zoEM*IE{|(ZQ65d5W`eTGbu=Y$rnx$wFrGg?dq<)Su3{`5E7UNrEz@s^m{m?p4vY-W z-lc+=gv;uI{sH1C5m?I%owLeIgx-?*5g;iUt356KT3aY3P(z?moM?x3<_>7KHMk^V zhM6n+-_YP%VMU|bThE!2-#n${U27ml+Re1R)!q?&TvLOx^YcmSpr&+w@)OnVHLR^v zw0F4zyNuh=VhId=Yt{U?--g$( zPreT?kKcT^v!JetkE>qXBq&uk32I+AVQhkOgT-D#M|+lf*dj}lY(kmbD})KN-H#tT z@O}To*Jt6y>(jG4R(SQRs-z@>IsAoR#I?J=fg!zTeS@O1pQo|*PyJ^1aAqGyN5Utv z$7%h0I;_7+=((^oiw2sbbcW)@!0`oG)JQ3rxYE!sR(x8?ti6x8Cyck$&r~_6P zMV*f?xVjWX`muSw6=fn#4RVmjKA%-b>A2Xy7KyAbP*>h*v9Z5RH1ZT^7svKO!IAB5dd!I?Rg ziv`vG+MJ8d{YV9LOj5#Pi8w<)0|7`9r6`-oU$$NDWNDo$=252MgRR}WIVj%R+oIq; z6caE^)hv0GVu5Ew5g`Z0^T+`p+UeXru1ckG>v*WYcLlNB{Zq&_(4Gw05N6k=gX|)r zqN37OcMV6oNpxb_0*A;h=fnNs{_r3`!NIe? z3=f|^8QT91mZHc$JmoV?*>b$__&OUgqE60Fua^4K28G_&ckdfcLe(jqd;UV3kqE2e z$r^C@TT8C8_v@yv=)U}5xAb>|rXI$4Q}^or_CAB3hu?zuZ%QZA&|cMPN4dcZN7JDLtW*%*gZmkcE&HixxwD!YWWX}~I zSqYw8CYX_P`5g?2V>cr1MwR`mE34LIbk#f16o+;hH9*6T!40)?P50eJw^JHT^}trJ z6ceM-98GwhVu7$kaOGs@fc7zOVm9?JHM(rT7BH@?Dx?yWO@dMY^+^ZSN+fP-EG-97aSaI;Ig; zUOX7&8H7QD-?$#iC6mNdwwCCh@i;2$x5u)cId=~k~!$=v6F_f>@Jxx z!ox_YBlN)S{QWlFk}v< z=|*8H$e6y<)-~w0wG=cNZ}_c!=>Y_sasiTRTZ6d7YO1ZHHj9qd3LGtD&0o-KgUR`V zMuoVbI*0oLrGhdwM!_Go=>91GsLg9@hsp#EWi%zF8B|AmLqurwUOQkAl#~{Mh{ znP+qoujy{;7_AWP1YyfKSPkxXVf_2r|5_%_dhP#{XVv}x!{^T)_W$qY*Wdp4OUJ4i z{ipk{Xk~+6e}{fe_+-FSC5JQoZZM_^L0_WMGfDqRenq!DOJW`R;W#$2Uy7xU0!u5W z)x{}C>%f&u*T8@bKn_+Hh(;z>>)}Ou@b6J_<^iV+&B%z3=88y%sYW3PAIC8oqNiGn z%O8sGlVIJDyRkL}*%Ut#FRv7+#tv9lQiR9gou+UNN`X^%zY9yg+OL=WpAbd_R{3FR zJ;zzX1uU}v2hXePKaZb0e%SxJmtTLo($oq3DS-ZaU}K}s@Lxx#1_m4{&C{>I5@ZtW zCEQTttpi1wRD71F7H>yHsKE>~JRzbxPpgyq`-kVO~vF^w;hP52i*es<4hzq-@(w)`OfUJ3ed@ZT9 zU(9$+y!DWK*&;5P)V3{RBdN-6Xk%lOeTjk@oX;*_kZbsR(fGv>-uW2HcdwVe?_M7A z;FT&p4O;X0JWOlD<4>0g0h8ZQa7v}kEBK8_bP%wki?rY_eVw7vY1X@7d%NUo<{ z{efAF_0XbGrjQTC1%{A+<+A+RIRgQJV+9<3#MIJ#^enXa8!{iN8n>rgx;j>!l5W4> zoI~kPn?mE0x>704a0Q;mJeO3DDA`A_$tZc*;Kn;ApT{^Y;0g<|K+xSn)=CX+S97%! zLkfDe5lripMS6+px;-Rxp#I`tjQaPgn#Jbx&V!>%cvOZllluo@G{s3m*o1_VCpUVq z*GALoD(b+pLA{yS>Q3UR1m?~s37#qP^l_kLGM>fb1PkqbVmhG;Cp;pUxpy)13Y|4( z#`d28EyDkwS972`Ien%RrBS0fji9~V=Tbu?4 zxM&M{2`|w|%tm@(b#mS-JjwBw<4Rj!u^edu9_ZTFE@Z3)Ie|NNuj{j+JsX?3Z<|wjQ@L#W+oTSC-D5hfLG#q&_sN)96hv($z)OgFCNv zeHXDVZG)>J0o`4RqCaRUK=I?uE9 zG7qM8k&asT1wjYu)|Et9Yw=fZW|>rIL{NY?mxJ|HFN z>^MnK@OQYg$RW#XXqMP1#dHqZ3XxlBrF){CKVf>#`WQ7LSq8^;9uvbcb;r-^)Kn_W z7wBtTk@A&8S~;h5LZqjU?%f!uHY$fPd?ZT@R8-(|uP2TrYy}NO2oCoZZ96Av3Yz{= z4*w`&VU2NQ&;?!SJaL0_w18VUcXcHN0Kou!XX(0})tK%7g)>AYgw@Xcex>yO#*3;}H^+30#yn$j zZ@XNeez4E~*7mt>o7Ze7m$!U(w0-Ncew(y^t6RW3*ua_<>^6%l*uut@|K$v$f05ta z?c!MQnLVYj#D;vChToxqyu0-Xj0}SN2<_T%Efv>Q`|0x5YCVp<>mtgv_Tyz-elt-I z9lN(r+)NK^F_$aM|NE`~_*=KjuLCVU|M$Fl{`=tZ^N01H`}i#}@hI=#cjQyK-EnAX zERZ!lF8Km+;rsx3NA>7}R<;VAy<5A_jHFD{ruWk~yI;33A>Plm=-Nyi`VFfa5{qg8 zIy=;ft~>d*4gJGb!RPnuY5%DzE;(G<0W7fpPY({O=f4jRpFgbs-^|b4Y~8iU`w6$4OKnL2xuB zF}mW>+iRZCYFUV0F30b8VKn8pEM$Dd<9TQh5b)te_ zwp#8+6d5~~7o1&ju8vGK;o|uZl87U;Up)9$@GLb?suu(fyabe*M9MesHPg6NyW*^R zxmjnT%7XeVmCK=4TqJJlZ-uG<{g0hGFaHL^=gHH@Pmkcg55wnvhYyL~h6!cayYMX$ zjQHy^U%fz2hJP6zJYRm`km#+chGKxC8`aOWN$&A5HkL<&R zx0>e$tK3O#iCKF!Zhpt~+116_%kbp&#r2!3V|OjKcZE(}j-@R6jg7B{3s9@W>{AD{ z55J{;z2<)=w|3VA=*WW3Y6Rd}5JEoD2VbU}}fy(z|QqBld z_7P>1V5h?`=DiOY=zUz7qhs^#%3N$|Z*p}M0@t>gd8EzilnAOoggHxF$gX;pFV(AV z&c4qRgIzAcVQDwL7@)*F8acp%|3e34a(92AEA#QZF~ox zK3vZBFZ}Ce|Cg-ySp6-q|NBo4A6M=F;lusE_w(!A{H^R)aeY0SS^OI3g$*1Skjt^a15Op${ly zimHTE51W2!mE7yE*Z41$mR$e-@8RR>{SSu+`w!#)UVi=KzsCBaOBH_2?|o>zP5_oR z;2vEx{w-#GLct12^@tFDpItRqS5GK%x7`Gb(?S}rVAv^QGJCq~Ob9Oi@P|L3J{LH^ z`7cWZ`g`z4`S$>&SSccy)`n46NhyB_x|66@xkjQa&WAwqet*5}e|2SRNf*#<{|}x& zdt9ymdiebCq5kW={Fbo)iO=9PID!!+$zl!X=Nd%i9%2J#Yz&--#%3-rK{~ z_dOO$%4}J8CVwrV(TXYq2Ynqnb+GQF(-qVNyW17zm}eVRiMfX4R#1gU2?@vD33_YL z5xms+2*OcBB;3ykn}7M{Zm-q6)%p8;G>g#qUQ}J$L)wILJp}7eod@pTmG$K54=H5o4C*6A+j9pxQJ}UOd<0f@3I9D6e?*3ZTWeD7?Vq7XoC6T0@MzUT7C^{pd zE&skQ`AZT-e3qhAaJcH~%Ah&5Mdt6IjWV_CR)(ef(vWR-_)U$YG6yRkN^&V9HP#qc zAx;v08>WKZ(1c7#NMwW)a{*(QUKS|D%apO1ru_EO;W*1tn6B6F6w}IFw1~ZpCO00| zE^R?@Y%($QpbjCuZY~v>8A~h)B*Lsw>KdWBmrWx?1wyb=98M*h53kQ&YTS^fgcTjp zgsS=7w9=Q`wVO7A{PvWwdLj~UXPwc#u(hCoWs4NZhQh;WLh+_a$LKVtZ623DCaW7 z@r<(jkes$Y7rjV&-|_ZW@%y)KLePjz&)lx~-fMR1aoIy=&6Qxf;8&b&@0 zKQkBAWCAp?&ITO6eEG?HMu=G);y4aakymKy*nfZi@@0SmZKy8aTpgdBbtV$7DSX+5 z&gA+JSb08kS5KGv*u8eGLQ3n{qIjw0%g2rUe41Y9oKty|!>_Ho_I~;q^~U?@a-Uzv z`j_iDcijx#gb~|?F6Hl?o}X^4Cw-oqs;lguwk<#3t4F?%{r`UXANC~BX6nB@e)6QA z|M&RtA^+n(e*H~isS|{lF>Y?TC~EGbFKk5=+Y}0b3>lu0R-@+%!(4k`AP;sQQ`JFO zb3vGLHXyqDsm3TYAlnZ!8WL?q9Mh%-qk10s(12)$(OTSc^S8eG>N$F04#fC{a_n(m z@-D?pQqG{te#pn;M62)UgvJ=eeLdQGQVd3hZ8&KcoZHcnw0+JsS;Q7o(#r?LJJGZ~CBe zVBrsJ$*4f9Bxu)3QmaO)Y0pReey#LqOSg%sjQ9-^^F{{iB}g@vRUp+xwLx*UR|YNJ zB)&oX@B39}2LIs>Ecy<@CuIlb4f&0T@Bb~1ge;2x7*$+QHVG#Jk1@kxjOBF1v55Nz zf(zomXHTA2>pwky{`}$m@4ftb#D6u-hY*Ru0a6*-!Y|xA67pA=kj7Meg?^4QAqa!N ze(9L{>KUt=5|rZ@7#z%QGYdZR!0uI`ct{YHc5@mh=zstFfApL;hUs%oDBMjv=NXDH z)2|UVi_R$)isB@hqX}U|U`5%aAgDVO?Jc845$PI^r)V*qQc*n}?BJdj965p3jy-#A zeA!kiBHC9LR}hqoqpQ+^=Gqr8N)vA1Ycw>?d2%gFF^Sjq+{NMBG8>_7AIb+L(}XHm zqzd)|^Z_M=?Vy7lR1;klio@Oq#F!_ssSGO%FgT76%FuT_`fdE1Vl~~hZZh}%u*xv} zHsK>%5xFFpVDHP}KV(s(#L+;#BKM|NcmwHJDzsBMYwr52`LaR;@hgLS*nnei!qFx6 zodyY2&S@e|9h}VEsnnG>RZViv1euU`SLV~>BzZ{{5jp_<``Pr4Dvghr%}r-xundG?LKBgcNqH zeg$9&xl_K^b#~gO4qwP@bV`NHCwJ8epg#a=ov(caxFJxMkp^}Ra&H|3s zO?m#r>dynY4@0_ccn)`Ro$Up!g0Sow@P<5x>I2Fc{cXoAT;JUnds?$5%wnq3=9AfH zp}dol&JEu#A72EVdltc<@W6YkyWsZP!0Wl}x#w!Z2UunZQo(0LO-UvPkse|@P66vv zc+e@yhQ7J1{@V=25Cw&T%`#G8j6{IG$lo-Eg685~j11_apLD9dPDhUrtC&3ADbT7o zq*An13#FQt>uDfYk6hJUETVv={7l- z(;P~}b*}NBp+;g!B#Xj!L$1O|JnZ2hxz|~{CZgWs^%M~ z{P|VQU)^Q;U!X#O>Jv`<4gN0A(!y>Re&<%NTXb%QU5@8@`y?5Z^RxRKR^pkdFa z3EI#>+d+X!lj<$lKGYXTrAgPaL_XI1?SRr&L>*)sw1zE(Mpj=`;)P;iAiO@gI=(y$ z&oADbUHy3cG7JN6svhgGXW~e+AVR0evSVe(h1{& zgq($08e>I%^->=oPVgz_v}iBO?xR@xUvH5%I&>-i0wx)iQx|W_{x2) z!3;%tx8CQ;Yd`dNCTlyWdZMqb*hJfi_V0oQ%?3Mm>#}m0V{L1+On#tyj@M(i|0s7L zq3`v@_72+Kp5b@f8k;m|K&*dwp&bhsrVSoR!c?`;j^)4G zjb%xf`POU6&Mvd$yRH4%2bCr}4-3DY3%}Lbe=*4$MqCgshY_C@%K(?T<1dqjt~{UhmS(+`U}(_2#9b%r_cza6ezlyZdQQ&8V2Ne()jCqpp_v6s z=F4>|g~f!ZP(mmfDmo)PQ=udgXR+Ve?3>Q67XKrY3fY$Ck_f1!39u1$acb|LwScxh zJ?^gk%`wx~+ui_)InR_KOI|Ih7I?^IV792$;>^09(SBvwBB#5C*vP&9{NqRICmOnD z4vzPFQ#IDR*MAw0Nr=bb4CfoV|KafI^QZf@`yZY@d079wkDnb;KN3O5b93d8Bne#D zg`^T#k;#16X*yiyuDw(Ux{L!6IuM23$gAIK+n%4V8w))mSseH_I6#3rK^@GnR73=* z90r-mri+3z_LlKmmeY;?9{lm61Uuhi`IJxqBSkNhP85-tYPU{V+kXzy8Oh$3N)27^C30eDZQkI%0f;#_g*XG z+V$~|AGiF!&FrnCy?@rSwB~A;@;h%@_j1|)e4Ubrj%jqkV{*bXroNtE@q{#pO7~#3 zV{-3HwCZUmkI6vrgp{|8-@k8|8-M(Ggo0p0ROfnFjav8lCTYya<_aXblaD@arCFe( zlnYoi^ZI3ODR!!FEIm69-SuCkyEb--ikY&0p{(=%T%jOJG7aJt00OT1cIonQ6OBml zWRND9kqr(z$MFiHzAq9tVi^`L$+)OX@q`FmRG8}>b1j&GdvF7^t+F&BXjlnI)O3(B zSp35Y{U?d-L!PODJ*hNE1s^43CTn+H)Iz=z!h4uFuN?7tgf_-|BP_`ZL0^whuegfr z+JL7f;xpPQC`fE^iK4xuhY~qNZ@7L_g#OT+TFIq?jOjb8-C);TZUNVtFK(|D9}H`2 zSY;dLiX9Z!akPY}4HFj>G_r)Kgo`_=AaT&VS=JbdHpBIY*)n_n`|pVXLa~HjT>QIbU|-gmb9PD zhka?wakP`yW|TcMa8#TI`pFL6BQ&8mM52_48HMd^X;vDi1P<{-VLZEtU@F_NIZ~K1 zCDC>!?ZXIX=9obDkSgqj9~ijK7UsRpwJY0NKNF;7r0ji(jxDuE1>Mx#y=+5@f(^5> zV42A&Z=t;~d*$v>ryjGpnQeDbU-$*zFPfbM5PZbhm`;w6cI$%1M4}N%_^oG6b~TY| zN(9k%1@w}`LefM5MTN*}Oi>A_efM0>tZ8qT@9!4-Y5P_7Uga+22s zB5GZ$nb5-RE+W&l>__Ca4c!V7ZJE6`!N*MQqKF7Jm|=z|ewKO{{VqIMM7^zv)=GO@ z215DbR?nwasB=E1$*c|aehY?~7> zP?@ziHPhMSBlqylJS8YN(WH3&C#W(T_8Ih*Ip+ZxjA(k?ATG(HdvD3HjmG=lrQ-Nyd`v%iQ zYVz~Ju1TZJl*S^liL;m3PH3K(bZ;iOXngUSbmRPF(_0KqR$u#q1Z%7Pa>gPqV$MjM z2PC$`t#hW^HgjDRnSA`XH9&94{HS#i#Vl?tYE@dtaEXalzGjM`gOk=Fb>+y+^>2apGjBrv={Dyx`pgoDdp4VHo8M^$6Lv8D4gsGwF zg+<8rJ($GZ!i!2!CA81)!n7bx*H_2yjW0e(P`0NdW^NR-!hzRA(K)9 z#QJ%KUPoIQq|ekHpjcUNMB#-CS(_%k{f|X@Y!2T)H!bE2ZA1fWA4%RWQ@8Y+rXQc1 z?3?=jR_h#St|OSz;`X38CwFpj{wE1m4W`kl;P2*0aTB_sh*57Em@oRkT08_Qc&uj- zCnv8^j=)6jn_i&r5z91B(JG=S$2R3njZ)J#&q~H)5~(8;T<~ivo}dUA8s@jHGpgZ& zMXwfy7!M;!^UrI3ei=wH35!MIVM}<>*ux;^E57x5n6wLpk)Vo3IPp6abR#wp1jlo0 z7r3zh$lxMsI75?(+Jtuv9jZ+7${ngAwMvI7N5G6L!)!+zo~Z#J4`M3ccC?z285i?` z6HiA=4X97Ml*WCUWlRtpP4z^!i&3I(xp+Is7{0+Yf!2MyQhmE(nckH9mRO8p$qJ(=JVV5qTGpRC%H6`n%*(`;7}brYn)#)Hk(oh6!aOs^H*9 z!lfGgmT{5I+M0~Hh{%BQ0TF_WwpQj0)Id+52mFSJB@D;_%$TR^^1cG&=;ZpmShe3p zw^JHT(Jhw9v(rY8){5`K>F+7NA^P)3Qjj)^1-&5x2{NJD^csIg)kbU#aCHacVvKv2 zn=)*v!H9FE<(Z~SdaJ%Fsw_#h2Ni|lKtoj81)-9XPEKBJK;WG;uB-;*vmMli7$SH0 zn$hN!&2#y+r=Y4v0fR$OGr4VXh*9j)>Ehk_-zks7m`ahsY5j1N#S^mZ?f(nwf9^lo zujYRrK0bJ;|9LOJ9>Xn-M`4V~j5A4^?^|_S4wt82qZxjOJXha+>ZA<0F+pbN*HK?S zF=mF}{ZLSIgbw!q_-|6#x8HBU_!pF$yFXUD`N8}ajQ@lE$4{!~Kb}87co_fp@#{1G zOP#>?xbBUq&TXo_fn(kKvgxRrYwoT=7^5-(x7B9D@Lj~!l!#j@Ne+T>HKlB_;4E92 z-UWGgcQP;TG_FMnHP}1L!CjVRP_GHMN;N@Wh$dI8Mv&PdsMFcD2k>AU|2=-)_TL@r zm!Zk;fSsr{-~V&)?8)Kt+WkLI9_qi{%ddz1Z|VdvGss`K6U%?ipV*W%o4oNA`kACt zGJ{$jb}a&uTZvS%t1g_W2})vA0G1?Dr9wI-Ot8?Z^fa@}L-gUOe9qa!pnvsCgj|lq z<#A4Hg|eT?J3LDh^2_#AsZ<{A?UknPF^@7$2Q=QhIoPvXz@UQujCokh8$oiZyfuLSP8DN_+RIR!gqCW|^gZ6BB_COZub zcY_$_stIyQx&^|ZNxJPaY`x{AR~3V$WZiBMHZkN=K_p?_=|xMC-)aN$dFk(I+=hNa&1KhE@GO zCJ)_Biic5}dAJs|9L^{!wXTiRrZF0&nJw?v0?{voR;8t|2z}@S30&!CO=L!C_EI-^ zMU#X^To`Ezp0OB8-;I0Q(+#lu++u4+(yE_W8wyo2`5A+u*s`bycT+{w5o z<~IwP*j*&viqGkmx7ljzsT|XIwf1KP)YHvha62wBTM`Kn-qsKB~d;JFa?AVPU(#)G51ye>@8(+@fX80Qv8+Q zIQ&&d*wAOLaFb#<($W?$U=|~bQ;;zcghk!U)C<`53l{;{-fOVc4jKd}v7xgO&42GE|04KQ*&HeGm7n@V}m zp8WnIDkGdLGKTxiy_$+p#i`mEKn$j+V(}i-25*XLik{x8ldFdDzGK6uj~};8vC&JO z&}gnnGJn4UTcH6rSW#mz@=P?xZUH0LCp@zERu78D7ZtLLu43)a48J=z+5bn@&_!w1 zMh`>|_#G-IG6NW}#;tr7%~I|Wy0T9Gn8n~)WaWHf zmkxv1<8g4e(uT{$c7N5eGQ4S+vQ0^n5>pi^;_QY9sQH(FffPpttmQ3iD^Z?3F^FI~ zR>VT(dHfX1>ud}MX>^2WzVLmc5_HzqqUNRb1FSf8%|4)6mYCc92Yt?OEv=fCcHOVI*-Fye2PyPaivuwN~N{i*}@Lh#=%1pjR}_=OnW zEX43;A%=xWE*B!X?1N+>p0kB`&U)fmi0XT~5SQ1ag6r_ff?l;4_^U49eery|7|*vo z@$^OcW--b)%cJaz{c1{S5p%~#PgcAk+`f5Wek>|Jk8Gt#BUIEqV zxE7!;FfPC8UkiTWyoAPM4oz&;gU z60{)Nu;Ksey6Z*8T{H*+JJ_FXwF&D}Z_D1>uY3PrrX&i-T!d3PnT9rvjwVYM175KI ze{i_}ta|_3;nQai`~Uax>tPg99*3AIniqmLJpvbX%1bvhC?b`)t)OM0F6A+Dpv*#D zXEs+Wc5fSM%~@Rl*4#qTc8j|4$hHk!BFE!nrs%O_3|!3o@KN7DVv>Y`q@{;*xU?}tp`q`BZ&Dl#}?5N?e`Y!sy z)R;edk;|yE*ltdufBa}xOHAv^vdH^tSqjBdp76>1donlvw8(j%a;X@ekvl~qF-a3X zH@m$oG(`&Xf)XKSaVSWd&g)I4?uBAy*Ll?egy{oC|j z40lF5?xnj`^;k?wW%J)xi^B#b)#uRl5F^!koOQ_MkZ7x-f3uJRX>+Bo0>7|qtzzwq za_eFKqK;sr?yVGgHJYBg7o=`5v#SAZG_Y{dj3>R41uYlitM`JQ_ITZ5S!j#8CcSY|Nl)SP}{Y+K9PG`K44zSQeZ(gv=1QpAB_$K-{Pko)(yG951KT~5|t?Jn~ zH%F`3lv+`Dsa=`krT}jdBpuqT#+RAkZ75o7?hT)M73|;l*RcNk^8DoN;`(fTIJ(z= z51$=8uAKirc(VVn{(C<^wCH!7;%G|HOBxX-$yP^8m*#dj+}}n22eS-|IXc|mf71HV zRH^i6Z}0Z@c8CGZkc-J)Vo=FFNR)kZcJ=BS9bcTHlh+rg=WotmUtFVaUSFXfuFrPS z)!F6M>(d`j^y6J=#snx%Ta*j6ekIq5o}7oD+U5fOC4wDa>zIK|ZfKloSOWR|!vY}hh={1vg!q?3KtD5s0Mt_J zYS*xt8$kL&wWp{~CBBADOk zVaD!^#1up9=dt}07jJ7u;g*ZHKrz@M-P4L8K$&AB9{@&lmaQ3%2`b9PnDOt~rCkI? zy=-Aj-IZ`|7o2rtT+xW+=FNt;vBs^+P^Iq0j4L}%5J~We3-=cnPR;&kvcR;LgR_(%eT;6W zJb^)h+)>E}o!9NLvs2^J@ZUvM@mcAa%3x+{atqHvFoMi5&BrH6v4EbU^P-ES@|)Jz)9?SJ~9_-jVQP-g^vrZpJ-EVeSD+Ft79M^vL3!% zfbQ6b^`$u-8dymu(1&d?GMR6e8ri^CT(lCu>kSH}tv+WsNu1f1*=Rj0GXPhzdb&cm%~@tKQts|I96YX0}CkbvT&neq6J3g*i{NQ9`3+t~+X);Q4Ox zBqhSQS_w~VVByc({4hVa#SkplFgCbzuq{SGk39ulVr_K)1+=qmOWtCM;Cd|^BGJ4! z!wyOfub%d1_$}ErC0BKn=Lmv%G#`(3oX3$QNwRDIOJ^w;$|$C&{bd4W8x?|?ah$<@ zW}qX76{l&Ur<QMwD-&l?+-g-gUpGYOW3B&uE+iFI zx=-gyQFLNYzfQ1z4MxsR+_wvdmB-eCOF006AxP@Ybo^xgrVL?};9HqdrCCkL#5e)0 za^j1apkfuf3>^3bL%6h4?E+3zC}-~2a{8D-q^hRGgkt5M=t3qp8MPzDO*D!zW?i0( zs*HEgLsV}WoQVfXLw#%wyG71cuHMtQ?f@L zQJ$+(t!Hf6Y}fQ7Zf-B+xkoNfn~e)^v1Gf&R0sr8bgda7M$2QKB>dL+cW1kfw(@z^ zj^Q+u3NDQV^2~fgkcg(#RO$9Y-+Xqce>Fxi_=oCD;y=NSI-Da9r&+iuLTNpZ((`w- zs3UY25`0FP?k8sUC_VI0%EWbP4O36uCm;-B;FK{(9%exlrfk>6&E6CqW(RCuW6ndI zBP=?^t{!29cigqTdsmyqnCQ4^*9#_~i>hL{S>DVNL<2ollUk14jP-OdfVh}LK${Yw z*@WvMV8jVkI4)!(_$s4ip~XAeXyy916F;r$;Ntb0^OLgx!KRp&w;pyD2QnvS-OnFg zo(X6eDK*K0zC2KF>WPscIEFc9(dkH|SlvMdU;Ngnfn_j^01LZ)!aeyLBmJ7wdVeTy}2IAf<6-q(A_1LxfS5noDy; zmX*4pm5Qft2p-H-I$z|&i8p!G5lWdHJGQ`L%}Z2)(vue{oz}}S$je`kRtI){Y)~G~Q?SDov zZTl9DA}VGT5Vpc%y$Jv+3tyfpG!V;&Pcj!MremlV;zhX)5qz<<*RW+|z<7H_eB40I zZZT|J5&h<=h`&0Q4<&%7QG2U-1p&3)s*)GZqwa{$Oen4UQE6EwpQ%)40wsk!g~^WH z2{3cjA_|m4^aD$Xl%O>7E=_1e_4F4YdK)D9`o_E(MS6=_-ZEC}(p3@ZYmC)}924q| z{MF}Wre-$G0y4c0V?dbDHFmp?=KBk-^hf#b4fvN4H$R{0iLhI|ZZZtW)S#Qwb0*s`d z578-=Fm)nAxB3YSEth#d2y#+Kb2EX3nYW%|6=ndm2$Pp$6>PUqm>v7FAaz^QN^mr- zPEq~VRLQb(cg!X3C^)`G=hp%H`uO_%+6nik^EcnU{^1S!>Giu` zOX|*VqI2ze5xLQQSJPo&Fpd&f>{L_2#oSwJaXVRx3*|2vFfvK#gs_P0?B=`OyQQVv zeC20hf8RENf!=J8>yXUTnVfksYc2?fMHu)^z5p?VzAx0C+?`yY$x1w*a_$HElvEj!?#E(o~C$0b|EIn zG~r`RlS~+Igp+Z`ir@yEU_&NI#Ah?zd3|{p#6<4wLO+Ciqbmz>7Dm39j^i5&o1|l# zz9A*GLU73q7Q7h&e;K0VNc#jW1!s9RX75@fZ&3U+)uDTNxK=j{JGa|h6dFxAH>;qq z(plPUhc#D((U^ce*+m#A#4I9)iPS8ETI)B5en)1Es(d`=ySIs>9r00OSD_$o+0$l6 zN4{pOg-Vs5bJ0DkwB197=sSL^XF_HwnF|c$=>abIgNzoIc)KBa%x$+tU=i0o)`qk& zqd@Pj%dj=g3yWT?6nWjit_kZo6CE4-tOvXq?;x9TE}NK)35(4aQ=Y^POXpb3z!JIO zI+tQGoHHSc-7>p!jHM((4i~X{wy^EQxi9pVMJy2cFyZ_GTete9jbpmv4l^q}~uPL_ZOP z6UmVvrnOxsbyghOnttOh`G9E`gqf*x#%^nMM2eIZ*dcY4Nc~{|HAzW~QYiI}Wo75n zCckJh36?b9kGGO@`>y%&OtHv;MKq;1&QcbeiCSuBH?Tr`_D-83?W#;#1GKdLBlf0C z<({+a=Ypbb-5^i5-1$!p!c9fA4KSX|uAI9Fo-#WF1*@2MraZdwCj5K^3uU;pTIc#O zR<5Tn$~LuW&x6C^K77(AIByF(Y`WQ&zd%`%mm`nLQiHbO`;L*fi@8B;*MyWvaQ$1_ zlR7BDY?9##K@)yM1gj?X*@coKRF`$Wh7TG2zms1>{{Pk4@#(9x;VfPkj_&;b=TG+! z9#`}KA3uKbkpF)lzenghlFU#-M*^EVVCd9!340%x+}e8d2)zNb$H&F$_tuB{L-b+m z!(cG@u=N3b`Q;hAk&v}@DfqkjmtX3a=;!>*5n(sxR1}nU`epl3%bOi!zz9R=TKBZ96*IBMajTCFO=VMK&=!Xz1zB+^%(b~mqzzf^%8@YP6B%k(9g=rS;0ELizo}}b z^*cQkfRB4&$y-;L{nl<8p%XZ80QIli_;njhp*fro>Zq;(c~i3r*6)-`WgD7I&`?I2 zfd=5k%N9XFL2Ug;)7Qnz&aZVxwAEgPyOKVH^g_|bbiWTfu~IhC(MqTi*)$jzwe9AJ zr*l_jp=Ch!Y2S%Ml1LEMEg-K=rAwU8Iz}4RFB^cu^A@7uUC4Mi#x!aB*1d0mVS8{J zdjE9~2pz;UHLD89s2st~g{^|J{Q{<4O-%dmYY3=9dF}Fc3{-jL38N{$Wg+7u9?!!Y zo@9;6U;I&<5}Q6OBDUoaR=?|ulSFUBgtF{i_?8Gp8avaXZz{4j)y3LmY>J9CpfN8h zur48?6P@xIrmPL5$pZ~zXw(3?^uB`NnnW4g*m+`3X}99wu^!-=xq8&J4MoO6EW=FZ zWnP-F_xrL%VGF-DJ#7`d=F7X`HO^g6jkTQ)$4!b|71XMf3`Phd7L$>0bpE(I7#?zm7$zilB30T$TXzT}bjkK-GGks=&_Adc&_mm6S~qQnd8}Ro^v8=u+?*QB#shbd@Ef zE}t}S5DML(StRYPs`OS{c0uX*)?B3(%1GdpggV~gnbIAXv$zdr=QmC^7d$4&RgbPO zpfW`S!-<{3F&>l9m9WmsT9wvS`+QL4v((CmR(etWO7Vn<`mpgH+b_c69)W~samYnD zosR^K-I~-d+m9Busba4%>S668m!{@5YB8G|36weQZcu}AOOrU#gQ^WmQ-d0mSw4hX z;WRg?K^hC#8fd4dsX+~j^>uA9>aXk4%`+@Y+Q=AK9-PpBLbSHr2Ys=C)>6So37N@0 zSV|xC#o}DYGKi|*^yesyB<;tFZrK;E8ICqVRV=TbuJ4@VU5LXq>;<*ucEN>6TE6Rrv+>@%g%}&Zs^aUs?#M^c*{}+= z$3;9o>>kZs2ft291bf4acXiut04a^4mNr#}mcCQLN3!(Cy|DMF{=JZh9xys+SpoGT zOk0m0q4M^>e)I_Cu*zHfTU+Qjj?s5--du*4SFiu;Z@cjCYx4)>&Rx8=e-L&<1!s_= zcY_6_aLdZoQ(Gt|DPgfhoS~o1g;SC!McG95y)`(0J8*Tt$iNmju3i{y^CSi~T>)5q zWJ=p^ZT;Fov<2W|Cx+#{D#Lg5H-vaYYAh{utJO z4^l4l`2K1mHWOLsCE-HmTeCEa)V5rPHvbgNoq2@<9lHhE!oj~}zmTJbVtpN_k(VOT z&g&2GP_tylg5|A&{KjCR26>G!LkACwQu;jpif~cCHHTi7(M~^v_oBIm{6ddW#+3~X zrMcIhj*&{FJ7Z)%u%!r0(%NcTZlL0xl-`{F=*5O6k)Pl7##_!}hju*z{n1-f zqJg-yv^?;IfbKykjbbP~45O?Rs#)c{Z2N|RduAcGv{(@ zo0+6zH0BwLm*&=i{rtH06N;OehPp+jS%)ibW8a7Wd2`>b`UWYi(i@r(ep0b`8p^Yl`?Apl&G*fLU^I}<~znrT{`dgo~464MgM-0`o0}Dk!B2$0^nf66teKWsiZpP)ECx^Pjvl}98 zsX0uUx|2AnM0=w6>IHf-{LAp*`36FsCKgcXkVUi56Z_7)D9@ewv{$+>Aq?|H?Mhyp zrU42Rkt#qze!-r7MA^h%&AUc(9Udt(6_*b{$oX%8n3gOOe?|+46LOV(0bCh;We67PYbPAZ17S;Cs5l* z!oJrG>ZCgdyOb4Om8(xJ?A&v#N4z<>^>$1T>}Mjb4RueC(R=Tq?Yw@Y z@hk_}mPOLF`m!SJg-n+gO20FOD+@=1H@n3eJSxDOGJvVdFJ-+xd#N$OQ?%bRDymz72?d!xH^;LM#q6t|B#eS$qLyO_C}A-eHh-A17TCza}ZBFkf(OmP5BD zX>>Dk2MP5#&bm%HZF+$9;pa0nSj`78I|RS9%YZv{JS0dXNnOU?CVfRCYrh*ExYN4p zcPZvxBc}$i8hqw*^h&DwBpPyz2dG%rg#Qh`KY#f$K*9CjuEWbWSH~x3>tD!xsS)vU zRSe7OdNUfmU$y-kDyMT1brYj+*Si$MOnA>vH?F?VX))GY`}U9XH|w zM!1L3elaj#FgrOEe1`rC-9Dz`t39-Zpr$*6zY&#o8i^A)#*)eWsu+t3?o62T3`Llk zd|eZFq*y2l_YT=4Tg_iiL_g*BQvI$YHEL_oY}hd!2mw+l?JM zE6$bgS#X0wx_fsQ@1|m}YBdUA*Q85gWL_40_hO?8(c2p=1Za|CgE3EHA_DY0YcvZ(#qRH)HN$mt;SK2V0)E~hGAbcM$q`42fV@AM~m%TdBZ!;ApPpzhI)}j z3ds?Ilo%TzqoT4IIPnC!?ZmlL8wypHd#leI`$dBRH0U?-tC$-_e7}-xmrZBE7GGxZ zKQgJ1ZD5ZrDUQsCa2cMs!3kPgM4be9qO)(-F1R;-m<8w+Y9wv}cVGXC=Vv5|V?ti{MW2F$!G8 zlmG?AfC*6G)^`GPDm*{|X7d4q!yk{$LT&y+h&HgJ_n)}9bE!jh~Y?PeJd5K8Ps(YxEnLZ?Iy;pUw<&JmjC)Dmek z*o{PjT|L@`NvvGlUJEN2IyEn>bhNdFPV^Yc5AdjYN`}bq`VHq@rc`Q|3NR5D$iU2D z)W%hhf=I<^EonlN@(remgzLc^_nIim%q|*2`|$HipfVP55pzaj zGukP0?5wAMm-8UI3o_M+G`H)U1_a#borubNY8z3_)twfRrTMhyWP_N*aE>G%`iFpy zkUxkpi`yHz0j4nRePFzWT@Pm)!%bzV*1>LJKGwiT4y=ufdLgWJ@UTWrSWs>B_bIw% ziiK26>Kp|X$1x~E{F96GKbd@Js77E45h!k~2Z|U)d48&dlcsT&nA@1Z5SU!`lap5{ zM_?}@+nC#1Qa+_}VQuo!jSBDS%SRfHQ6#9M5gc8}#Q{QK13_>+x1EIx`ww&P6WlVD z39?pkl!;wZ9Mz&`Z4#-46iW#uA2+cW#mK?1cOloE{@k7ng~}l|s|8#bt2jJ&1zKUG z*D2Y~P^x!#3nNL}``I9pv?NQ-BMc>w7vt#U`ur$=>WZt}Vu`##VB}=&&@Rl*;pS?_ zQ6wqwgknK&2;5gqwWBtJmZ~*(Mc`*i8Fdw+Ym*#&GjQKjBjHAQx~eoL9xBQ1=)7VrX823U!xg*hccEI8|pW!_2I4c=v29~Uu`z;SMT%Y?CynI zEFb4j2Ih&~Bd|7M*SmS&J7*3z!LM(Ib6F;;SgCI+H(23xarwZevl97BJ+p`2piLc& zQg`Irq>a$Ap3BgFOKOX0af=s!EeI&#`wIP>*A+poPKm8%A{Q^U{djqtvsPi`XYvlu z(xmqpczdvY@8Dp6K&7I5uLQ4l5MFCq@FYzzBZHi~LJm8i>9bbmYSaHE$~CS>!#Z9d zJ{es&c`LGyrd-j>Cf(em7_RkfQ;Cj<%^DyQWK0BMJ&PK3Qj(873mT`? z9?APEfA*HLxcCcB1?PX|+sR*bXms)7j{$akd7d8#^d5gtS?vFP;kGp1RA?wRJAVIN zrKNP~)|NY{YfD1i5CIF=QJNuW4+jVG_N(`h76JnF*BsSX&8d9-3!}DjV_Fvsx%ycv z^3sSrK!5eov@p|Ig0)P)-|5UcY~{mV40Iyg)P87>Vz4z`x%A09bKC>@T7$&)a<bp0C%0sU3+fSE3BfaJA4a!r#wDpin>i4^QzcKsrH$rE9{nz zFuQMy8I^W$?hBx2iv4N4OVi~H8C-T&-$&j{lNxMfl&xA0xLI)i;oDCp#2(bwI<;#Y z6#op>JS9aJZDc zVQyr3R8em|NM&qo0POwyb{jXcD2(sF^%R(PW{s4eNL_rJ(HUp`6iG>RV$1p@C)u+y z`8r@XNMhUsIsjTS$M#z1AtZsDOP^c;Zg+ffCw^N=; z5+-yMPDwJ`JDFmkhI2ei{%bwI{r&y@XHTBM|L*VaSO0ha@Zj*j4xT(a+<*M!@w2DT z|7-u?>HhxX|3dp~!r%GJq{8CA_V0|V%G~edhg13^5t4Ftgl-PEVj?3!Qw4v2N0J%J z6ip;jWR@ma5fqa#WmLbHXv_sVb^3#5JkAm#hg%HK$Pub5)|Q&57=i;^rRDA1!8hHrVehBM6Qm`Ei9w4G3?&=7qQaWTl?b z@J#uL8s2){`fPu{q)g8CGytl;>VRT1k0-Vz~OB;?+D zunzW{3#9J^I@0Thh1_}((3xlqT~Md*Zsfx zX8mn8B7zY`oDb6u;q09vg5l({_#&tiHhhKu#YNL|%DC8R9;Ugytb128!3iTin{gIW z-7sW49Ala|;&PwZ$jpw8kL?5UYXW+;?Kvp2KK59 zl(_SZT~7|Li!biPWfk)pIBjC4#BC`8Ucm25zwMnktz%sSug$FZ+|qG_NdT(y_uDO| zj+n^om01fxAJ7y_6xaYT+#{MUhg@0c)4eiwO5 z;Id832_BI|)&#p9(@3EJr)f}++Jjpz-X-?xiwi*N&x{R)2+b%vP7;1g;v@T{BKwxv zbT0HlNPl%2iqH(dt3ieN<>CG;e^8odHWTd3dv9u+77#W;svD%0xtoPrXT-Ma%}2x9 z!=~7M35~EcLK4KdnD3}mT+Cn68C6H|B@1hRvfv#oML$?$e6xEL6q<;9)Qn`i5LX! zvIfA6afM+k%Nt+-!c2Y7 zCq(Q?Ed{f=8tnMvUrzDi(`Pc9`P_M?({-KxO}SJTMCp*~2o@IY^O#8+n4ps+({a={ zR7mv`RnzaxC$=F($d_4?Tmv~d`vX+jyBi2Hp;C$YR)bB;RWucp3##T}lwhfKRAp&G z3hn|qmx{~=ks)dj2?`Nx;4A2}qGOt|Fdmc8X=DQu7db8iQ}KidqXG>OF`ExGRvo@l zH63IOwmxw}|4rgLBIm!CnHlII%f@cau$@qA1yUY|l0=!2w!ssMj43)JpE(6EUz zl1Oq#EFqR*CW#1h{b<8g=&d#);)X`#IEr}2lyRYvr1ov@TWd<_g}%)`wLbl`pc;rL zI1V{W=AqzRg=3l!qmyL@Jm+f%iLpGj$>0a{TgDafn`1HY%*X%*gMl&jo!+`93{ zk3qTp&5Ng%Ct5vl919}li!Zj1PfxGTuCI3jZ=PD+fbb{+2xEd(Cdgod6_GFA*w!z! z)u`Z-O3+9R96*PptT)|IVLKxKkkzTNDmoZT?G#>+cj`%niiBWox?z}8&YB3VG#TU` z_Qe<57ss#8t}l;I&UPvvnc+8xaEc?U=2bb{#{+)Lh!A8vNXQLIUL5+O?%1oT@(E1` z#^DW|gUfkolg=$Hnh(@etE;T4jayRp)rPPeU+2ugv1*b(N9aadq9e2yNvi*UD0%d@ z_Sxm@(@=}A`rS8z&yKu@2#qO8;(P{P{7Shb2bGubk#>(qxyK)B#5FqdF1%+#K!x(Q zH{K$AUBp^|L=9ppJm53k*Z|%u3|z5jAbm0g`7I+-$ujAk_Kl=dG9v;fp^nq=goFv7 z$WXhc5EL#z2m78iJ2>>>$}wVGq3z|o)53MDR@(coTliF#28ENgTU7YePN2MazPtyh zZe!lb{CkZyWicD9QFW(rQ>N&_1pEO^K+na%*75IwY+dRry2HNz)JCGT1GT)3sAMVp zK1_u4Nq;a652j(EY3R4U-or9<7>xE+bg!d|HlNyX)HZdOHy#bCNxu&RQi?Gel87E` z$@*={AIO^Y-_KgsqPR{d?KgdDOVN-N(a%=&8tGG|(g9&{$|+MXzS#cm_4S+Z{Bq~W zM_r_eZE9#*w5tzHZ|q5hRVIg1E|mxI!G+&x&@HnCyKstMY?Q2N5qtb>%^#}T4t`5idGw}c>`w_!)b!uwS6ryFefEvyCajJ<5AqEc z2s4ziw~XI1BnXzAp_}V;N(4dWEn#%a*zY1sT|y6aeK8c8k8EH#9>JEPpPcb|BF z_~+PGILIbOIMOa~xkGE_tA;eFJpbvsv^Cw3&SUItErsVUsXN0Zb9S~Kp%=w(>rrW6 zy!!4MhA22Md9=4Tp=z3qh7q6b70K3m9VM)MUoyBk~wOkc(alh+J+u&4fG`c zTfwtb9-*Iu;Fm4m7oeYmRC6sAVai${XmUfu2wqQ!3U+nCEa87|v5KbpA7*!0|0^a5 zQ6%_fc~nt?>8vL{lhE7;i%C-I1GWatx-{F{4WP|?NEGTNs2=2!8^Ws8c1YXGVHX8y z3lDZZVYrAH+ugQ#2RG|N0Y0O~Yd59^2qH-tkJr@|{XsoK$5BM2M2cI-ZT5?1nmdd$$Nr|{X3Bi&`|5!R2VM$`d z+14X;tB1B8i$uYvzt%S=)I24>4$)Z-6bpi4k_r-GMPfu58si%-G(|4~S#1P2_#p?a zajfQ^$PHTS^-CVfda0VFjW`!EWq!BU@f)eE0r?|zsV$q7d6l1&>+@@4u}*BJ97xQ! z(J=MYeWrUb%|;28Q*Cx0X&G8`hg_Z=qHN5?%9u& zd-evUo)|_X;kUrGWmwDQ7AJ2t&shp=L-wGY(iBBgOtmgH3vZ|~7!k^BJf;!VgF?5B z^8*LyhGHGjE3C*kORkAR+qt~KZBB?nmptA6x-zN%8l2*2 zOwDue^PL`=p7OXs;x(K5#}!y6SxNF^*gYHoAzWS7q&icI;6E3ke-_?-$stiV@vTRBZms`pdB1L<{7&3@kx%^l_3ABtZ;i{! zePB1v6;^`ONM*NE-u&9}=iv75MOi<4uU|;v^KkkFJr6$rzI^^?BJ>4t-*_xHUjlH!W71y!KY!cyX?w~4{B7H(on`+02HgO2wfD8Z7;xJx#fTs>fCy zik>g7zv}8m_AGZ!^l&mCt*_)cCNs{i$)~S)uK606F)_#kirzJo7ltFNmQQ}VRJCX5 zoTX^qFo~4{&(%)X(lL}Tt`z9XPT??3FGA=X~>|H8|$z841 zMi?NT5`neM&^fETMCdJ<9|4k*vD(wpueF7$3^fEA#ff%kXYQNV}P~x7s^`k85gBc78rd9n_SLqJE+R%Z9a;sw*#737K&l zS}cK~Z*gJ4c~cEz1NzRTuiNVlH#)1$m2?R6BT1CVFcv%wagr?Z7^NZ)jnQVmS}=?1 zK%HElhi4b3m#@z+-c(D(=pZ~HDi;!*s7SGzzW8GM^y>V_v#apx?A!4A^~v|)*02niZG=2EEQ1{N%VTG{Zqf$J(NFy(UI_p>{0mOp6VW~ z5_&GIa-)IfD4n4=F>rhV7UgD2CayH}ixr=iab7?3UtS3*S~WH4Dvi;=VKcI2s<`Cs zm&Ckim^fS2uIv@NP-`W4iYM4yh#q`RO<@P1xo#U)8k8})p%Ll$C?+>8WgMCpmRbee zyZXszU;pGj&AKzPc|E(8}M5bY7tTe*v?!=PCXkH8~HPivCi=xiQ7hGKm zBK_Dr-_8xe{uY3QOup1eUxd>Ci-_2Lu1@7Me-G6-ee80T^dw8&a`1E1_ z_db3-He1U|P_M(Z_a!=xWAxpdH<#h%)$9NM$1eQ)+WY}Wk1t-^KM1>_f-`f#84IfY zwK-y*`;iLhn52Zo5^;up1_F>IN>MhEzihiS)Y3Xt%%e=h2V1*$b5Ojs*LuOlH6~z~ zs#)?V#RAWWB0>&~=aBX(Hu)ZKFFVC5BEbM@HDXXXAP^9s;C=j*WOm!?NH zfu+$>-%(JqQBv1y6P4_gm0X*!>UcV&^#$Xm`Oynxwt^21pAYwk`@@3(1qaXmHavX# zWN804Sc)S1@RZLmWy|rx|CJke}Nxiziz0ct1;kO|Eo6^ZNw0EkS>RGFVfQ#b4Cr_VL z05O}Z`s;1;dBwteiA1#>DLf-dge3{JCHAt2cDxiYb9wIfWM0J>=>9qO zhJBFNuyy{d^ue&&K4%g{=Kg>_&fhRzykM~>GHB+}*67yy(Aew`*H3FdyhZk0;gOZ# z$z_5WIhWtTkT`ZD;%-#gzq+z&O-5H;fu=aL%cubwb_{MPJ8aZ<7u`;2G}QxJ!BR|& zMsqabd5Q(X62aY|odep(youS=A8K^jfGuEL*{+aEP&Nrl0n{fQR4b9Vsj;*)F_j@S z_=Cor-SOYsY($`r_MH|07sP)DPoExCamlC3imFVF~Y-0 zs3Y{j8oD-PFs4zl#NnZu@UVRa#OHGeMVg1Pkr|lV=tC|9Jn&lLz~MAHN>I8vIYUx6ja zB-l&1?8#dPiZZGAEKe=oj)+i$8D@AwM0cK6C-wIgy=!a>qjw_>+o-ZQgKJ!w-JF>_1F~SFZup zZT}CR9UNBkKlYzIeOUjwmtTMT?@!ko!boG?k!cKTIi|5$N(m4bc2T7}tD^x~^8)!= zQft4M@tAn)A@{OHTr#O`Tf|0EmEF+B#wPm`1v5CGUA}AB@b{weiy^%8F_f>dFMZ#= zJmkSEReBn<=JR=&)`rKQE)@bMzoFoiN}E^kMTuz*uW9oc@N{eh*Jop2m}J^QVHy%E zycXts`&jT990*xZsjU2E$8fl3^{tK`M->t3l?l!4S^RS;GgD8cKGr^?G2rWr`6@Um zcnO+r#fQ^~og{hE2zE*!43G#b6#U(M`jw^1`^F_Wcp{_{%`l5GTrUoHk5)bh((MI^ zilY(_AJ7=nq{d2-{7_=AgEOB<6x65;8d&YtjFukimPL&Ze;9w>Lash(Ais@thr5$P z-N2ZVxJFTXzKdr#Axq|qnwMDKR#!&w!qL@BSNdS@r#pRw;l{b8+}5uv{7RXX-~^_t zCw^{0@R}v_bN0j4d9brn!VNwA1AEK(EklLaj-0nD^<6W0n8l`&OL-Lb)f%=PoG3wuI*DN-lcOD#F!lN>bncP1JqbW`j!X_k? zJh{<>y*8Rw@1hPY8|*g|Tir=KmB8HjB*8O9o<0tAOvbZ#oM557PfRCN;e;psdy zTSE&|l99c{C^Q=(kkHeWr)*jqasO<$pFA8JEIt5$2L`2wai>BkWiAtcO4Yy1t88m$t#xkbv%Pm(d@z6d?8q%Tuk}a;evo&TUHjlvB}FB2Buyn{axIP zRPD}FtT5E{`QMUvY;~?=*nGj{wydB1$|p57lVudYw>>E7!F||(?vZ@(2}g3%GF2PN zgU<78z08AYU8JMdeL>KHx^*QH)>{0Pn^`6m8WB_@Mf?3$rvEd~$^B7(z4rg1I^d@E z|M#CiKX_W(|KC4+*#EzeUw_X8O5i7a<@urmZG<{gzTn&5{>NfkcrNbF9;bBGL)qDU zKuXZragw0mpKxc9LzdUjEU{CH=^V5bBDc~?_e48?!t|W=F=|Az436zQCWd3`j-S`5 zsZ^LR(AT&kLti3u=|BrrfAVK9Kf;wmD8G;GG)e z6q`gvA9Cw$CEj#)T^tKOv!@i6*pM&N@H;e+cefsakwI`Dpjo?|Jq7_rcT059>eo@mpfzQQp7r$ft6< zd^(QY!y0tw|1WyNtvci@278eziwedyq{~)wV62d8&)?Y z7S#ZBcBm6wck*o;`iHH8&+pgM{!>+4a=5euSYZF39voKBe;+bj72$*F2%svJkyoj<0uNH08G}WPHTqd1x+L@&aV@0D|Mr*x0`Z%hneJz!C`( zWkS*$lFWfUxU3U$>tlNr5GB&fyD=F1`KiT>^eABM-kVRngbo)Qn_LWcPnX6ruwr@t zzWf;gE3OAyYF6#t3!9{rt>FVKMoSLoT(r;nc;!GF=7nx%Vn zqJm$xTJA;^89SC2oLzCQj!ZS-;`tAfh$FOLJor}dEHzK67X%Kx1eBUY$~W&d)3{Z; z;;eePS!bfkg8D3#%b`|WByQ5T!qor%$4;G>|A681p)IVYFKv2YIX+klJ2vQtLGP@yA ztw}0@X;I;CsXCdT#$cyM z_Tj=?&2xiQR#IDH)?SU9-!Xl5b#eAGJb8U_{pRY}U5o8qp;MP*DT{t%e9e`Jc(HT{!`4;ricUE&uE3ljjffzx(+0cZ%f%>mk~XX{2MAG!0Oo^8J{U zGs2X8MA;F|rW_8|kkk1KO@Y~Ed&i!JR(Dl%RxWXJKzp4G<@UP&xT_*rPd#iAZdalf`UPSf-pcIP|g$j zfHJ12N=Wsv>8I|Jd;Rqq|E1EB>)-!9d|bW%;qc(#Vf^3AuYdg4SYLFh!ms(g4~^Fe zz|scXql?DB#jH;#SRtt%5yJ1YtLEzJ2}SOEn)u`EpeJts)c7b*x%oW|8%&7OCDdBw=0Tz;1U+UFL`VX zvpvwPQNg~_MDQ%_5NAObPBM3(kk#IjFZ2sk&yS-NPR_E{Y=~;xv_tMp+J)})^u7_Y9UFU(j_s)9q z^zqYbXTh^g_szYHL{GZ+HW<6O{Cv9DACH^tbHTaVQ19-qrMnD)o9!5vN>WKADW{RF zw*wTNkXl^1-+pOnUIjk2q)$O#;$%@&?#P~jLkIVx0epbS&qV}UcXaJD|68z z_BNW_cv!o%1;Me&#L$B}g!H<(RAgo>u_TZPvqq_Fgyvp0jSv+G!AfyBm25t|K6|Ng zLz)s+bVL)X=6CCrzTB?ev=JP?X31Qewr_Hzx%XMr>v}kXEX<*8n{bnm8xe`i>b>4#Bpr?F+kV0Uwq3OzTVT%aF&go;E{p|Fb#%2<-!2`C;3O$2JJM)V z68EakyiO)RGZ)ol0yMGC1{}Y9`N=gS#4HYR90#b#D>QZNzdwKZGC+YgRF`kAj!(`y z6A9N8zU+cBIsE}E&u8xH>5`7!t7;WeTE`Z}OD$hMZsh0FXrXgXX^fS`N z`%$^iPqF@Rnse99&`lVzT~H~1@AUk1W194NZmO=bf7-VEe6Jq)KKB3n<$u_dK%1%m z^7zS&_P8jR|BUYo43v(dG zFO*}C`;vDlW|DFSRrW(Z9w)l{j$YVx`vRgpuHqdwfLrJx@Pdn~2JUX70seOe*I88_ z&#~sT{`^L!Q_}RoO%w;^B9rcN$DD+I7bcbpA}(P41^9_Y=V-iYV%^~UQ$nGZm?Wy) zJAoY<#%`A`s+*+mfEuqsXu4;k;p<{_l%?H= z>GDk>-i8OF*rafLtFTTdq+b44inOtim%Ym zQ6>ao@YgRLQ(rw}Ra1g;90P-c*==UQXCBzS3KS0sqS9_o;{^TB|M|aq&Ktw@IVTkE zCZ6*QMVRT=h?+&`6bnUhlFZSBFe0#`Y*G-^9g6mr(V~cS4aZZom`F?n&v#YmZg})YkThE@NJom(6$fd z1CnV%6)aK(dja}@62f-S!49g4t_sCr?*n4Ylh{;-l?50a#|LHTJ0ATu{!Ou(?pimQ z`+itu7=D}Zk*$bal1#Aoztq5Xu*Ut&<)E z9e^~Sr3W50`u_%j|72Ku62g;i1Tvu!spIJ(7*n2o5q$5eu+m8~@2qU;n;jbyH5hwI z*(4!_9jjjfSVHbP-&38PwyDDxG8>&zA@j*ybpq%QfSU5Pj{r9W$}-Zxu0igtgP{C@ zqwWR5X{ssDpVkkt{a}ioxIQXf>uFTb`5w#o;YQYCsW(ZQjXGBd&CI^uo zVmqz_*4N=drz9J6bJ_lHGZaG<6di1qkpg2R0`x`xrZE&W7w=+ZKo@$_+3j^YdW2ZT zdKeqODpe)wE<+xXw~ARHpA6)2RGq*;Tu*AKS0rzb|3-8pEAZ)!SOjD?{$; z)^Cm4FA$>j)rXbgkO7t#L#k0$q?{}-JR3OLN&e;a7Q`QL+QwfX6_`5t$mt6Y!pu0MtmRr+3?^E2G4uwPoX<)8> z+2F8D=asphuyN6#X)&PU$44k&F!)IqjEYuHj~Wc5WQUMLm@ z!t0Z(ah-c7H;trHs(4KI3?#y5pjbP^a0tO5%hQ;LZ1%2 z{~aDS;GSj4+6aI<3m4Gvc+)gIUQEN2P1EpX$E~gGeW|tO602y6$h%Y!XLD|#yJMSARqmc{TXzrF>F0-8>81sWD$5JJ$o&E ztDP=vgYA?@aP8~iS#hr#@^6{nZfp7hDSi!g;dUDL>`I1BdnsGEvI6YN?Y(=5?ME?* z;ST%w%6+ZD3`Kdj-sj0{KlFDdYdffVqOYykMB9k=?}CP&4R-9-W#ux*+SX{9{6K4t zr?K0AmOGHp_xfUc2W@Z9@Vjk|O&T;H)<3<_j)e=;29G3Rs@iBLs4}1tHC!}V+SzT} z?5$YG^55;ovecLP)@#YmF0>KN^>3c{=dr12P=u;?T@^3|hYNdL{lnCPE0rRK!QZh$0B7!PKIVcIfcst8_G7gF zgyrqUQ_W&UyU?ihFK$=qXq@P-JfHQJX1$eQXwbdHT_@}JH_u#twVbDVPR_+(iDmQE z6e;NFnFUMc%PEz@VnS3XA(RXioe`d?P?CtV*jG0Drc>48e`Qi3+tOST0kt#%Hli+0 z?X6i0XzQbKchxt?Oq;g70TOebDMOaLT2d|WkjubqQLDw7bv>i~%CbdHcMY+Td;R&x zkJ3*xbj=(b@Aal?taq>fG9Hrb5~z@4BD zW>_jB0#puz%w*F=!5MqY_$|xnM*j@{eDlu$Dv&pqtsRy!Jv}>qWvy#jAlkRxa`86d zIF=|T5Jz#=Q&1(z+{Y!_xg09@XV+a*&ps?@a?f@VX9V%_5u68NE^QLt#m;JBTDT(NqMi)FLCp=^7>-iN= zNQ0=f2CE&DdtahePdj-`27)J~yk-3UeZ$=Nb)RpN#(Zq9K%zVO z=+jo31v*N(fJHM;FLO(=Q+;FU*?CacKZ?3Gszk+1S-(*3^Zs0+AWAX~;uQb_uKKEU z`M8NjBzQ7N6U@j4hn?ei1ySD@i5syD3zuYE)TMYr1THGf^^Um~%)mXk0oqnsnh-Rs z1SD!Y$QUgC;e`I1#P%W2)WDup8l-}c5;BvuyDn-WUkTwoOq^Ga_&h=zn^^tpuIb;qNqVgmuvc z+O#ZbKba5v(w5_BC$G&YduHIMI1TiZ9lA$oLT`vfDG@UY+u72rG)@T|;)lX`b`il; zwqbLmFl9=j?M&K-5zfppf$kwy*b6@}aLN|u+UDAoZLOaP(lS!^zC_2C+M|MQYVKaP zAw|K4Sy`~maW*>EIiqYM zh?D~x)FnjJRI8cL!tE|1)3xkJGVM^TWIGpyf1xYLrR)-yyt(cxNd zp42=g)d09yx>ShFQZ+~BB1U5YC+2bSk@Xb6WC^*ric={7mnGJkh_z5heq%jeQaW*jATY%CQcQsp& zs8T0n_@%S4SfcMr!fGC?C9Sdgbpt1j`jruB3LGjl1sY;0Q>TUfMHxGNb+5cfP?`!< zmG2u&6RFA12fHSXGE*9h#3s&OVmqOEV$!{t;G*%xYtoJLlTB|iI9Yw|3lglY_RATI zxQIC;aUPIZg!_Myay%Ghup7M&LQ(qpWh{g&Zm z3z?J>AlA<-^b~DnkUmp)fMR96kq$3h$l7|++y7Y9kImux=hllkLmTOVwT~okm#JI& zP1BFhP4-Q?ztuVin(GLrw75Me&dHrzoc~3FRfB1CD)_rOQrv`YC}Px`2Ih-Cuoe%2 z3Lfhj#L3Aklp`>a`=%G@d&Dx$Q?!aG%CSv(Q=`x^o+V9~3EA;!Z<(){z9pI-)2Ou}N3c-RsiG-?>ce8sn(hDp0n7zwIqgcDz(K#kZy z5FF2~UEsq0BZG^m;S5bCY7^cyC{&r`l@+QYwTeQOBVfjrVYZ_U&(wgA2QigzJ6g@i zjEnieiKnBb2GplrO5;AwGA0O)rg|dV#VAp?T)Z7*4BudyK4)Go`X&A8DR10*vN)nkO}D+NDV^BJU!SDlc?hze+B(-?&gQU5VT#-PFPvCX|h+ zf`cCkmum1^#zi)3Ycl2{A_K|?LJF5cGPEKBJK;WG;uG|gAXFI43 zF+}e0HKWZdo9FUtuY;;R3K$%Mn#pa8Lrljmoi5&;|DEzUjHwhEoYoIVSv(=j-u}O^ z{^$Oa{c8UA;p4-H`k(jm>oMHYcofE%%s7*z`My=R<#2iWHJagf$aD4Gr%uX{8xv%P zejWAY6Juuh-46vdN9bVx&;KEnef#|ujDJD7x%*?an;*zHB&2&a@ax0NacGZP5H9<*?3c!*?s#Hj)gb5Z}m7ZpHd5At7mCreQ81%1x ziIB^YxIE5jtx)zed533dLVnqvDwWEky}i=ZJ?2rS>43(2HwSxm3wZC~V1GcRGMnaJ zLd;anl1IfAaW0DtcGL$sBL#yywJPstt$k9h^q%*gTSD-Ygc0)XcO!oFOO@L|4O6Y)c&arqadL( zS{YXL`6P&Ik>7p^|{q2TQ5>h;wTa>=H-V=2Ul zW^in!CO5FAA57BfppxQS4>_8WMwa&dHz^KfN}_xcUj!q@^uhz$``-fOVc4jKd}v7xgO&42GE|04KVV4Y`XBE zHkIlIic?h?Kn$j+V(}ij4c-*h6uo|{POcip`;HBtK7QOX z#YQiALZi7R$^88aY=s8gU`36=$TQI#y9JEAKjD$Rw|Y=KzNnC0sEV~eGyLw@Wd9#o zLl>o48$A#?;CHB;$P8e>8n^OcodteZ{c=XxlIX#)u7th*Wq!6RHcPoj=*l|zV-|yJ zk(Kj_U40n59*={&l{Q>1w)?w|mEldplx<3ql$feW5ob37Bw%eA7I6)YxV)nvc%l(Kj?FQYw50O ziI@IvL(%2>#{Won4**-fXIf}lwOigU?3SVh`e4N0EO$H0^kBbQ2>VqB?1kXJT?qc$ zZtx2+yjh6h%|Z+dkz6iBa@hyTLOf>+@tpO6J@NEK`DQW7H_M~!i~Vvj_RBkA?@P$pVnWW=MMx)2Q1Sc~Tq6+tzd!E}{`SlN z?~6v#7W=KHxux|8V=iV`9eE6SQ^*#V*2AOeNz&VFL_#sbQ3xjr{Pop_R3guB`7;1> z8odIl({U|8U0_^()4vw{!g&de#~hm2st29#8>PgUoXv2p8tM^?mGVaoYAh3h|*;bpdKJ~Wjz5TlP|7A*|aLh$GrITrB)97fj zWHI0c`~L@r`_HQPza2h5eAxfLk6#a?kn%XhOwqg$w9yD$)G06B%%F%==C*>Cg}Ri- z$bm8ob)DH3NpT=BI3O4ks||T_LHNSEk_KA=BA zr!2wo#=EE_(8KkJqM3&GZDk<~{0E!Ezhmf#}^*HO0%OTNLrT@)B3Z%`Iz6$)p zvbBn}FUqZl`HMP&jk>o|jRmM+chaJavV{x@bB7ISpC zzyGB5qp4Eq(ca$e?d=c)njsgHy~LoBdypvm=IrX#H9EdHMJKN>PS4+*zrMIe-@Lv; zKU|;fqN}sZtJkMLoao29(CYO3`pwn(*FWed0N`MVPV?)0<L%Cb_=-daNKFOL zCR6k`#BDKVm#kQ&i;Egz5l`oWPNoX+TSf%p0uiRDnj@U4DHpbyid#@^_yVf$QHdr3 zGsp*0MbCN!kO@xE8Gx=4kul96P)`t!0E(l6#Rw;f1p{Zd%_~ZzL2Z-ab`ciD%Zx*# z`L5G{CJg$RHOru|qG6m5aB=?gT zuBhN+b*t4Y&nJ$_E-EDoTQH`&{7kJ^B}kRGoCpOg04)L}m$$4|7!m ziKdv*2)hBI1ZL9YU08*JCyD)Yj4(3d0dTuz1}%&Vn>vi8P?HgevusWXgL0d3nPK00 znH0hNMh`P~XC$T=Vn2`VpSXBiGYYp{yakHE4rxs*h5%)bjeGzY(OI@;I3}nl7h}f1 zXP0&n6!o%&F|{h;+%7ol#<-#p$<3P$Z)1&Hn-!(BZvhE8YAuLu%#ckcHwOrz=u+%% z0Wo*N(0ArKjh#XYr$o>jtmq9v0mvOU;#wdwsA_=5(JTYYy@h0C;0VtTEa;&aYaCkp z1)mvPrBOk5!JSx8PpdEnB!)`5I6;EkQ0Pjn#f&REP7q1(hzs`@7p|NA(PV*XF9&BS zLHZcoPI&@@0=c7-4Sim>$IecTOT&K`RmEqeXDWl4smU!o2f+w3!!#eCB*g+WMau=) zm=QseIZ7yd3xXR_271EqjO@5hhqCfx9D$SA^?YP5)EZH6T?!u;y?&xix%KglJzgCH z`H=PSCnJReF8dci;>BEyVS@AzT%>l_+4*MC~fsQ!%5=Iw#-H|s;u#F zafiK{BuX+M&khAZ1a`a@R=8dUch~7H{6IxpWk9d>eMF`>86!S!4_^9tdlck61$pMa z_V>A2=3@j~-hwk4?do0_;RN*PR_N~-1Roi*(nop}_(CUzptPKnECvh6U+#9gqTF`* zI5_iYLNiSDyc(`bkh{Li%0nSj^;h~PoJyGy?Ls2(k+y~De(m{8w35eKzR-6)Gg8t5 zPuf~;F_lp!!OH{28Q4o3M*ajAuW(J|ofATtSEr8=XELQx#xt4B(F}{XB+iYJj*eW& zMI@atu>F)lAAw*SHBlR@;DRfJkv~?4L462U;_IAkHv$)iJiY+6eVkR0LT#QAAq)u; z5wPT=c?nN32qpO~BTOY4ZNx>&jl0$XuQ&XRAvzqQZ*^Fu(Vpb&yT}S%XU4f&U2UAq zc*D%M$^=JK_#7u{J{Su^b`ceV-0%pB!B)Mwo&T9z`pj&L?CNkZ)p}gBbcH!jvr$5$WUdu8 zP4IlTc#;xfT&;vBHn8yLZGM=a+hPcoYZx2cIoKA{L65x-y2RS({sDSt+m^h=62bLa zHbkO%afTg~8eYBLo8h-)*OXk^L_rn-gS;|v7R9aZ3g_#F=zOgE zKc5RpMU~d+yi*jN*we2QtY3qXvlI92!eQmHwct_?Kwt=xdNUnAnZGGR7$x{tW>jfb z6EZPQ0IQt%A||L<#V!K}KEV(!?Nqyf6BV5^cWk--m_ekfro@C|WleM;lbej%k>VyA z#Tc_L&qYLs#NP4TQ=J@dc@7`g*^AjUeWkwBiAZwM06l$t8tUg(?84)w3bCOKKs5Cf--Ir1G^fFo3w2 zLqMAnq1lA%Az;J_RX8qWBls$#Wue78+GyqdZzq1btAmTzZ_ZE70tB04THbotSscim zm~}sYba^JAVWiX~3;pGRa#K%?1i>-PF$<+5jbe2N6@2kqqXw42Faj*>_K~Xxrdh&` zas_3>3MB+fJ)QDb>Fp=QXoVZ4*F8Wu!m=aYKaZEH$Mf9%v@#dEYQDW_rS`0}F!1 z5WSh&?OIk+Ln{@pzae-qQ|Ww>4=3K_RYxdga_rawi#0E;&F5|d$Fcsm(DNN%BRwdN zbSt4gBVpG_U(&dwIxwHYOd};M&SoxbEoqE1EN1@ay0dE2K)~LW3yp&bR;$p++}5Z> zR@FZvn6`b3MiCXW3J6_D>M+xhfgvWD5hhm7ve>^4H0~?wAZj@WWacP zMSR>q&2BMlToL`|sffQimk%X?r%`*Wc?AKr-Kvro&ZF*#&rB$-^{BKglh0HtGl5cv zJcY@Q-3c&r)glU%L-Yenh?LN2pX5u}HK*CiH_i7w0nsUpRcXd;MF{J9+=Fe4*y93&rNV0d$Z2r|LK>?>909(FI_ z*$};P0f@U&#l_JvC+3<52)Vr-AuV{jp=QDtw~c{kM0yd8)DuE+L!smm{K*BfI&2y{ zrF9hV7OPHnn(EHPfKCX4C5QYL)1{Dbi^UwMp$x@DMuLuPSLBlpI-<SvQ(ozgch4 z-~x=Ko)6I}l`wT8Lbv(}3oVy;=ox5W$aYw=NH9Efz(AUS;=hsfSKb^n%?)49E&`-x#SH~A`&d;vV z>nnd-{`EKL_~IYv`}2#_T|}ta0e%NJ2^E}Buq&~*Oj?W}SSQ8KPR-G+5fvqsaa3$9 zyg7gK@@yAfyuKKmUwm_Qe(~+utFwzYyXe)~)ya3q7jKTgK7V=s<{zL!-<-d>IJ-9K z1IHG|<^OGN59$%r$AFeK6U!NIoYjz70lIXdbOgWPjb{$}6#>@)K>Vnf$@KjJ8 zo&&>Ul)+jl=wD%Rz4cPF+9+iuI?7CY6Ha549mTHH>S;zIdL28>J+Iw33~JG=RA z_ikw^H(&W#sPEe*P>?7l2^~Q|2gFSTmoneCa3~eRk&>`G-aI~x^(?s$F3`wzSwaD^ zT@iq;#WOr9ui@*jT}nWa8~}Ib7mI9^MKsodt=US@0gPGGr8sdworR93Sc`}VgoW8N z*IqpL)iO&|b)pN>%yJu(nJ1Ll&h+f6zhbc6xhL+Z(rhQ(DC~su_?9OA+TmLy6;D$< zA-fP0WSa0Xrb#A@H^Rv{V?}TSPOu>pB;vD~R$gBo1~HL4yP$_~Z**lL&cetS({X%5 zVUu)h(>J7~RtPS+!Gbp<;BP~89BH4RrQj^D#_U~dpm$`(sLIDs*S(aL$A%cFXL_F_w}DJ$&pU>TZ4Acr>@cM#0crOQ4Wx9?ai*+USLiIW0zk z&n`~2Uu{Z*gZIammuDBJ=l^}AyAW0j(lnXd^aVe4Prn0FZu7k)gwUIwAMDyRit=iL z3#&Ozh%mX#W;VB5%;d+EB(X$i~X{wy^EQxi9pVMJy2cFyZ_GTete9jbpmv4l^q}~uP zL_ZOP6UmVvrnOxsbyghO8ohCse898|!pzh;W4ErdZ^_BAU`0XDN%#L@l+m8(5(|d#6p2c2y>=0a{x9 zh`s4jS#x&%Tu`*F8|3MhJO4|Aa8nU&1B~agE9WkPr_9bk!7Ap;lt(w-grARKp$wN+ z>zoc_YxO(Nroo`P52EFteVtk7fOmyUDo*;K4kR&PJRvf|5sa{Qg04n5YT>9_tt!G{M1*z1C3?DBI(+7$ zFLdhfnxG#9tsyoY6_&PF!c0&fWm}sHvf@&hYjcxG8?xG!BVocPGSsd*B=3~K4agUM zQ|+17@AOmvKJJAjZ(U*bTf1q5PT;@+)W2@y*KII`=5Ru&qq+vEeG zHMkAE|GEc+4q}>`RRv_a9Kp?nt%9<80n@G~rv3Ld1XQ8Cc6mDns=V@q(Ujk^kns_Z z=iv=cvc}F|{83vcHhow`Y|A06e%BW#iQa|@W!by%EfI_~D$}BGDzY}!#oAb)Jd16j!x8mTj9^jd|depQHMaDub z!%XI7UYfA?`?96O7Jh9sZ56!c%e&z<&RtK9wVe*fO&z-`s8uP;hr%kf)uU})>fZ=_sCAGC2ko4<{hDTD7S&k*0u#=1}A;l1br*;7&DAcv7C-L z7IB?|iD?qXu1Q#&mXx8;l8_az*YlHV2X=ne8-5k4q->IqqOA|8`mRAjmx9lTnvzVS zt1KaP`J{P+Q0NBDB57|`rMKF$3rfeg<|?gFMgpfK)bS3_lvZ5M;x?F_-#FP^@R%T1 zJ-WVt$`la{Cw207RlG5a~^dg5TC^xG_je$UFU)Pr=%WoXGtNiYiS9z788qmhO*-BTK z`-m2|uG+jM?n+;XsQI&A7~M^)3vsxHy`Z+-F1Qd$%XhtSHr|`J5M$$4ReYV-9r-9a z8&;wAxQNGx-J`iv@avRBus6JTSGVm3kkTk>X;Wor={psCBujtX3ww|1-wTQ80i%PK z6;Ln2wDsr_DsTVmM~_ertGva(wS|u37=8EV&1HCb_4>d6u?zpcHh)0w+{J7A2Vpl< za0V%QH&{Rlx2#+}wS{7m5*ACu8T#2=I3V?5JPhw!x z6@b-8rnK$W)~}7FTFd;~OSOKrUmlAI&?2|C256Zwu^{)_h^mOboC~qFl~>C(H?;(F zA_GLn$VS45N;IXDX)-U$bnTk*!~gSt{%`(qz!kaBA>X?zlrI$H@E6iYJrii2V%_?g(r3yC{LP|HEFo(oJlk zI{_RDOL0Vkbt}+?gKW!~&tDfhbFueU;1HW4@L~)XeqJy$$+Oy4rln?0FsBlM5Skv{+s-YZe8j(j$@Onn_?+R>`Xy1-W@RmOAOKg>;ajH zISGq^Dr7Wb35hObG(ZRY?Ka@d!%7*Nkr6d_H&+eW;uNTLN>rG;%t_o&ig11dFx^RV zIV)>*n;j`(%`$Gv^zz28$ocyD>`WN z$FTl;kaD5N_g5RSnaDyf2^TWonx$E!w&gOk`KMs+%qtY=*e%c&4*n(kg&Z{$>riA~1}g5U)0@*Dz1YwsG8*_lTccGzK$t`}6XS*?bHHKBJend`0ul)*kzQ5kvLtz(Ns_$Q0l}rhQRZ-^{O>n{j#P$)WD> z?1l(iY7SGT?j(*X(Vi&2dV!t{|28~$zJZXZi3LD&tQ?Jdf=r+O#Ke6cmllSu>)-(7mH$M1MiZ@rJI;F99RA+OP7*qKk5o; zAMEe%!++K-<{vZ<+L;Xy`WD#Gf2=4|<8@Y(X+@ZUfwj~ET2J~myhgL?X`%Is4PI*K z1Zw+8*!P-2opk45m$IU(a`mZ&oqLY;h&KnfUd8mlekS7Dka~KI-n)jj^Y*EwE;v_f zk?;Om1*XaRM56JJ(5)sJNg^H9p;U0#DZDa3PYuj<$jPU1!(v+bh@U6B1%mS?E_Oc5 zZ!aChvm9Vs7D?CY%ZjuYGF@6I{mvAwEF2Bq>=tYAr~q%u0H!Lxl=b@TrN#u4JFKXe z@N{S4_2co=DP}8u8xZ@3CHm=vSR80vMQZM{_yWqBBvk;s!yxTGPP!(3O;Su@zTny{ zhi*;M=w{>&66$lDb)9nB^Z@I_&u3_`nh#)h2!3go0e9$lNRUR7x{SR|`ie%@em8yK zPV27UrI>q-TsMH#;4_z_SL&)y(nF5%02S++@V~+L=PzFdD7gN|b$I#a>iFbr{R^2d zH6lK)iecHl-b|0)Z@2vic24Ia>Ly0tu6HShned*UZoKVX(UeI7)vJet70rFxHDnS zGZbNF@^wwzkz%1J+&g5GY&Cy55&b&1C%46jC>TLMdu;r&?k+C+2RV$@=e`uHX0MYk zf4i}x&x&*9dluZFknY~y#k;B4t6Gf$*fr^r7@3y^-@VwVLiF|q3jvy>*kH_)n1}#< z;Z?)fkZRZwK0xRjKw5cQh`MG4s?|6u5^S%s(J<_b#t0hU^ME%v`}ATvSKjc>Gf2O> zx1nBSkwS8WASK2I$aGQJ44ilZ-FD*KsSSlH%e}kL8~a6r0W|10@~fB|MSQ=KY?n=E z!4_X;@xL;ukZoX(Eh&!7hj1C5xWNfpT11@$c%rj!)-Jd=fInx9GGuWZ%TcOTQxRjY z!ZTZ2=tpxB5t4x1d*t2^QEknr1PC(TUDCe~K=<=47jF}eV`kMCO9hyS z3uIvCFlytfM?s`ww3d28lkyFwiG=IH9QT?i%FHeriH$I}YAiQZOxoH)U;FU$OQ13q zaS?MyVl&z)bL^}~zsq@$-36IyM4H?6O#=dM^iD+On%YKGb9JXhWNAL_IoTj4F`Ofb zhyEd;BjgVv%;NTjZh$FFdmk8YVb{ai#&A;^s&%kin2$B^kppYvqFxAV9XzbDCoHHo z`uh~!GQ~nFCUuU2isKj*A^yq5`Cm*vG*lxng$NWk)&oV1qC7uU!b#IOOU!LdU|Mxpr$4tRL!okr&1wM`#wrfa zU4d2@>2*rBGnDGBZeb*8yPgdqNlUWSJi<@{c`=SouFsG1r>?llEtbd|1V&EQ4(-D1 z9B!^=97U1>Pbe1jhQNK*R6A-jXsKGWDgr-C%BZUlU7O_Kn}PeL8VNVb(^aK0@lZ*2 zCnvAgDu+(${1PCI_4TULV84i-H|@AQ{Tj{iJCw1!*igS&tq*UlN2kh_{c5v$zj~iH zXLm2$V);0KGB8i<9)Yz9yWY+F-Z^u)34VPuoXav%#Y%lsxxosji^~T#ot4O6>X|)g zgEn`{to<3`3t~UK&qFm#8 zdRWH`r0037C5Q_;I=j^veBK?|r>9)e%cj1$NikgO+14dGA~tJ)NRTlRg!L?H)JaJ` z_AF?eQhOxttNhtp%HrZLI2D}#m2W41)zPDi7k><}K%n>dd&*+}?+dr3@uosU zvDxwaSCy91rCVF>psp-H>`R)|}w*c-sm%H}ds8?7;Q+N0l_D*?x%oKH-IObKck5bi{ zr7P@~jxf7#iy4)6aPAABXNvu4yi3#N3>jQ@R^Lb7OOqOGWR$I14!Bux{^8qCCd3}p z*Ob~d4vK$0RRC1|3IR3 I;{bXB0Gzaq9smFU literal 0 HcmV?d00001 diff --git a/crds/doc-ru-nfsstorageclass.yaml b/crds/doc-ru-nfsstorageclass.yaml index d35c40b..db1a64c 100644 --- a/crds/doc-ru-nfsstorageclass.yaml +++ b/crds/doc-ru-nfsstorageclass.yaml @@ -4,11 +4,11 @@ spec: schema: openAPIV3Schema: description: | - Интерфейс управления StorageСlass для CSI-драйвера nfs.csi.storage.deckhouse.io. Ручное создание StorageClass для данного драйвера запрещено. + Интерфейс управления StorageСlass для CSI-драйвера nfs.csi.k8s.io. Ручное создание StorageClass для данного драйвера запрещено. properties: spec: properties: - server: + connection: description: | Настройки сервера NFS properties: @@ -18,10 +18,16 @@ spec: share: description: | Путь к точке монтирования на NFS сервере + subDir: + description: | + Поддиректория в NFS разделе. Если поддиректория не существует, она будет создана. Если значение subDir содержит следующие строки, они будут преобразованы в соответствующее имя pv/pvc или пространство имен: + - ${pvc.metadata.name} + - ${pvc.metadata.namespace} + - ${pv.metadata.name} nfsVersion: description: | Версия NFS сервера - options: + mountOptions: description: | Опции монтирования properties: @@ -39,7 +45,7 @@ spec: Монтирование в режиме "только чтение" chmodPermissions: description: | - Права монтирования субдиректории в NFS разделе + Права для chmod, которые будут применены к субдиректории тома в NFS разделе reclaimPolicy: description: | Режим поведения при удалении PVC. Может быть: diff --git a/crds/nfsstorageclass.yaml b/crds/nfsstorageclass.yaml index fdac42a..87832b7 100644 --- a/crds/nfsstorageclass.yaml +++ b/crds/nfsstorageclass.yaml @@ -32,10 +32,13 @@ spec: description: | Defines a Kubernetes Storage class configuration. required: - - server + - connection properties: - server: + connection: type: object + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. description: | Defines a Kubernetes Storage class configuration. required: @@ -50,6 +53,7 @@ spec: message: Value is immutable. description: | NFS server host + minLength: 1 share: type: string x-kubernetes-validations: @@ -57,6 +61,18 @@ spec: message: Value is immutable. description: | NFS server share path + minLength: 1 + subDir: + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + description: | + Sub directory under nfs share. If sub directory does not exist, it will be created. If subDir value contains following strings, it would be converted into corresponding pv/pvc name or namespace: + - ${pvc.metadata.name} + - ${pvc.metadata.namespace} + - ${pv.metadata.name} + minLength: 1 nfsVersion: type: string x-kubernetes-validations: @@ -68,7 +84,7 @@ spec: - "3" - "4.1" - "4.2" - options: + mountOptions: type: object description: | Storage class mount options @@ -84,10 +100,12 @@ spec: type: integer description: | NFS server timeout + minimum: 1 retransmissions: type: integer description: | NFS retries before fail + minimum: 1 readOnly: type: boolean description: | @@ -96,6 +114,7 @@ spec: type: string description: | chmod rights for PVs subdirectory + pattern: '^[0-7]{3,4}$' reclaimPolicy: type: string x-kubernetes-validations: @@ -138,3 +157,15 @@ spec: Additional information about the current state of the Storage Class. subresources: status: {} + additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.reason + name: Reason + type: string + priority: 1 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + description: The age of this resource diff --git a/hooks/common.py b/hooks/common.py new file mode 100644 index 0000000..062e259 --- /dev/null +++ b/hooks/common.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from lib.module import module +from typing import Callable +import json +import os +import unittest + + +NAMESPACE = "d8-csi-nfs" +MODULE_NAME = "csiNfs" + +def json_load(path: str): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + return data + +def get_dir_path() -> str: + return os.path.dirname(os.path.abspath(__file__)) diff --git a/hooks/generate_webhook_certs.py b/hooks/generate_webhook_certs.py new file mode 100755 index 0000000..e144604 --- /dev/null +++ b/hooks/generate_webhook_certs.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from lib.hooks.internal_tls import GenerateCertificateHook, TlsSecret, default_sans +from lib.module import values as module_values +from deckhouse import hook +from typing import Callable +import common + +def main(): + hook = GenerateCertificateHook( + TlsSecret( + cn="webhooks", + name="webhooks-https-certs", + sansGenerator=default_sans([ + "webhooks", + f"webhooks.{common.NAMESPACE}", + f"webhooks.{common.NAMESPACE}.svc"]), + values_path_prefix=f"{common.MODULE_NAME}.internal.customWebhookCert" + ), + cn="csi-nfs-webhooks", + common_ca=True, + namespace=common.NAMESPACE) + + hook.run() + +if __name__ == "__main__": + main() diff --git a/hooks/lib/__init__.py b/hooks/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/certificate/__init__.py b/hooks/lib/certificate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/certificate/certificate.py b/hooks/lib/certificate/certificate.py new file mode 100644 index 0000000..10a0661 --- /dev/null +++ b/hooks/lib/certificate/certificate.py @@ -0,0 +1,265 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import re +from OpenSSL import crypto +from ipaddress import ip_address +from datetime import datetime, timedelta +from lib.certificate.parse import parse_certificate, get_certificate_san + +class Certificate: + def __init__(self, cn: str, expire: int, key_size: int, algo: str) -> None: + self.key = crypto.PKey() + self.__with_key(algo=algo, size=key_size) + self.cert = crypto.X509() + self.cert.set_version(version=2) + self.cert.get_subject().CN = cn + self.cert.set_serial_number(random.getrandbits(64)) + self.cert.gmtime_adj_notBefore(0) + self.cert.gmtime_adj_notAfter(expire) + + def get_subject(self) -> crypto.X509Name: + return self.cert.get_subject() + + def __with_key(self, algo: str, size: int) -> None: + if algo == "rsa": + self.key.generate_key(crypto.TYPE_RSA, size) + elif algo == "dsa": + self.key.generate_key(crypto.TYPE_DSA, size) + else: + raise Exception(f"Algo {algo} is not support. Only [rsa, dsa]") + + def with_metadata(self, country: str = None, + state: str = None, + locality: str = None, + organisation_name: str = None, + organisational_unit_name: str = None): + """ + Adds subjects to certificate. + + :param country: Optional. The country of the entity. + :type country: :py:class:`str` + + :param state: Optional. The state or province of the entity. + :type state: :py:class:`str` + + :param locality: Optional. The locality of the entity + :type locality: :py:class:`str` + + :param organisation_name: Optional. The organization name of the entity. + :type organisation_name: :py:class:`str` + + :param organisational_unit_name: Optional. The organizational unit of the entity. + :type organisational_unit_name: :py:class:`str` + """ + + if country is not None: + self.cert.get_subject().C = country + if state is not None: + self.cert.get_subject().ST = state + if locality is not None: + self.cert.get_subject().L = locality + if organisation_name is not None: + self.cert.get_subject().O = organisation_name + if organisational_unit_name is not None: + self.cert.get_subject().OU = organisational_unit_name + return self + + def add_extension(self, type_name: str, + critical: bool, + value: str, + subject: crypto.X509 = None, + issuer: crypto.X509 = None): + """ + Adds extensions to certificate. + :param type_name: The name of the type of extension_ to create. + :type type_name: :py:class:`str` + + :param critical: A flag indicating whether this is a critical + extension. + :type critical: :py:class:`bool` + + :param value: The OpenSSL textual representation of the extension's + value. + :type value: :py:class:`str` + + :param subject: Optional X509 certificate to use as subject. + :type subject: :py:class:`crypto.X509` + + :param issuer: Optional X509 certificate to use as issuer. + :type issuer: :py:class:`crypto.X509` + """ + ext = crypto.X509Extension(type_name=str.encode(type_name), + critical=critical, + value=str.encode(value), + subject=subject, + issuer=issuer) + self.cert.add_extensions(extensions=[ext]) + return self + + def generate(self) -> (bytes, bytes): + """ + Generate certificate. + :return: (certificate, key) + :rtype: (:py:data:`bytes`, :py:data:`bytes`) + """ + pub = crypto.dump_certificate(crypto.FILETYPE_PEM, self.cert) + priv = crypto.dump_privatekey(crypto.FILETYPE_PEM, self.key) + return pub, priv + + +class CACertificateGenerator(Certificate): + """ + A class representing a generator CA certificate. + """ + def __sign(self) -> None: + self.cert.set_issuer(self.get_subject()) + self.cert.set_pubkey(self.key) + self.cert.sign(self.key, 'sha256') + + def generate(self) -> (bytes, bytes): + """ + Generate CA certificate. + :return: (ca crt, ca key) + :rtype: (:py:data:`bytes`, :py:data:`bytes`) + """ + self.add_extension(type_name="subjectKeyIdentifier", + critical=False, value="hash", subject=self.cert) + self.add_extension(type_name="authorityKeyIdentifier", + critical=False, value="keyid:always", issuer=self.cert) + self.add_extension(type_name="basicConstraints", + critical=False, value="CA:TRUE") + self.add_extension(type_name="keyUsage", critical=False, + value="keyCertSign, cRLSign, keyEncipherment") + self.__sign() + return super().generate() + + +class CertificateGenerator(Certificate): + """ + A class representing a generator certificate. + """ + def with_hosts(self, *hosts: str): + """ + This function is used to add subject alternative names to a certificate. + It takes a variable number of hosts as parameters, and based on the type of host (IP or DNS). + + :param hosts: Variable number of hosts to be added as subject alternative names to the certificate. + :type hosts: :py:class:`tuple` + """ + alt_names = [] + for h in hosts: + try: + ip_address(h) + alt_names.append(f"IP:{h}") + except ValueError: + if not is_valid_hostname(h): + continue + alt_names.append(f"DNS:{h}") + self.add_extension("subjectAltName", False, ", ".join(alt_names)) + return self + + def __sign(self, ca_subj: crypto.X509Name, ca_key: crypto.PKey) -> None: + self.cert.set_issuer(ca_subj) + self.cert.set_pubkey(self.key) + self.cert.sign(ca_key, 'sha256') + + def generate(self, ca_subj: crypto.X509Name, ca_key: crypto.PKey) -> (bytes, bytes): + """ + Generate certificate. + :param ca_subj: CA subject. + :type ca_subj: :py:class:`crypto.X509Name` + :param ca_key: CA Key. + :type ca_key: :py:class:`crypto.PKey` + :return: (certificate, key) + :rtype: (:py:data:`bytes`, :py:data:`bytes`) + """ + self.__sign(ca_subj, ca_key) + return super().generate() + +def is_valid_hostname(hostname: str) -> bool: + if len(hostname) > 255: + return False + hostname.rstrip(".") + allowed = re.compile("(?!-)[A-Z\d-]{1,63}(? bool: + """ + Check certificate + :param crt: Certificate + :type crt: :py:class:`crypto.X509` + :param cert_outdated_duration: certificate outdated duration + :type cert_outdated_duration: :py:class:`timedelta` + :return: + if timeNow > expire - cert_outdated_duration: + return True + return False + :rtype: :py:class:`bool` + """ + not_after = datetime.strptime( + crt.get_notAfter().decode('ascii'), '%Y%m%d%H%M%SZ') + if datetime.now() > not_after - cert_outdated_duration: + return True + return False + + +def is_outdated_ca(ca: str, cert_outdated_duration: timedelta) -> bool: + """ + Issue a new certificate if there is no CA in the secret. Without CA it is not possible to validate the certificate. + Check CA duration. + :param ca: Raw CA + :type ca: :py:class:`str` + :param cert_outdated_duration: certificate outdated duration + :type cert_outdated_duration: :py:class:`timedelta` + :rtype: :py:class:`bool` + """ + if len(ca) == 0: + return True + crt = parse_certificate(ca) + return cert_renew_deadline_exceeded(crt, cert_outdated_duration) + + +def is_irrelevant_cert(crt_data: str, sans: list, cert_outdated_duration: timedelta) -> bool: + """ + Check certificate duration and SANs list + :param crt_data: Raw certificate + :type crt_data: :py:class:`str` + :param sans: List of sans. + :type sans: :py:class:`list` + :param cert_outdated_duration: certificate outdated duration + :type cert_outdated_duration: :py:class:`timedelta` + :rtype: :py:class:`bool` + """ + if len(crt_data) == 0: + return True + crt = parse_certificate(crt_data) + if cert_renew_deadline_exceeded(crt, cert_outdated_duration): + return True + alt_names = [] + for san in sans: + try: + ip_address(san) + alt_names.append(f"IP Address:{san}") + except ValueError: + alt_names.append(f"DNS:{san}") + cert_sans = get_certificate_san(crt) + cert_sans.sort() + alt_names.sort() + if cert_sans != alt_names: + return True + return False \ No newline at end of file diff --git a/hooks/lib/certificate/parse.py b/hooks/lib/certificate/parse.py new file mode 100644 index 0000000..fd88c0d --- /dev/null +++ b/hooks/lib/certificate/parse.py @@ -0,0 +1,31 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from OpenSSL import crypto +from pprint import pprint + + +def parse_certificate(crt: str) -> crypto.X509: + return crypto.load_certificate(crypto.FILETYPE_PEM, crt) + + +def get_certificate_san(crt: crypto.X509) -> list[str]: + san = '' + ext_count = crt.get_extension_count() + for i in range(0, ext_count): + ext = crt.get_extension(i) + if 'subjectAltName' in str(ext.get_short_name()): + san = ext.__str__() + return san.split(', ') diff --git a/hooks/lib/hooks/__init__.py b/hooks/lib/hooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/hooks/copy_custom_certificate.py b/hooks/lib/hooks/copy_custom_certificate.py new file mode 100644 index 0000000..b3f3e83 --- /dev/null +++ b/hooks/lib/hooks/copy_custom_certificate.py @@ -0,0 +1,84 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from typing import Callable +from lib.module import module +from lib.hooks.hook import Hook + +class CopyCustomCertificatesHook(Hook): + CUSTOM_CERTIFICATES_SNAPSHOT_NAME = "custom_certificates" + def __init__(self, + module_name: str = None): + super().__init__(module_name=module_name) + self.queue = f"/modules/{self.module_name}/copy-custom-certificates" + + def generate_config(self) -> dict: + return { + "configVersion": "v1", + "beforeHelm": 10, + "kubernetes": [ + { + "name": self.CUSTOM_CERTIFICATES_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Secret", + "labelSelector": { + "matchExpressions": [ + { + "key": "owner", + "operator": "NotIn", + "values": ["helm"] + } + ] + }, + "namespace": { + "nameSelector": { + "matchNames": ["d8-system"] + } + }, + "includeSnapshotsFrom": [self.CUSTOM_CERTIFICATES_SNAPSHOT_NAME], + "jqFilter": '{"name": .metadata.name, "data": .data}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + ] + } + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + custom_certificates = {} + for s in ctx.snapshots.get(self.CUSTOM_CERTIFICATES_SNAPSHOT_NAME, []): + custom_certificates[s["filterResult"]["name"]] = s["filterResult"]["data"] + if len(custom_certificates) == 0: + return + + https_mode = module.get_https_mode(module_name=self.module_name, + values=ctx.values) + path = f"{self.module_name}.internal.customCertificateData" + if https_mode != "CustomCertificate": + self.delete_value(path, ctx.values) + return + + raw_secret_name = module.get_values_first_defined(ctx.values, + f"{self.module_name}.https.customCertificate.secretName", + "global.modules.https.customCertificate.secretName") + secret_name = str(raw_secret_name or "") + secret_data = custom_certificates.get(secret_name) + if secret_data is None: + print( + f"Custom certificate secret name is configured, but secret d8-system/{secret_name} doesn't exist") + return + self.set_value(path, ctx.values, secret_data) + return r \ No newline at end of file diff --git a/hooks/lib/hooks/hook.py b/hooks/lib/hooks/hook.py new file mode 100644 index 0000000..e412fae --- /dev/null +++ b/hooks/lib/hooks/hook.py @@ -0,0 +1,56 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from typing import Callable +from lib.module import module +from lib.module import values as module_values +import yaml + +class Hook: + def __init__(self, module_name: str = None) -> None: + self.module_name = self.get_module_name(module_name) + + def generate_config(self): + pass + + @staticmethod + def get_value(path: str, values: dict, default=None): + return module_values.get_value(path, values, default) + + @staticmethod + def set_value(path: str, values: dict, value: str) -> None: + return module_values.set_value(path, values, value) + + @staticmethod + def delete_value(path: str, values: dict) -> None: + return module_values.delete_value(path, values) + + @staticmethod + def get_module_name(module_name: str) -> str: + if module_name is not None: + return module_name + return module.get_module_name() + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + pass + return r + + def run(self) -> None: + conf = self.generate_config() + if isinstance(conf, dict): + conf = yaml.dump(conf) + hook.run(func=self.reconcile(), config=conf) diff --git a/hooks/lib/hooks/internal_tls.py b/hooks/lib/hooks/internal_tls.py new file mode 100644 index 0000000..3b351c6 --- /dev/null +++ b/hooks/lib/hooks/internal_tls.py @@ -0,0 +1,434 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from datetime import timedelta +from OpenSSL import crypto +from typing import Callable +from lib.hooks.hook import Hook +import lib.utils as utils +import lib.certificate.certificate as certificate + +PUBLIC_DOMAIN_PREFIX = "%PUBLIC_DOMAIN%://" +CLUSTER_DOMAIN_PREFIX = "%CLUSTER_DOMAIN%://" + + +KEY_USAGES = { + 0: "digitalSignature", + 1: "nonRepudiation", + 2: "keyEncipherment", + 3: "dataEncipherment", + 4: "keyAgreement", + 5: "keyCertSign", + 6: "cRLSign", + 7: "encipherOnly", + 8: "decipherOnly" +} + +EXTENDED_KEY_USAGES = { + 0: "serverAuth", + 1: "clientAuth", + 2: "codeSigning", + 3: "emailProtection", + 4: "OCSPSigning" +} + +class TlsSecret: + def __init__(self, + cn: str, + name: str, + sansGenerator: Callable[[list[str]], Callable[[hook.Context], list[str]]], + values_path_prefix: str, + key_usages: list[str] = [KEY_USAGES[2], KEY_USAGES[5]], + extended_key_usages: list[str] = [EXTENDED_KEY_USAGES[0]]): + self.cn = cn + self.name = name + self.sansGenerator = sansGenerator + self.values_path_prefix = values_path_prefix + self.key_usages = key_usages + self.extended_key_usages = extended_key_usages + +class GenerateCertificateHook(Hook): + """ + Config for the hook that generates certificates. + """ + SNAPSHOT_SECRETS_NAME = "secrets" + SNAPSHOT_SECRETS_CHECK_NAME = "secretsCheck" + + def __init__(self, *tls_secrets: TlsSecret, + cn: str, + namespace: str, + module_name: str = None, + common_ca: bool = False, + before_hook_check: Callable[[hook.Context], bool] = None, + expire: int = 31536000, + key_size: int = 4096, + algo: str = "rsa", + cert_outdated_duration: timedelta = timedelta(days=30), + country: str = None, + state: str = None, + locality: str = None, + organisation_name: str = None, + organisational_unit_name: str = None) -> None: + super().__init__(module_name=module_name) + self.cn = cn + self.tls_secrets = tls_secrets + self.namespace = namespace + self.common_ca = common_ca + self.before_hook_check = before_hook_check + self.expire = expire + self.key_size = key_size + self.algo = algo + self.cert_outdated_duration = cert_outdated_duration + self.country = country + self.state = state + self.locality = locality + self.organisation_name = organisation_name + self.organisational_unit_name = organisational_unit_name + self.secret_names = [secret.name for secret in self.tls_secrets] + self.queue = f"/modules/{self.module_name}/generate-certs" + """ + :param module_name: Module name + :type module_name: :py:class:`str` + + :param cn: Certificate common Name. often it is module name + :type cn: :py:class:`str` + + :param sansGenerator: Function which returns list of domain to include into cert. Use default_sans + :type sansGenerator: :py:class:`function` + + :param namespace: Namespace for TLS secret. + :type namespace: :py:class:`str` + + :param tls_secret_name: TLS secret name. + Secret must be TLS secret type https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets. + CA certificate MUST set to ca.crt key. + :type tls_secret_name: :py:class:`str` + + :param values_path_prefix: Prefix full path to store CA certificate TLS private key and cert. + full paths will be + values_path_prefix + .`ca` - CA certificate + values_path_prefix + .`crt` - TLS private key + values_path_prefix + .`key` - TLS certificate + Example: values_path_prefix = 'virtualization.internal.dvcrCert' + Data in values store as plain text + :type values_path_prefix: :py:class:`str` + + :param key_usages: Optional. key_usages specifies valid usage contexts for keys. + :type key_usages: :py:class:`list` + + :param extended_key_usages: Optional. extended_key_usages specifies valid usage contexts for keys. + :type extended_key_usages: :py:class:`list` + + :param before_hook_check: Optional. Runs check function before hook execution. Function should return boolean 'continue' value + if return value is false - hook will stop its execution + if return value is true - hook will continue + :type before_hook_check: :py:class:`function` + + :param expire: Optional. Validity period of SSL certificates. + :type expire: :py:class:`int` + + :param key_size: Optional. Key Size. + :type key_size: :py:class:`int` + + :param algo: Optional. Key generation algorithm. Supports only rsa and dsa. + :type algo: :py:class:`str` + + :param cert_outdated_duration: Optional. (expire - cert_outdated_duration) is time to regenerate the certificate. + :type cert_outdated_duration: :py:class:`timedelta` + """ + + def generate_config(self) -> dict: + return { + "configVersion": "v1", + "beforeHelm": 5, + "kubernetes": [ + { + "name": self.SNAPSHOT_SECRETS_NAME, + "apiVersion": "v1", + "kind": "Secret", + "nameSelector": { + "matchNames": self.secret_names + }, + "namespace": { + "nameSelector": { + "matchNames": [self.namespace] + } + }, + "includeSnapshotsFrom": [self.SNAPSHOT_SECRETS_NAME], + "jqFilter": '{"name": .metadata.name, "data": .data}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + ], + "schedule": [ + { + "name": self.SNAPSHOT_SECRETS_CHECK_NAME, + "crontab": "42 4 * * *" + } + ] + } + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + if self.before_hook_check is not None: + passed = self.before_hook_check(ctx) + if not passed: + return + + regenerate_all = False + secrets_from_snaps = {} + diff_secrets = [] + if len(ctx.snapshots.get(self.SNAPSHOT_SECRETS_NAME, [])) == 0: + regenerate_all = True + else: + for snap in ctx.snapshots[self.SNAPSHOT_SECRETS_NAME]: + secrets_from_snaps[snap["filterResult"]["name"]] = snap["filterResult"]["data"] + for secret in self.tls_secrets: + if secrets_from_snaps.get(secret.name) is None: + diff_secrets.append(secret.name) + + if self.common_ca and not regenerate_all: + if len(diff_secrets) > 0: + regenerate_all = True + else: + for secret in self.tls_secrets: + data = secrets_from_snaps[secret.name] + if self.is_outdated_ca(utils.base64_decode(data.get("ca.crt", ""))): + regenerate_all = True + break + sans = secret.sansGenerator(ctx) + if self.is_irrelevant_cert(utils.base64_decode(data.get("tls.crt", "")), sans): + regenerate_all = True + break + + if regenerate_all: + if self.common_ca: + ca = self.__get_ca_generator() + ca_crt, _ = ca.generate() + for secret in self.tls_secrets: + sans = secret.sansGenerator(ctx) + print(f"Generate new certififcates for secret {secret.name}.") + tls_data = self.generate_selfsigned_tls_data_with_ca(cn=secret.cn, + ca=ca, + ca_crt=ca_crt, + sans=sans, + key_usages=secret.key_usages, + extended_key_usages=secret.extended_key_usages) + self.set_value(secret.values_path_prefix, ctx.values, tls_data) + return + + for secret in self.tls_secrets: + sans = secret.sansGenerator(ctx) + print(f"Generate new certififcates for secret {secret.name}.") + tls_data = self.generate_selfsigned_tls_data(cn=secret.cn, + sans=sans, + key_usages=secret.key_usages, + extended_key_usages=secret.extended_key_usages) + self.set_value(secret.values_path_prefix, ctx.values, tls_data) + return + + for secret in self.tls_secrets: + data = secrets_from_snaps[secret.name] + sans = secret.sansGenerator(ctx) + cert_outdated = self.is_irrelevant_cert( + utils.base64_decode(data.get("tls.crt", "")), sans) + + tls_data = {} + if cert_outdated or data.get("tls.key", "") == "": + print(f"Certificates from secret {secret.name} is invalid. Generate new certififcates.") + tls_data = self.generate_selfsigned_tls_data(cn=secret.cn, + sans=sans, + key_usages=secret.key_usages, + extended_key_usages=secret.extended_key_usages) + else: + tls_data = { + "ca": data["ca.crt"], + "crt": data["tls.crt"], + "key": data["tls.key"] + } + self.set_value(secret.values_path_prefix, ctx.values, tls_data) + return r + + def __get_ca_generator(self) -> certificate.CACertificateGenerator: + return certificate.CACertificateGenerator(cn=f"{self.cn}", + expire=self.expire, + key_size=self.key_size, + algo=self.algo) + + def generate_selfsigned_tls_data_with_ca(self, + cn: str, + ca: certificate.CACertificateGenerator, + ca_crt: bytes, + sans: list[str], + key_usages: list[str], + extended_key_usages: list[str]) -> dict[str, str]: + """ + Generate self signed certificate. + :param cn: certificate common name. + :param ca: Ca certificate generator. + :type ca: :py:class:`certificate.CACertificateGenerator` + :param ca_crt: bytes. + :type ca_crt: :py:class:`bytes` + :param sans: List of sans. + :type sans: :py:class:`list` + :param key_usages: List of key_usages. + :type key_usages: :py:class:`list` + :param extended_key_usages: List of extended_key_usages. + :type extended_key_usages: :py:class:`list` + Example: { + "ca": "encoded in base64", + "crt": "encoded in base64", + "key": "encoded in base64" + } + :rtype: :py:class:`dict[str, str]` + """ + cert = certificate.CertificateGenerator(cn=cn, + expire=self.expire, + key_size=self.key_size, + algo=self.algo) + if len(key_usages) > 0: + key_usages = ", ".join(key_usages) + cert.add_extension(type_name="keyUsage", + critical=False, value=key_usages) + if len(extended_key_usages) > 0: + extended_key_usages = ", ".join(extended_key_usages) + cert.add_extension(type_name="extendedKeyUsage", + critical=False, value=extended_key_usages) + crt, key = cert.with_metadata(country=self.country, + state=self.state, + locality=self.locality, + organisation_name=self.organisation_name, + organisational_unit_name=self.organisational_unit_name + ).with_hosts(*sans).generate(ca_subj=ca.get_subject(), + ca_key=ca.key) + return {"ca": utils.base64_encode(ca_crt), + "crt": utils.base64_encode(crt), + "key": utils.base64_encode(key)} + + def generate_selfsigned_tls_data(self, + cn: str, + sans: list[str], + key_usages: list[str], + extended_key_usages: list[str]) -> dict[str, str]: + """ + Generate self signed certificate. + :param cn: certificate common name. + :param sans: List of sans. + :type sans: :py:class:`list` + :param key_usages: List of key_usages. + :type key_usages: :py:class:`list` + :param extended_key_usages: List of extended_key_usages. + :type extended_key_usages: :py:class:`list` + Example: { + "ca": "encoded in base64", + "crt": "encoded in base64", + "key": "encoded in base64" + } + :rtype: :py:class:`dict[str, str]` + """ + ca = self.__get_ca_generator() + ca_crt, _ = ca.generate() + return self.generate_selfsigned_tls_data_with_ca(cn=cn, + ca=ca, + ca_crt=ca_crt, + sans=sans, + key_usages=key_usages, + extended_key_usages=extended_key_usages) + + def is_irrelevant_cert(self, crt_data: str, sans: list) -> bool: + """ + Check certificate duration and SANs list + :param crt_data: Raw certificate + :type crt_data: :py:class:`str` + :param sans: List of sans. + :type sans: :py:class:`list` + :rtype: :py:class:`bool` + """ + return certificate.is_irrelevant_cert(crt_data, sans, self.cert_outdated_duration) + + def is_outdated_ca(self, ca: str) -> bool: + """ + Issue a new certificate if there is no CA in the secret. Without CA it is not possible to validate the certificate. + Check CA duration. + :param ca: Raw CA + :type ca: :py:class:`str` + :rtype: :py:class:`bool` + """ + return certificate.is_outdated_ca(ca, self.cert_outdated_duration) + + def cert_renew_deadline_exceeded(self, crt: crypto.X509) -> bool: + """ + Check certificate + :param crt: Certificate + :type crt: :py:class:`crypto.X509` + :return: + if timeNow > expire - cert_outdated_duration: + return True + return False + :rtype: :py:class:`bool` + """ + return certificate.cert_renew_deadline_exceeded(crt, self.cert_outdated_duration) + +def default_sans(sans: list[str]) -> Callable[[hook.Context], list[str]]: + """ + Generate list of sans for certificate + :param sans: List of alt names. + :type sans: :py:class:`list[str]` + cluster_domain_san(san) to generate sans with respect of cluster domain (e.g.: "app.default.svc" with "cluster.local" value will give: app.default.svc.cluster.local + + public_domain_san(san) + """ + def generate_sans(ctx: hook.Context) -> list[str]: + res = ["localhost", "127.0.0.1"] + public_domain = str(ctx.values["global"]["modules"].get( + "publicDomainTemplate", "")) + cluster_domain = str( + ctx.values["global"]["discovery"].get("clusterDomain", "")) + for san in sans: + san.startswith(PUBLIC_DOMAIN_PREFIX) + if san.startswith(PUBLIC_DOMAIN_PREFIX) and public_domain != "": + san = get_public_domain_san(san, public_domain) + elif san.startswith(CLUSTER_DOMAIN_PREFIX) and cluster_domain != "": + san = get_cluster_domain_san(san, cluster_domain) + res.append(san) + return res + return generate_sans + + +def cluster_domain_san(san: str) -> str: + """ + Create template to enrich specified san with a cluster domain + :param san: San. + :type sans: :py:class:`str` + """ + return CLUSTER_DOMAIN_PREFIX + san.rstrip('.') + + +def public_domain_san(san: str) -> str: + """ + Create template to enrich specified san with a public domain + :param san: San. + :type sans: :py:class:`str` + """ + return PUBLIC_DOMAIN_PREFIX + san.rstrip('.') + + +def get_public_domain_san(san: str, public_domain: str) -> str: + return f"{san.lstrip(PUBLIC_DOMAIN_PREFIX)}.{public_domain}" + + +def get_cluster_domain_san(san: str, cluster_domain: str) -> str: + return f"{san.lstrip(CLUSTER_DOMAIN_PREFIX)}.{cluster_domain}" diff --git a/hooks/lib/hooks/manage_tenant_secrets.py b/hooks/lib/hooks/manage_tenant_secrets.py new file mode 100644 index 0000000..36d817b --- /dev/null +++ b/hooks/lib/hooks/manage_tenant_secrets.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from typing import Callable +from lib.hooks.hook import Hook + +class ManageTenantSecretsHook(Hook): + POD_SNAPSHOT_NAME = "pods" + SECRETS_SNAPSHOT_NAME = "secrets" + NAMESPACE_SNAPSHOT_NAME = "namespaces" + + def __init__(self, + source_namespace: str, + source_secret_name: str, + pod_labels_to_follow: dict, + destination_secret_labels: dict = {}, + module_name: str = None): + super().__init__(module_name=module_name) + self.source_namespace = source_namespace + self.source_secret_name = source_secret_name + self.pod_labels_to_follow = pod_labels_to_follow + self.destination_secret_labels = destination_secret_labels + self.module_name = module_name + self.queue = f"/modules/{module_name}/manage-tenant-secrets" + + def generate_config(self) -> dict: + return { + "configVersion": "v1", + "kubernetes": [ + { + "name": self.POD_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Pod", + "includeSnapshotsFrom": [ + self.POD_SNAPSHOT_NAME, + self.SECRETS_SNAPSHOT_NAME, + self.NAMESPACE_SNAPSHOT_NAME + ], + "labelSelector": { + "matchLabels": self.pod_labels_to_follow + }, + "jqFilter": '{"namespace": .metadata.namespace}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + { + "name": self.SECRETS_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Secret", + "includeSnapshotsFrom": [ + self.POD_SNAPSHOT_NAME, + self.SECRETS_SNAPSHOT_NAME, + self.NAMESPACE_SNAPSHOT_NAME + ], + "nameSelector": { + "matchNames": [self.source_secret_name] + }, + "jqFilter": '{"data": .data, "namespace": .metadata.namespace, "type": .type}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + { + "name": self.NAMESPACE_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Secret", + "includeSnapshotsFrom": [ + self.POD_SNAPSHOT_NAME, + self.SECRETS_SNAPSHOT_NAME, + self.NAMESPACE_SNAPSHOT_NAME + ], + "jqFilter": '{"name": .metadata.name, "isTerminating": any(.metadata; .deletionTimestamp != null)}', + "queue": self.queue, + "keepFullObjectsInMemory": False + } + ] + } + + def generate_secret(self, namespace: str, data: dict, secret_type: str) -> dict: + return { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": self.source_secret_name, + "namespace": namespace, + "labels": self.destination_secret_labels + }, + "data": data, + "type": secret_type + } + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + pod_namespaces = set([p["filterResult"]["namespace"] for p in ctx.snapshots.get(self.POD_SNAPSHOT_NAME, [])]) + secrets = ctx.snapshots.get(self.SECRETS_SNAPSHOT_NAME, []) + for ns in ctx.snapshots.get(self.NAMESPACE_SNAPSHOT_NAME, []): + if ns["filterResult"]["isTerminating"]: + pod_namespaces.discard(ns["filterResult"]["name"]) + data, secret_type, secrets_by_ns = "", "", {} + for s in secrets: + if s["filterResult"]["namespace"] == self.source_namespace: + data = s["filterResult"]["data"] + secret_type = s["filterResult"]["type"] + continue + secrets_by_ns[s["filterResult"]["namespace"]] = s["filterResult"]["data"] + + if len(data) == 0 or len(secret_type) == 0: + print(f"Registry secret {self.source_namespace}/{self.source_secret_name} not found. Skip") + return + + for ns in pod_namespaces: + secret_data = secrets_by_ns.get(ns, "") + if (secret_data != data) and (ns != self.source_namespace): + secret = self.generate_secret(namespace=ns, + data=data, + secret_type=secret_type) + print(f"Create secret {ns}/{self.source_secret_name}.") + ctx.kubernetes.create_or_update(secret) + for ns in secrets_by_ns: + if (ns in pod_namespaces) or (ns == self.source_namespace): + continue + print(f"Delete secret {ns}/{self.source_secret_name}.") + ctx.kubernetes.delete(kind="Secret", + namespace=ns, + name=self.source_secret_name) + return r + diff --git a/hooks/lib/module/__init__.py b/hooks/lib/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/module/module.py b/hooks/lib/module/module.py new file mode 100644 index 0000000..25e034d --- /dev/null +++ b/hooks/lib/module/module.py @@ -0,0 +1,59 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.module import values as module_values +import re +import os + +def get_values_first_defined(values: dict, *keys): + return _get_first_defined(values, keys) + +def _get_first_defined(values: dict, keys: tuple): + for i in range(len(keys)): + if (val := module_values.get_value(path=keys[i], values=values)) is not None: + return val + return + +def get_https_mode(module_name: str, values: dict) -> str: + module_path = f"{module_name}.https.mode" + global_path = "global.modules.https.mode" + https_mode = get_values_first_defined(values, module_path, global_path) + if https_mode is not None: + return str(https_mode) + raise Exception("https mode is not defined") + +def get_module_name() -> str: + module = "" + file_path = os.path.abspath(__file__) + external_modules_dir = os.getenv("EXTERNAL_MODULES_DIR") + for dir in os.getenv("MODULES_DIR").split(":"): + if dir.startswith(external_modules_dir): + dir = external_modules_dir + if file_path.startswith(dir): + module = re.sub(f"{dir}/?\d?\d?\d?-?", "", file_path, 1).split("/")[0] + # /deckhouse/external-modules/virtualization/mr/hooks/hook_name.py + # {-------------------------- file_path --------------------------} + # {------ MODULES_DIR ------}{---------- regexp result ----------}} + # virtualization/mr/hooks/hook_name.py + # {-module-name-}{---------------------} + # or + # /deckhouse/modules/900-virtualization/hooks/hook_name.py + # {---------------------- file_path ----------------------} + # {-- MODULES_DIR --}{---{-------- regexp result --------}} + # virtualization/hooks/hook_name.py + # {-module-name-}{-----------------} + + break + return module \ No newline at end of file diff --git a/hooks/lib/module/values.py b/hooks/lib/module/values.py new file mode 100644 index 0000000..449e444 --- /dev/null +++ b/hooks/lib/module/values.py @@ -0,0 +1,60 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def get_value(path: str, values: dict, default=None): + def get(keys: list, values: dict, default): + if len(keys) == 1: + if not isinstance(values, dict): + return default + return values.get(keys[0], default) + if not isinstance(values, dict) or values.get(keys[0]) is None: + return default + if values.get(keys[0]) is None: + return default + return get(keys[1:], values[keys[0]], default) + keys = path.lstrip(".").split(".") + return get(keys, values, default) + +def set_value(path: str, values: dict, value) -> None: + """ + Functions for save value to dict. + + Example: + path = "virtualization.internal.dvcr.cert" + values = {"virtualization": {"internal": {}}} + value = "{"ca": "ca", "crt"="tlscrt", "key"="tlskey"}" + + result values = {"virtualization": {"internal": {"dvcr": {"cert": {"ca": "ca", "crt":"tlscrt", "key":"tlskey"}}}}} + """ + def set(keys: list, values: dict, value): + if len(keys) == 1: + values[keys[0]] = value + return + if values.get(keys[0]) is None: + values[keys[0]] = {} + set(keys[1:], values[keys[0]], value) + keys = path.lstrip(".").split(".") + return set(keys, values, value) + +def delete_value(path: str, values: dict) -> None: + if get_value(path, values) is None: + return + keys = path.lstrip(".").split(".") + def delete(keys: list, values: dict) -> None: + if len(keys) == 1: + values.pop(keys[0]) + return + delete(keys[1:], values[keys[0]]) + return delete(keys, values) \ No newline at end of file diff --git a/hooks/lib/password_generator/__init__.py b/hooks/lib/password_generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/password_generator/password_generator.py b/hooks/lib/password_generator/password_generator.py new file mode 100644 index 0000000..df63ea8 --- /dev/null +++ b/hooks/lib/password_generator/password_generator.py @@ -0,0 +1,77 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import string + +def generate_random_string(length: int, letters: str) -> str: + return ''.join(random.choice(letters) for i in range(length)) + +SYMBOLS = "[]{}<>()=-_!@#$%^&*.," + +def num(length: int) -> str: + """ + Generates a random string of the given length out of numeric characters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.digits) + +def alpha(length: int) -> str: + """ + Generates a random string of the given length out of alphabetic characters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_letters) + +def symbols(length: int) -> str: + """ + Generates a random string of the given length out of symbols. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, SYMBOLS) + + +def alpha_num(length: int) -> str: + """ + Generates a random string of the given length out of alphanumeric characters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_letters + string.digits) + +def alpha_num_lower_case(length: int) -> str: + """ + Generates a random string of the given length out of alphanumeric characters without UpperCase letters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_lowercase + string.digits) + +def alpha_num_symbols(length: int) -> str: + """ + Generates a random string of the given length out of alphanumeric characters and symbols. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_letters + string.digits + SYMBOLS) diff --git a/hooks/lib/tests/__init__.py b/hooks/lib/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/tests/test_copy_custom_certificate.py b/hooks/lib/tests/test_copy_custom_certificate.py new file mode 100644 index 0000000..c10e9c9 --- /dev/null +++ b/hooks/lib/tests/test_copy_custom_certificate.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.tests import testing +from lib.hooks.copy_custom_certificate import CopyCustomCertificatesHook + + +MODULE_NAME = "test" +SECRET_NAME = "secretName" +SECRET_DATA = { + "ca.crt": "CACRT", + "tls.crt": "TLSCRT", + "tls.key": "TLSKEY" + } + +hook = CopyCustomCertificatesHook(module_name=MODULE_NAME) + +binding_context = [ + { + "binding": "binding", + "snapshots": { + hook.CUSTOM_CERTIFICATES_SNAPSHOT_NAME: [ + { + "filterResult": { + "name": SECRET_NAME, + "data": SECRET_DATA + } + }, + { + "filterResult": { + "name": "test", + "data": {} + } + } + ] + } + } +] + +values_add = { + "global": { + "modules": { + "https": { + "mode": "CustomCertificate", + "customCertificate": { + "secretName": "test" + } + } + } + }, + MODULE_NAME: { + "https": { + "customCertificate": { + "secretName": SECRET_NAME + } + }, + "internal": {} + } +} + + +values_delete = { + "global": { + "modules": { + "https": { + "mode": "CertManager" + } + } + }, + MODULE_NAME: { + "internal": { + "customCertificateData": SECRET_DATA + } + } +} + + +class TestCopyCustomCertificateAdd(testing.TestHook): + def setUp(self): + self.func = hook.reconcile() + self.bindind_context = binding_context + self.values = values_add + def test_copy_custom_certificate_adding(self): + self.hook_run() + self.assertGreater(len(self.values[MODULE_NAME]["internal"].get("customCertificateData", {})), 0) + self.assertEqual(self.values[MODULE_NAME]["internal"]["customCertificateData"], SECRET_DATA) + +class TestCopyCustomCertificateDelete(testing.TestHook): + def setUp(self): + self.func = hook.reconcile() + self.bindind_context = binding_context + self.values = values_delete + def test_copy_custom_certificate_deleting(self): + self.hook_run() + self.assertEqual(len(self.values[MODULE_NAME]["internal"].get("customCertificateData", {})), 0) + + diff --git a/hooks/lib/tests/test_internal_tls_test.py b/hooks/lib/tests/test_internal_tls_test.py new file mode 100644 index 0000000..de842cd --- /dev/null +++ b/hooks/lib/tests/test_internal_tls_test.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.tests import testing +from lib.hooks.internal_tls import GenerateCertificateHook, default_sans, TlsSecret +from lib.certificate import parse +import lib.utils as utils +from OpenSSL import crypto +from ipaddress import ip_address + +NAME = "test" +MODULE_NAME = NAME +NAMESPACE = NAME +SANS = [ + NAME, + f"{NAME}.{NAMESPACE}", + f"{NAME}.{NAMESPACE}.svc" +] + +hook_generate = GenerateCertificateHook( + TlsSecret( + name=NAME, + sansGenerator=default_sans(SANS), + values_path_prefix=f"{MODULE_NAME}.internal.dvcr.cert"), + module_name=MODULE_NAME, + cn=NAME, + namespace=NAMESPACE) + +hook_regenerate = GenerateCertificateHook( + TlsSecret( + name=NAME, + sansGenerator=default_sans(SANS), + values_path_prefix=f"{MODULE_NAME}.internal.dvcr.cert"), + module_name=MODULE_NAME, + cn=NAME, + namespace=NAMESPACE, + expire=0) + +binding_context = [ + { + "binding": "binding", + "snapshots": {} + } +] + +values = { + "global": { + "modules": { + "publicDomainTemplate": "example.com" + }, + "discovery": { + "clusterDomain": "cluster.local" + } + }, + MODULE_NAME: { + "internal": {} + } +} + +class TestCertificate(testing.TestHook): + secret_data = {} + sans_default = SANS + ["localhost", "127.0.0.1"] + + @staticmethod + def parse_certificate(crt: str) -> crypto.X509: + return parse.parse_certificate(utils.base64_decode(crt)) + + def check_data(self): + self.assertGreater(len(self.values[MODULE_NAME]["internal"].get("dvcr", {}).get("cert", {})), 0) + self.secret_data = self.values[MODULE_NAME]["internal"]["dvcr"]["cert"] + self.assertTrue(utils.is_base64(self.secret_data.get("ca", ""))) + self.assertTrue(utils.is_base64(self.secret_data.get("crt", ""))) + self.assertTrue(utils.is_base64(self.secret_data.get("key", ""))) + + def check_sans(self, crt: crypto.X509) -> bool: + sans_from_cert = parse.get_certificate_san(crt) + sans = [] + for san in self.sans_default: + try: + ip_address(san) + sans.append(f"IP Address:{san}") + except ValueError: + sans.append(f"DNS:{san}") + sans_from_cert.sort() + sans.sort() + self.assertEqual(sans_from_cert, sans) + + def verify_certificate(self, ca: crypto.X509, crt: crypto.X509) -> crypto.X509StoreContextError: + store = crypto.X509Store() + store.add_cert(ca) + ctx = crypto.X509StoreContext(store, crt) + try: + ctx.verify_certificate() + return None + except crypto.X509StoreContextError as e: + return e + +class TestGenerateCertificate(TestCertificate): + def setUp(self): + self.func = hook_generate.reconcile() + self.bindind_context = binding_context + self.values = values + def test_generate_certificate(self): + self.hook_run() + self.check_data() + ca = self.parse_certificate(self.secret_data["ca"]) + crt = self.parse_certificate(self.secret_data["crt"]) + if (e := self.verify_certificate(ca, crt)) is not None: + self.fail(f"Certificate is not verify. Raised an exception: {e} ") + self.check_sans(crt) + +class TestReGenerateCertificate(TestCertificate): + def setUp(self): + self.func = hook_regenerate.reconcile() + self.bindind_context = binding_context + self.values = values + self.hook_run() + self.bindind_context[0]["snapshots"] = { + hook_regenerate.SNAPSHOT_SECRETS_NAME : [ + { + "filterResult": { + "data": { + "ca.crt" : self.values[MODULE_NAME]["internal"]["dvcr"]["cert"]["ca"], + "tls.crt": self.values[MODULE_NAME]["internal"]["dvcr"]["cert"]["crt"], + "key.crt": self.values[MODULE_NAME]["internal"]["dvcr"]["cert"]["key"] + }, + "name": NAME + } + } + ] + } + self.func = hook_generate.reconcile() + + def test_regenerate_certificate(self): + self.check_data() + ca = self.parse_certificate(self.secret_data["ca"]) + crt = self.parse_certificate(self.secret_data["crt"]) + if self.verify_certificate(ca, crt) is None: + self.fail(f"certificate has not expired") + self.hook_run() + self.check_data() + ca = self.parse_certificate(self.secret_data["ca"]) + crt = self.parse_certificate(self.secret_data["crt"]) + if (e := self.verify_certificate(ca, crt)) is not None: + self.fail(f"Certificate is not verify. Raised an exception: {e} ") + self.check_sans(crt) diff --git a/hooks/lib/tests/test_manage_tenant_secrets.py b/hooks/lib/tests/test_manage_tenant_secrets.py new file mode 100644 index 0000000..8e77059 --- /dev/null +++ b/hooks/lib/tests/test_manage_tenant_secrets.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.tests import testing +from lib.hooks.manage_tenant_secrets import ManageTenantSecretsHook + +hook = ManageTenantSecretsHook(source_namespace="source_namespace", + source_secret_name="secret_name", + pod_labels_to_follow={"app": "test"}, + destination_secret_labels={"test":"test"}, + module_name="test") + +binding_context = [ + { + "binding": "binding", + "snapshots": { + hook.POD_SNAPSHOT_NAME: [ + { + "filterResult": { + "namespace": "pod-namespace1" ## Create secret + } + }, + { + "filterResult": { + "namespace": "pod-namespace2" ## Don't create secret, because ns has deletionTimestamp + } + } + ], + hook.SECRETS_SNAPSHOT_NAME: [ + { + "filterResult": { + "data": {"test": "test"}, + "namespace": "source_namespace", + "type": "Opaque" + } + }, + { + "filterResult": { + "data": {"test": "test"}, + "namespace": "pod-namespace3", ## Delete secret, because namespace pod-namespace3 hasn't pods + "type": "Opaque" + } + }, + ], + hook.NAMESPACE_SNAPSHOT_NAME: [ + { + "filterResult": { + "name": "source_namespace", + "isTerminating": False + } + }, + { + "filterResult": { + "name": "pod-namespace1", + "isTerminating": False + } + }, + { + "filterResult": { + "name": "pod-namespace2", + "isTerminating": True + } + }, + { + "filterResult": { + "name": "pod-namespace3", + "isTerminating": False + } + }, + ] + } + } +] + +class TestManageSecrets(testing.TestHook): + def setUp(self): + self.func = hook.reconcile() + self.bindind_context = binding_context + self.values = {} + def test_manage_secrets(self): + self.hook_run() + self.assertEqual(len(self.kube_resources), 1) + self.assertEqual(self.kube_resources[0]["kind"], "Secret") + self.assertEqual(self.kube_resources[0]["metadata"]["name"], "secret_name") + self.assertEqual(self.kube_resources[0]["metadata"]["namespace"], "pod-namespace1") + self.assertEqual(self.kube_resources[0]["type"], "Opaque") + self.assertEqual(self.kube_resources[0]["data"], {'test': 'test'}) + self.assertEqual(self.kube_resources[0]["metadata"]["labels"], {'test': 'test'}) + diff --git a/hooks/lib/tests/testing.py b/hooks/lib/tests/testing.py new file mode 100644 index 0000000..f0257aa --- /dev/null +++ b/hooks/lib/tests/testing.py @@ -0,0 +1,57 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +import unittest +import jsonpatch +import kubernetes_validate +import jsonschema + +class TestHook(unittest.TestCase): + kube_resources = [] + kube_version = "1.28" + def setUp(self): + self.bindind_context = [] + self.values = {} + self.func = None + + def tearDown(self): + pass + + def hook_run(self, validate_kube_resources: bool = True) -> None: + out = hook.testrun(func=self.func, + binding_context=self.bindind_context, + initial_values=self.values) + for patch in out.values_patches.data: + self.values = jsonpatch.apply_patch(self.values, [patch]) + + deletes = ("Delete", "DeleteInBackground", "DeleteNonCascading") + for kube_operation in out.kube_operations.data: + if kube_operation["operation"] in deletes: + continue + obj = kube_operation["object"] + if validate_kube_resources: + try: + ## TODO Validate CRD + kubernetes_validate.validate(obj, self.kube_version, strict=True) + self.kube_resources.append(obj) + except (kubernetes_validate.SchemaNotFoundError, + kubernetes_validate.InvalidSchemaError, + kubernetes_validate.ValidationError, + jsonschema.RefResolutionError) as e: + self.fail(f"Object is not valid. Raised an exception: {e} ") + else: + self.kube_resources.append(obj) + diff --git a/hooks/lib/utils.py b/hooks/lib/utils.py new file mode 100644 index 0000000..a486efd --- /dev/null +++ b/hooks/lib/utils.py @@ -0,0 +1,54 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import json + +def base64_encode(b: bytes) -> str: + return str(base64.b64encode(b), encoding='utf-8') + +def base64_decode(s: str) -> str: + return str(base64.b64decode(s), encoding="utf-8") + +def base64_encode_from_str(s: str) -> str: + return base64_encode(bytes(s, 'utf-8')) + +def json_load(path: str): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + return data + +def get_dir_path() -> str: + return os.path.dirname(os.path.abspath(__file__)) + +def is_base64(s): + try: + base64_decode(s) + return True + except base64.binascii.Error: + return False + +def check_elem_in_list(l: list, elem) -> bool: + for i in l: + if i == elem: + return True + return False + +def find_index_in_list(l: list, elem) -> int: + for i in range(len(l)): + if l[i] == elem: + return i + return \ No newline at end of file diff --git a/images/controller/Dockerfile b/images/controller/Dockerfile new file mode 100644 index 0000000..f98b4ad --- /dev/null +++ b/images/controller/Dockerfile @@ -0,0 +1,17 @@ +ARG BASE_ALPINE=registry.deckhouse.io/base_images/alpine:3.16.3@sha256:5548e9172c24a1b0ca9afdd2bf534e265c94b12b36b3e0c0302f5853eaf00abb +ARG BASE_GOLANG_21_ALPINE_BUILDER=registry.deckhouse.io/base_images/golang:1.21.4-alpine3.18@sha256:cf84f3d6882c49ea04b6478ac514a2582c8922d7e5848b43d2918fff8329f6e6 + +FROM $BASE_GOLANG_21_ALPINE_BUILDER as builder + +WORKDIR /go/src +ADD go.mod . +ADD go.sum . +RUN go mod download +COPY . . +WORKDIR /go/src/cmd +RUN GOOS=linux GOARCH=amd64 go build -o controller + +FROM --platform=linux/amd64 $BASE_ALPINE +COPY --from=builder /go/src/cmd/controller /go/src/cmd/controller + +ENTRYPOINT ["/go/src/cmd/controller"] diff --git a/images/controller/api/v1alpha1/nfs_storage_class.go b/images/controller/api/v1alpha1/nfs_storage_class.go new file mode 100644 index 0000000..afd2340 --- /dev/null +++ b/images/controller/api/v1alpha1/nfs_storage_class.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type NFSStorageClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec NFSStorageClassSpec `json:"spec"` + Status *NFSStorageClassStatus `json:"status,omitempty"` +} + +// NFSStorageClassList contains a list of empty block device +type NFSStorageClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []NFSStorageClass `json:"items"` +} + +type NFSStorageClassSpec struct { + Connection *NFSStorageClassConnection `json:"connection,omitempty"` + MountOptions *NFSStorageClassMountOptions `json:"mountOptions,omitempty"` + ChmodPermissions string `json:"chmodPermissions,omitempty"` + ReclaimPolicy string `json:"reclaimPolicy"` + VolumeBindingMode string `json:"volumeBindingMode"` +} + +type NFSStorageClassConnection struct { + Host string `json:"host"` + Share string `json:"share"` + NFSVersion string `json:"nfsVersion"` + SubDir string `json:"subDir,omitempty"` +} + +type NFSStorageClassMountOptions struct { + MountMode string `json:"mountMode,omitempty"` + Timeout int `json:"timeout,omitempty"` + Retransmissions int `json:"retransmissions,omitempty"` + ReadOnly *bool `json:"readOnly,omitempty"` +} + +type NFSStorageClassStatus struct { + Phase string `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` +} diff --git a/images/controller/api/v1alpha1/register.go b/images/controller/api/v1alpha1/register.go new file mode 100644 index 0000000..9e37972 --- /dev/null +++ b/images/controller/api/v1alpha1/register.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + NFSStorageClassKind = "NFSStorageClass" + APIGroup = "storage.deckhouse.io" + APIVersion = "v1alpha1" +) + +// SchemeGroupVersion is group version used to register these objects +var ( + SchemeGroupVersion = schema.GroupVersion{ + Group: APIGroup, + Version: APIVersion, + } + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +// Adds the list of known types to Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &NFSStorageClass{}, + &NFSStorageClassList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/images/controller/api/v1alpha1/zz_generated.deepcopy.go b/images/controller/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..d74af89 --- /dev/null +++ b/images/controller/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import "k8s.io/apimachinery/pkg/runtime" + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NFSStorageClass) DeepCopyInto(out *NFSStorageClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyBlockDevice. +func (in *NFSStorageClass) DeepCopy() *NFSStorageClass { + if in == nil { + return nil + } + out := new(NFSStorageClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NFSStorageClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NFSStorageClassList) DeepCopyInto(out *NFSStorageClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NFSStorageClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuestbookList. +func (in *NFSStorageClassList) DeepCopy() *NFSStorageClassList { + if in == nil { + return nil + } + out := new(NFSStorageClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NFSStorageClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/images/controller/cmd/main.go b/images/controller/cmd/main.go new file mode 100644 index 0000000..cbb8ed9 --- /dev/null +++ b/images/controller/cmd/main.go @@ -0,0 +1,131 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "d8-controller/api/v1alpha1" + "d8-controller/pkg/config" + "d8-controller/pkg/controller" + "d8-controller/pkg/kubutils" + "d8-controller/pkg/logger" + "fmt" + "os" + goruntime "runtime" + + "sigs.k8s.io/controller-runtime/pkg/cache" + + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + apiruntime "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var ( + resourcesSchemeFuncs = []func(*apiruntime.Scheme) error{ + v1alpha1.AddToScheme, + clientgoscheme.AddToScheme, + extv1.AddToScheme, + v1.AddToScheme, + sv1.AddToScheme, + } +) + +func main() { + ctx := context.Background() + cfgParams := config.NewConfig() + + log, err := logger.NewLogger(cfgParams.Loglevel) + if err != nil { + fmt.Println(fmt.Sprintf("unable to create NewLogger, err: %v", err)) + os.Exit(1) + } + + log.Info(fmt.Sprintf("[main] Go Version:%s ", goruntime.Version())) + log.Info(fmt.Sprintf("[main] OS/Arch:Go OS/Arch:%s/%s ", goruntime.GOOS, goruntime.GOARCH)) + + log.Info("[main] CfgParams has been successfully created") + log.Info(fmt.Sprintf("[main] %s = %s", config.LogLevelEnvName, cfgParams.Loglevel)) + log.Info(fmt.Sprintf("[main] RequeueStorageClassInterval = %d", cfgParams.RequeueStorageClassInterval)) + + kConfig, err := kubutils.KubernetesDefaultConfigCreate() + if err != nil { + log.Error(err, "[main] unable to KubernetesDefaultConfigCreate") + } + log.Info("[main] kubernetes config has been successfully created.") + + scheme := runtime.NewScheme() + for _, f := range resourcesSchemeFuncs { + err := f(scheme) + if err != nil { + log.Error(err, "[main] unable to add scheme to func") + os.Exit(1) + } + } + log.Info("[main] successfully read scheme CR") + + cacheOpt := cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + cfgParams.ControllerNamespace: {}, + }, + } + + managerOpts := manager.Options{ + Scheme: scheme, + Cache: cacheOpt, + //MetricsBindAddress: cfgParams.MetricsPort, + HealthProbeBindAddress: cfgParams.HealthProbeBindAddress, + LeaderElection: true, + LeaderElectionNamespace: cfgParams.ControllerNamespace, + LeaderElectionID: config.ControllerName, + Logger: log.GetLogger(), + } + + mgr, err := manager.New(kConfig, managerOpts) + if err != nil { + log.Error(err, "[main] unable to manager.New") + os.Exit(1) + } + log.Info("[main] successfully created kubernetes manager") + + if _, err = controller.RunNFSStorageClassWatcherController(mgr, *cfgParams, *log); err != nil { + log.Error(err, fmt.Sprintf("[main] unable to run %s", controller.NFSStorageClassCtrlName)) + os.Exit(1) + } + + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(err, "[main] unable to mgr.AddHealthzCheck") + os.Exit(1) + } + log.Info("[main] successfully AddHealthzCheck") + + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error(err, "[main] unable to mgr.AddReadyzCheck") + os.Exit(1) + } + log.Info("[main] successfully AddReadyzCheck") + + err = mgr.Start(ctx) + if err != nil { + log.Error(err, "[main] unable to mgr.Start") + os.Exit(1) + } +} diff --git a/images/controller/go.mod b/images/controller/go.mod new file mode 100644 index 0000000..8d4a6a9 --- /dev/null +++ b/images/controller/go.mod @@ -0,0 +1,71 @@ +module d8-controller + +go 1.21 + +require ( + github.com/go-logr/logr v1.4.1 + github.com/onsi/ginkgo/v2 v2.14.0 + github.com/onsi/gomega v1.30.0 + k8s.io/api v0.29.2 + k8s.io/apiextensions-apiserver v0.29.2 + k8s.io/apimachinery v0.29.2 + k8s.io/client-go v0.29.2 + k8s.io/klog/v2 v2.120.1 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e + sigs.k8s.io/controller-runtime v0.17.4 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.16.1 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-base v0.29.2 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/images/controller/go.sum b/images/controller/go.sum new file mode 100644 index 0000000..3473c3a --- /dev/null +++ b/images/controller/go.sum @@ -0,0 +1,204 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= +k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= +k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.4 h1:AMf1E0+93/jLQ13fb76S6Atwqp24EQFCmNbG84GJxew= +sigs.k8s.io/controller-runtime v0.17.4/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/images/controller/pkg/config/config.go b/images/controller/pkg/config/config.go new file mode 100644 index 0000000..abd10d0 --- /dev/null +++ b/images/controller/pkg/config/config.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "d8-controller/pkg/logger" + "log" + "os" + "time" +) + +const ( + LogLevelEnvName = "LOG_LEVEL" + ControllerNamespaceEnv = "CONTROLLER_NAMESPACE" + HardcodedControllerNS = "d8-csi-nfs" + ControllerName = "d8-controller" + DefaultHealthProbeBindAddressEnvName = "HEALTH_PROBE_BIND_ADDRESS" + DefaultHealthProbeBindAddress = ":8081" + DefaultRequeueStorageClassInterval = 10 +) + +type Options struct { + Loglevel logger.Verbosity + RequeueStorageClassInterval time.Duration + HealthProbeBindAddress string + ControllerNamespace string +} + +func NewConfig() *Options { + var opts Options + + loglevel := os.Getenv(LogLevelEnvName) + if loglevel == "" { + opts.Loglevel = logger.DebugLevel + } else { + opts.Loglevel = logger.Verbosity(loglevel) + } + + opts.HealthProbeBindAddress = os.Getenv(DefaultHealthProbeBindAddressEnvName) + if opts.HealthProbeBindAddress == "" { + opts.HealthProbeBindAddress = DefaultHealthProbeBindAddress + } + + opts.ControllerNamespace = os.Getenv(ControllerNamespaceEnv) + if opts.ControllerNamespace == "" { + + namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + log.Printf("Failed to get namespace from filesystem: %v", err) + log.Printf("Using hardcoded namespace: %s", HardcodedControllerNS) + opts.ControllerNamespace = HardcodedControllerNS + } else { + log.Printf("Got namespace from filesystem: %s", string(namespace)) + opts.ControllerNamespace = string(namespace) + } + } + + opts.RequeueStorageClassInterval = DefaultRequeueStorageClassInterval + + return &opts +} diff --git a/images/controller/pkg/controller/controller_suite_test.go b/images/controller/pkg/controller/controller_suite_test.go new file mode 100644 index 0000000..81e10e2 --- /dev/null +++ b/images/controller/pkg/controller/controller_suite_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller_test + +import ( + v1alpha1 "d8-controller/api/v1alpha1" + "fmt" + "os" + "testing" + + v1 "k8s.io/api/apps/v1" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + sv1 "k8s.io/api/storage/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + + apiruntime "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controller Suite") +} + +func NewFakeClient() client.Client { + resourcesSchemeFuncs := []func(*apiruntime.Scheme) error{ + v1alpha1.AddToScheme, + clientgoscheme.AddToScheme, + extv1.AddToScheme, + v1.AddToScheme, + sv1.AddToScheme, + } + scheme := apiruntime.NewScheme() + for _, f := range resourcesSchemeFuncs { + err := f(scheme) + if err != nil { + println(fmt.Sprintf("Error adding scheme: %s", err)) + os.Exit(1) + } + } + + // See https://github.com/kubernetes-sigs/controller-runtime/issues/2362#issuecomment-1837270195 + builder := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&v1alpha1.NFSStorageClass{}) + + cl := builder.Build() + return cl +} diff --git a/images/controller/pkg/controller/nfs_storage_class_watcher.go b/images/controller/pkg/controller/nfs_storage_class_watcher.go new file mode 100644 index 0000000..b5d46e7 --- /dev/null +++ b/images/controller/pkg/controller/nfs_storage_class_watcher.go @@ -0,0 +1,253 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + v1alpha1 "d8-controller/api/v1alpha1" + "d8-controller/pkg/config" + "d8-controller/pkg/logger" + "errors" + "fmt" + "reflect" + "time" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + NFSStorageClassCtrlName = "nfs-storage-class-controller" + + StorageClassKind = "StorageClass" + StorageClassAPIVersion = "storage.k8s.io/v1" + + NFSStorageClassProvisioner = "nfs.csi.k8s.io" + + NFSStorageClassFinalizerName = "storage.deckhouse.io/nfs-storage-class-controller" + NFSStorageClassManagedLabelKey = "storage.deckhouse.io/managed-by" + NFSStorageClassManagedLabelValue = "nfs-storage-class-controller" + + AllowVolumeExpansionDefaultValue = true + + FailedStatusPhase = "Failed" + CreatedStatusPhase = "Created" + + CreateReconcile = "Create" + UpdateReconcile = "Update" + DeleteReconcile = "Delete" + + serverParamKey = "server" + shareParamKey = "share" + MountPermissionsParamKey = "mountPermissions" + SubDirParamKey = "subdir" + MountOptionsSecretKey = "mountOptions" + + SecretForMountOptionsPrefix = "nfs-mount-options-for-" + StorageClassSecretNameKey = "csi.storage.k8s.io/provisioner-secret-name" + StorageClassSecretNSKey = "csi.storage.k8s.io/provisioner-secret-namespace" +) + +func RunNFSStorageClassWatcherController( + mgr manager.Manager, + cfg config.Options, + log logger.Logger, +) (controller.Controller, error) { + cl := mgr.GetClient() + + c, err := controller.New(NFSStorageClassCtrlName, mgr, controller.Options{ + Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log.Info(fmt.Sprintf("[NFSStorageClassReconciler] starts Reconcile for the NFSStorageClass %q", request.Name)) + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, request.NamespacedName, nsc) + if err != nil && !k8serr.IsNotFound(err) { + log.Error(err, fmt.Sprintf("[NFSStorageClassReconciler] unable to get NFSStorageClass, name: %s", request.Name)) + return reconcile.Result{}, err + } + + if nsc.Name == "" { + log.Info(fmt.Sprintf("[NFSStorageClassReconciler] seems like the NFSStorageClass for the request %s was deleted. Reconcile retrying will stop.", request.Name)) + return reconcile.Result{}, nil + } + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + if err != nil { + log.Error(err, "[NFSStorageClassReconciler] unable to list Storage Classes") + return reconcile.Result{}, err + } + + shouldRequeue, err := RunEventReconcile(ctx, cl, log, scList, nsc, cfg.ControllerNamespace) + if err != nil { + log.Error(err, fmt.Sprintf("[NFSStorageClassReconciler] an error occured while reconciles the NFSStorageClass, name: %s", nsc.Name)) + } + + if shouldRequeue { + log.Warning(fmt.Sprintf("[NFSStorageClassReconciler] Reconciler will requeue the request, name: %s", request.Name)) + return reconcile.Result{ + RequeueAfter: cfg.RequeueStorageClassInterval * time.Second, + }, nil + } + + log.Info(fmt.Sprintf("[NFSStorageClassReconciler] ends Reconcile for the NFSStorageClass %q", request.Name)) + return reconcile.Result{}, nil + }), + }) + if err != nil { + log.Error(err, "[RunNFSStorageClassWatcherController] unable to create controller") + return nil, err + } + + err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.NFSStorageClass{}), handler.Funcs{ + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[CreateFunc] get event for NFSStorageClass %q. Add to the queue", e.Object.GetName())) + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} + q.Add(request) + }, + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[UpdateFunc] get event for NFSStorageClass %q. Check if it should be reconciled", e.ObjectNew.GetName())) + + oldLsc, ok := e.ObjectOld.(*v1alpha1.NFSStorageClass) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[UpdateFunc] an error occurred while handling create event") + return + } + newLsc, ok := e.ObjectNew.(*v1alpha1.NFSStorageClass) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[UpdateFunc] an error occurred while handling create event") + return + } + + if reflect.DeepEqual(oldLsc.Spec, newLsc.Spec) && newLsc.DeletionTimestamp == nil { + log.Info(fmt.Sprintf("[UpdateFunc] an update event for the NFSStorageClass %s has no Spec field updates. It will not be reconciled", newLsc.Name)) + return + } + + log.Info(fmt.Sprintf("[UpdateFunc] the NFSStorageClass %q will be reconciled. Add to the queue", newLsc.Name)) + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: newLsc.Namespace, Name: newLsc.Name}} + q.Add(request) + }, + }) + if err != nil { + log.Error(err, "[RunNFSStorageClassWatcherController] unable to watch the events") + return nil, err + } + + return c, nil +} + +func RunEventReconcile(ctx context.Context, cl client.Client, log logger.Logger, scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (shouldRequeue bool, err error) { + added, err := addFinalizerIfNotExistsForNSC(ctx, cl, nsc) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to add a finalizer %s to the NFSStorageClass %s: %w", NFSStorageClassFinalizerName, nsc.Name, err) + return true, err + } + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] finalizer %s was added to the NFSStorageClass %s: %t", NFSStorageClassFinalizerName, nsc.Name, added)) + + reconcileTypeForStorageClass, err := IdentifyReconcileFuncForStorageClass(log, scList, nsc, controllerNamespace) + if err != nil { + err = fmt.Errorf("[runEventReconcile] error occured while identifying the reconcile function for StorageClass %s: %w", nsc.Name, err) + return true, err + } + + shouldRequeue = false + log.Debug(fmt.Sprintf("[runEventReconcile] reconcile operation for StorageClass %q: %q", nsc.Name, reconcileTypeForStorageClass)) + switch reconcileTypeForStorageClass { + case CreateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] CreateReconcile starts reconciliataion of StorageClass, name: %s", nsc.Name)) + shouldRequeue, err = ReconcileStorageClassCreateFunc(ctx, cl, log, scList, nsc, controllerNamespace) + case UpdateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] UpdateReconcile starts reconciliataion of StorageClass, name: %s", nsc.Name)) + shouldRequeue, err = reconcileStorageClassUpdateFunc(ctx, cl, log, scList, nsc, controllerNamespace) + case DeleteReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] DeleteReconcile starts reconciliataion of StorageClass, name: %s", nsc.Name)) + shouldRequeue, err = reconcileStorageClassDeleteFunc(ctx, cl, log, scList, nsc) + default: + log.Debug(fmt.Sprintf("[runEventReconcile] StorageClass for NFSStorageClass %s should not be reconciled", nsc.Name)) + } + log.Debug(fmt.Sprintf("[runEventReconcile] ends reconciliataion of StorageClass, name: %s, shouldRequeue: %t, err: %v", nsc.Name, shouldRequeue, err)) + + if err != nil || shouldRequeue { + return shouldRequeue, err + } + + secretList := &corev1.SecretList{} + err = cl.List(ctx, secretList, client.InNamespace(controllerNamespace)) + if err != nil { + err = fmt.Errorf("[runEventReconcile] unable to list Secrets: %w", err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + reconcileTypeForSecret, err := IdentifyReconcileFuncForSecret(log, secretList, nsc, controllerNamespace) + if err != nil { + log.Error(err, fmt.Sprintf("[runEventReconcile] error occured while identifying the reconcile function for the Secret %q", SecretForMountOptionsPrefix+nsc.Name)) + return true, err + } + + log.Debug(fmt.Sprintf("[runEventReconcile] reconcile operation for Secret %q: %q", SecretForMountOptionsPrefix+nsc.Name, reconcileTypeForSecret)) + switch reconcileTypeForSecret { + case CreateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] CreateReconcile starts reconciliataion of Secret, name: %s", SecretForMountOptionsPrefix+nsc.Name)) + shouldRequeue, err = ReconcileSecretCreateFunc(ctx, cl, log, nsc, controllerNamespace) + case UpdateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] UpdateReconcile starts reconciliataion of Secret, name: %s", SecretForMountOptionsPrefix+nsc.Name)) + shouldRequeue, err = reconcileSecretUpdateFunc(ctx, cl, log, secretList, nsc, controllerNamespace) + case DeleteReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] DeleteReconcile starts reconciliataion of Secret, name: %s", SecretForMountOptionsPrefix+nsc.Name)) + shouldRequeue, err = reconcileSecretDeleteFunc(ctx, cl, log, secretList, nsc) + default: + log.Debug(fmt.Sprintf("[runEventReconcile] Secret %q should not be reconciled", SecretForMountOptionsPrefix+nsc.Name)) + } + + log.Debug(fmt.Sprintf("[runEventReconcile] ends reconciliataion of Secret, name: %s, shouldRequeue: %t, err: %v", SecretForMountOptionsPrefix+nsc.Name, shouldRequeue, err)) + + if err != nil || shouldRequeue { + return shouldRequeue, err + } + + log.Debug(fmt.Sprintf("[runEventReconcile] Finish all reconciliations for NFSStorageClass %q.", nsc.Name)) + + if reconcileTypeForSecret != DeleteReconcile { + err = updateNFSStorageClassPhase(ctx, cl, nsc, CreatedStatusPhase, "") + if err != nil { + err = fmt.Errorf("[runEventReconcile] unable to update the NFSStorageClass %s: %w", nsc.Name, err) + return true, err + } + log.Debug(fmt.Sprintf("[runEventReconcile] successfully updated the NFSStorageClass %s status", nsc.Name)) + } + + return false, nil + +} diff --git a/images/controller/pkg/controller/nfs_storage_class_watcher_func.go b/images/controller/pkg/controller/nfs_storage_class_watcher_func.go new file mode 100644 index 0000000..43af91c --- /dev/null +++ b/images/controller/pkg/controller/nfs_storage_class_watcher_func.go @@ -0,0 +1,720 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + v1alpha1 "d8-controller/api/v1alpha1" + "d8-controller/pkg/logger" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/strings/slices" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ReconcileStorageClassCreateFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + scList *v1.StorageClassList, + nsc *v1alpha1.NFSStorageClass, + controllerNamespace string, +) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] starts for NFSStorageClass %q", nsc.Name)) + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] starts storage class configuration for the NFSStorageClass, name: %s", nsc.Name)) + newSC, err := ConfigureStorageClass(nsc, controllerNamespace) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to configure a Storage Class for the NFSStorageClass %s: %w", nsc.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return false, err + } + + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] successfully configurated storage class for the NFSStorageClass, name: %s", nsc.Name)) + log.Trace(fmt.Sprintf("[reconcileStorageClassCreateFunc] storage class: %+v", newSC)) + + created, err := createStorageClassIfNotExists(ctx, cl, scList, newSC) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to create a Storage Class %s: %w", newSC.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] a storage class %s was created: %t", newSC.Name, created)) + if created { + log.Info(fmt.Sprintf("[reconcileStorageClassCreateFunc] successfully create storage class, name: %s", newSC.Name)) + } else { + log.Warning(fmt.Sprintf("[reconcileLSCCreateFunc] Storage class %s already exists. Adding event to requeue.", newSC.Name)) + return true, nil + } + + return false, nil +} + +func reconcileStorageClassUpdateFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + scList *v1.StorageClassList, + nsc *v1alpha1.NFSStorageClass, + controllerNamespace string, +) (bool, error) { + + log.Debug(fmt.Sprintf("[reconcileStorageClassUpdateFunc] starts for NFSStorageClass %q", nsc.Name)) + + var oldSC *v1.StorageClass + for _, s := range scList.Items { + if s.Name == nsc.Name { + oldSC = &s + break + } + } + + if oldSC == nil { + err := fmt.Errorf("a storage class %s does not exist", nsc.Name) + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to find a storage class for the NFSStorageClass %s: %w", nsc.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Debug(fmt.Sprintf("[reconcileStorageClassUpdateFunc] successfully found a storage class for the NFSStorageClass, name: %s", nsc.Name)) + + log.Trace(fmt.Sprintf("[reconcileStorageClassUpdateFunc] storage class: %+v", oldSC)) + newSC, err := ConfigureStorageClass(nsc, controllerNamespace) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to configure a Storage Class for the NFSStorageClass %s: %w", nsc.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return false, err + } + + diff, err := GetSCDiff(oldSC, newSC) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] error occured while identifying the difference between the existed StorageClass %s and the new one: %w", newSC.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + if diff != "" { + log.Info(fmt.Sprintf("[reconcileStorageClassUpdateFunc] current Storage Class LVMVolumeGroups do not match NFSStorageClass ones. The Storage Class %s will be recreated with new ones", nsc.Name)) + + err = recreateStorageClass(ctx, cl, oldSC, newSC) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to recreate a Storage Class %s: %w", newSC.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Info(fmt.Sprintf("[reconcileStorageClassUpdateFunc] a Storage Class %s was successfully recreated", newSC.Name)) + } + + return false, nil +} + +func reconcileStorageClassDeleteFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + scList *v1.StorageClassList, + nsc *v1alpha1.NFSStorageClass, +) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileStorageClassDeleteFunc] tries to find a storage class for the NFSStorageClass %s", nsc.Name)) + var sc *v1.StorageClass + for _, s := range scList.Items { + if s.Name == nsc.Name { + sc = &s + break + } + } + if sc == nil { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] no storage class found for the NFSStorageClass, name: %s", nsc.Name)) + } + + if sc != nil { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] successfully found a storage class for the NFSStorageClass %s", nsc.Name)) + log.Debug(fmt.Sprintf("[reconcileStorageClassDeleteFunc] starts identifing a provisioner for the storage class %s", sc.Name)) + + if sc.Provisioner != NFSStorageClassProvisioner { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] the storage class %s does not belongs to %s provisioner. It will not be deleted", sc.Name, NFSStorageClassProvisioner)) + } else { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] the storage class %s belongs to %s provisioner. It will be deleted", sc.Name, NFSStorageClassProvisioner)) + + err := deleteStorageClass(ctx, cl, sc) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassDeleteFunc] unable to delete a storage class %s: %w", sc.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to delete a storage class, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileStorageClassDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] successfully deleted a storage class, name: %s", sc.Name)) + } + } + + log.Debug("[reconcileStorageClassDeleteFunc] ends the reconciliation") + return false, nil +} + +func ReconcileSecretCreateFunc(ctx context.Context, cl client.Client, log logger.Logger, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileSecretCreateFunc] starts for NFSStorageClass %q", nsc.Name)) + + newSecret := configureSecret(nsc, controllerNamespace) + log.Debug(fmt.Sprintf("[reconcileSecretCreateFunc] successfully configurated secret for the NFSStorageClass, name: %s", nsc.Name)) + log.Trace(fmt.Sprintf("[reconcileSecretCreateFunc] secret: %+v", newSecret)) + + err := cl.Create(ctx, newSecret) + if err != nil { + err = fmt.Errorf("[reconcileSecretCreateFunc] unable to create a Secret %s: %w", newSecret.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileSecretCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + return false, nil +} + +func reconcileSecretUpdateFunc(ctx context.Context, cl client.Client, log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileSecretUpdateFunc] starts for secret %q", SecretForMountOptionsPrefix+nsc.Name)) + + var oldSecret *corev1.Secret + for _, s := range secretList.Items { + if s.Name == SecretForMountOptionsPrefix+nsc.Name { + oldSecret = &s + break + } + } + + if oldSecret == nil { + err := fmt.Errorf("[reconcileSecretUpdateFunc] unable to find a secret %s for the NFSStorageClass, name: %s", SecretForMountOptionsPrefix+nsc.Name, nsc.Name) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileSecretUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Debug(fmt.Sprintf("[reconcileSecretUpdateFunc] successfully found a secret %q for the NFSStorageClass, name: %q", oldSecret.Name, nsc.Name)) + + newSecret := configureSecret(nsc, controllerNamespace) + + log.Trace(fmt.Sprintf("[reconcileSecretUpdateFunc] old secret: %+v", oldSecret)) + log.Trace(fmt.Sprintf("[reconcileSecretUpdateFunc] new secret: %+v", newSecret)) + + err := cl.Update(ctx, newSecret) + if err != nil { + err = fmt.Errorf("[reconcileSecretUpdateFunc] unable to update a Secret %s: %w", newSecret.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileSecretUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Info(fmt.Sprintf("[reconcileSecretUpdateFunc] ends the reconciliation for Secret %q", newSecret.Name)) + + return false, nil +} + +func reconcileSecretDeleteFunc(ctx context.Context, cl client.Client, log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] tries to find a secret for the NFSStorageClass %q with name %q", nsc.Name, SecretForMountOptionsPrefix+nsc.Name)) + var secret *corev1.Secret + for _, s := range secretList.Items { + if s.Name == SecretForMountOptionsPrefix+nsc.Name { + secret = &s + break + } + } + if secret == nil { + log.Info(fmt.Sprintf("[reconcileSecretDeleteFunc] no secret found for the NFSStorageClass, name: %s", nsc.Name)) + } + + if secret != nil { + log.Info(fmt.Sprintf("[reconcileSecretDeleteFunc] successfully found a secret for the NFSStorageClass %s", nsc.Name)) + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] starts removing a finalizer %s from the Secret, name: %s", NFSStorageClassFinalizerName, secret.Name)) + _, err := removeFinalizerIfExists(ctx, cl, secret, NFSStorageClassFinalizerName) + if err != nil { + err = fmt.Errorf("[reconcileSecretDeleteFunc] unable to remove a finalizer %s from the Secret %s: %w", NFSStorageClassFinalizerName, secret.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to remove a finalizer, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileSecretDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + + err = cl.Delete(ctx, secret) + if err != nil { + err = fmt.Errorf("[reconcileSecretDeleteFunc] unable to delete a secret %s: %w", secret.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to delete a secret, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileSecretDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + } + + log.Info(fmt.Sprintf("[reconcileSecretDeleteFunc] ends the reconciliation for Secret %q", SecretForMountOptionsPrefix+nsc.Name)) + + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] starts removing a finalizer %s from the NFSStorageClass, name: %s", NFSStorageClassFinalizerName, nsc.Name)) + removed, err := removeFinalizerIfExists(ctx, cl, nsc, NFSStorageClassFinalizerName) + if err != nil { + err = fmt.Errorf("[reconcileSecretDeleteFunc] unable to remove a finalizer %s from the NFSStorageClass %s: %w", NFSStorageClassFinalizerName, nsc.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to remove a finalizer, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileSecretDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] the NFSStorageClass %s finalizer %s was removed: %t", nsc.Name, NFSStorageClassFinalizerName, removed)) + + return false, nil +} + +func IdentifyReconcileFuncForStorageClass(log logger.Logger, scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (reconcileType string, err error) { + if shouldReconcileByDeleteFunc(nsc) { + return DeleteReconcile, nil + } + + if shouldReconcileStorageClassByCreateFunc(scList, nsc) { + return CreateReconcile, nil + } + + should, err := shouldReconcileStorageClassByUpdateFunc(log, scList, nsc, controllerNamespace) + if err != nil { + return "", err + } + if should { + return UpdateReconcile, nil + } + + return "", nil +} + +func shouldReconcileStorageClassByCreateFunc(scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass) bool { + if nsc.DeletionTimestamp != nil { + return false + } + + for _, sc := range scList.Items { + if sc.Name == nsc.Name { + return false + } + } + + return true +} + +func shouldReconcileStorageClassByUpdateFunc(log logger.Logger, scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + if nsc.DeletionTimestamp != nil { + return false, nil + } + + for _, oldSC := range scList.Items { + if oldSC.Name == nsc.Name { + if oldSC.Provisioner == NFSStorageClassProvisioner { + newSC, err := ConfigureStorageClass(nsc, controllerNamespace) + if err != nil { + return false, err + } + + diff, err := GetSCDiff(&oldSC, newSC) + if err != nil { + return false, err + } + + if diff != "" { + log.Debug(fmt.Sprintf("[shouldReconcileStorageClassByUpdateFunc] a storage class %s should be updated. Diff: %s", oldSC.Name, diff)) + return true, nil + } + + if nsc.Status != nil && nsc.Status.Phase == FailedStatusPhase { + return true, nil + } + + return false, nil + + } else { + err := fmt.Errorf("a storage class %s does not belong to %s provisioner", oldSC.Name, NFSStorageClassProvisioner) + return false, err + } + } + } + + err := fmt.Errorf("a storage class %s does not exist", nsc.Name) + return false, err +} + +func shouldReconcileByDeleteFunc(nsc *v1alpha1.NFSStorageClass) bool { + if nsc.DeletionTimestamp != nil { + return true + } + + return false +} + +func removeFinalizerIfExists(ctx context.Context, cl client.Client, obj metav1.Object, finalizerName string) (bool, error) { + removed := false + finalizers := obj.GetFinalizers() + for i, f := range finalizers { + if f == finalizerName { + finalizers = append(finalizers[:i], finalizers[i+1:]...) + removed = true + break + } + } + + if removed { + obj.SetFinalizers(finalizers) + err := cl.Update(ctx, obj.(client.Object)) + if err != nil { + return false, err + } + } + + return removed, nil +} + +func GetSCDiff(oldSC, newSC *v1.StorageClass) (string, error) { + + if oldSC.Provisioner != newSC.Provisioner { + err := fmt.Errorf("NFSStorageClass %q: the provisioner field is different in the StorageClass %q", newSC.Name, oldSC.Name) + return "", err + } + + if *oldSC.ReclaimPolicy != *newSC.ReclaimPolicy { + diff := fmt.Sprintf("ReclaimPolicy: %q -> %q", *oldSC.ReclaimPolicy, *newSC.ReclaimPolicy) + return diff, nil + } + + if *oldSC.VolumeBindingMode != *newSC.VolumeBindingMode { + diff := fmt.Sprintf("VolumeBindingMode: %q -> %q", *oldSC.VolumeBindingMode, *newSC.VolumeBindingMode) + return diff, nil + } + + if *oldSC.AllowVolumeExpansion != *newSC.AllowVolumeExpansion { + diff := fmt.Sprintf("AllowVolumeExpansion: %t -> %t", *oldSC.AllowVolumeExpansion, *newSC.AllowVolumeExpansion) + return diff, nil + } + + if !reflect.DeepEqual(oldSC.Parameters, newSC.Parameters) { + diff := fmt.Sprintf("Parameters: %+v -> %+v", oldSC.Parameters, newSC.Parameters) + return diff, nil + } + + if !reflect.DeepEqual(oldSC.MountOptions, newSC.MountOptions) { + diff := fmt.Sprintf("MountOptions: %v -> %v", oldSC.MountOptions, newSC.MountOptions) + return diff, nil + } + + return "", nil +} + +func createStorageClassIfNotExists(ctx context.Context, cl client.Client, scList *v1.StorageClassList, sc *v1.StorageClass) (bool, error) { + for _, s := range scList.Items { + if s.Name == sc.Name { + return false, nil + } + } + + err := cl.Create(ctx, sc) + if err != nil { + return false, err + } + + return true, err +} + +func addFinalizerIfNotExistsForNSC(ctx context.Context, cl client.Client, nsc *v1alpha1.NFSStorageClass) (bool, error) { + if !slices.Contains(nsc.Finalizers, NFSStorageClassFinalizerName) { + nsc.Finalizers = append(nsc.Finalizers, NFSStorageClassFinalizerName) + } + + err := cl.Update(ctx, nsc) + if err != nil { + return false, err + } + + return true, nil +} + +func ConfigureStorageClass(nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (*v1.StorageClass, error) { + if nsc.Spec.ReclaimPolicy == "" { + err := fmt.Errorf("NFSStorageClass %q: the ReclaimPolicy field is empty", nsc.Name) + return nil, err + } + if nsc.Spec.VolumeBindingMode == "" { + err := fmt.Errorf("NFSStorageClass %q: the VolumeBindingMode field is empty", nsc.Name) + return nil, err + } + + reclaimPolicy := corev1.PersistentVolumeReclaimPolicy(nsc.Spec.ReclaimPolicy) + volumeBindingMode := v1.VolumeBindingMode(nsc.Spec.VolumeBindingMode) + AllowVolumeExpansion := AllowVolumeExpansionDefaultValue + + sc := &v1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: StorageClassKind, + APIVersion: StorageClassAPIVersion, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nsc.Name, + Namespace: nsc.Namespace, + Finalizers: []string{NFSStorageClassFinalizerName}, + }, + Parameters: GetSCParams(nsc, controllerNamespace), + MountOptions: GetSCMountOptions(nsc), + Provisioner: NFSStorageClassProvisioner, + ReclaimPolicy: &reclaimPolicy, + VolumeBindingMode: &volumeBindingMode, + AllowVolumeExpansion: &AllowVolumeExpansion, + } + + return sc, nil +} + +func updateNFSStorageClassPhase(ctx context.Context, cl client.Client, nsc *v1alpha1.NFSStorageClass, phase, reason string) error { + if nsc.Status == nil { + nsc.Status = &v1alpha1.NFSStorageClassStatus{} + } + nsc.Status.Phase = phase + nsc.Status.Reason = reason + + // TODO: add retry logic + err := cl.Status().Update(ctx, nsc) + if err != nil { + return err + } + + return nil +} + +func recreateStorageClass(ctx context.Context, cl client.Client, oldSC, newSC *v1.StorageClass) error { + // It is necessary to pass the original StorageClass to the delete operation because + // the deletion will not succeed if the fields in the StorageClass provided to delete + // differ from those currently in the cluster. + err := deleteStorageClass(ctx, cl, oldSC) + if err != nil { + err = fmt.Errorf("[recreateStorageClass] unable to delete a storage class %s: %s", oldSC.Name, err.Error()) + return err + } + + err = cl.Create(ctx, newSC) + if err != nil { + err = fmt.Errorf("[recreateStorageClass] unable to create a storage class %s: %s", newSC.Name, err.Error()) + return err + } + + return nil +} + +func deleteStorageClass(ctx context.Context, cl client.Client, sc *v1.StorageClass) error { + if sc.Provisioner != NFSStorageClassProvisioner { + return fmt.Errorf("a storage class %s does not belong to %s provisioner", sc.Name, NFSStorageClassProvisioner) + } + + _, err := removeFinalizerIfExists(ctx, cl, sc, NFSStorageClassFinalizerName) + if err != nil { + return err + } + + err = cl.Delete(ctx, sc) + if err != nil { + return err + } + + return nil +} + +func GetSCMountOptions(nsc *v1alpha1.NFSStorageClass) []string { + mountOptions := []string{} + + if nsc.Spec.Connection.NFSVersion != "" { + mountOptions = append(mountOptions, "nfsvers="+nsc.Spec.Connection.NFSVersion) + } + + if nsc.Spec.MountOptions != nil { + + if nsc.Spec.MountOptions.MountMode != "" { + mountOptions = append(mountOptions, nsc.Spec.MountOptions.MountMode) + } + + if nsc.Spec.MountOptions.Timeout > 0 { + mountOptions = append(mountOptions, "timeo="+strconv.Itoa(nsc.Spec.MountOptions.Timeout)) + } + + if nsc.Spec.MountOptions.Retransmissions > 0 { + mountOptions = append(mountOptions, "retrans="+strconv.Itoa(nsc.Spec.MountOptions.Retransmissions)) + } + + if nsc.Spec.MountOptions.ReadOnly != nil { + if *nsc.Spec.MountOptions.ReadOnly { + mountOptions = append(mountOptions, "ro") + } else { + mountOptions = append(mountOptions, "rw") + } + } + } + + return mountOptions +} + +func GetSCParams(nsc *v1alpha1.NFSStorageClass, controllerNamespace string) map[string]string { + params := make(map[string]string) + + params[serverParamKey] = nsc.Spec.Connection.Host + params[shareParamKey] = nsc.Spec.Connection.Share + params[StorageClassSecretNameKey] = SecretForMountOptionsPrefix + nsc.Name + params[StorageClassSecretNSKey] = controllerNamespace + + if nsc.Spec.ChmodPermissions != "" { + params[MountPermissionsParamKey] = nsc.Spec.ChmodPermissions + } + + if nsc.Spec.Connection.SubDir != "" { + params[SubDirParamKey] = nsc.Spec.Connection.SubDir + } + + return params +} + +func IdentifyReconcileFuncForSecret(log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (reconcileType string, err error) { + if shouldReconcileByDeleteFunc(nsc) { + return DeleteReconcile, nil + } + + if shouldReconcileSecretByCreateFunc(secretList, nsc) { + return CreateReconcile, nil + } + + should, err := shouldReconcileSecretByUpdateFunc(log, secretList, nsc, controllerNamespace) + if err != nil { + return "", err + } + if should { + return UpdateReconcile, nil + } + + return "", nil +} + +func shouldReconcileSecretByCreateFunc(secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass) bool { + if nsc.DeletionTimestamp != nil { + return false + } + + for _, s := range secretList.Items { + if s.Name == SecretForMountOptionsPrefix+nsc.Name { + return false + } + } + + return true +} + +func shouldReconcileSecretByUpdateFunc(log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + if nsc.DeletionTimestamp != nil { + return false, nil + } + + secretSelector := labels.Set(map[string]string{ + NFSStorageClassManagedLabelKey: NFSStorageClassManagedLabelValue, + }) + + for _, oldSecret := range secretList.Items { + if oldSecret.Name == SecretForMountOptionsPrefix+nsc.Name { + newSecret := configureSecret(nsc, controllerNamespace) + if !reflect.DeepEqual(oldSecret.StringData, newSecret.StringData) { + log.Debug(fmt.Sprintf("[shouldReconcileSecretByUpdateFunc] a secret %s should be updated", oldSecret.Name)) + if !labels.Set(oldSecret.Labels).AsSelector().Matches(secretSelector) { + err := fmt.Errorf("a secret %q does not have a label %s=%s", oldSecret.Name, NFSStorageClassManagedLabelKey, NFSStorageClassManagedLabelValue) + return false, err + } + return true, nil + } + + if !labels.Set(oldSecret.Labels).AsSelector().Matches(secretSelector) { + log.Debug(fmt.Sprintf("[shouldReconcileSecretByUpdateFunc] a secret %s should be updated. The label %s=%s is missing", oldSecret.Name, NFSStorageClassManagedLabelKey, NFSStorageClassManagedLabelValue)) + return true, nil + } + + return false, nil + } + } + + return true, nil +} + +func configureSecret(nsc *v1alpha1.NFSStorageClass, controllerNamespace string) *corev1.Secret { + mountOptions := GetSCMountOptions(nsc) + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: SecretForMountOptionsPrefix + nsc.Name, + Namespace: controllerNamespace, + Labels: map[string]string{ + NFSStorageClassManagedLabelKey: NFSStorageClassManagedLabelValue, + }, + Finalizers: []string{NFSStorageClassFinalizerName}, + }, + StringData: map[string]string{ + MountOptionsSecretKey: strings.Join(mountOptions, ","), + }, + } + + return secret +} diff --git a/images/controller/pkg/controller/nfs_storage_class_watcher_test.go b/images/controller/pkg/controller/nfs_storage_class_watcher_test.go new file mode 100644 index 0000000..7e2d25d --- /dev/null +++ b/images/controller/pkg/controller/nfs_storage_class_watcher_test.go @@ -0,0 +1,452 @@ +/* +Copyright 2023 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller_test + +import ( + "context" + v1alpha1 "d8-controller/api/v1alpha1" + "d8-controller/pkg/controller" + "d8-controller/pkg/logger" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe(controller.NFSStorageClassCtrlName, func() { + const ( + controllerNamespace = "test-namespace" + nameForTestResource = "example" + ) + var ( + ctx = context.Background() + cl = NewFakeClient() + log = logger.Logger{} + + server = "192.168.1.100" + share = "/data" + subDir = "${pvc.metadata.namespace}/${pvc.metadata.name}" + nfsVer = "4.1" + mountOptForNFSVer = fmt.Sprintf("nfsvers=%s", nfsVer) + mountMode = "hard" + mountModeUpdated = "soft" + timeout = 10 + mountOptForTimeout = "timeo=10" + retransmissions = 3 + mountOptForRetransmissions = "retrans=3" + readOnlyFalse = false + mountOptForReadOnlyFalse = "rw" + readOnlyTrue = true + mountOptForReadOnlyTrue = "ro" + chmodPermissions = "0777" + ) + + It("Create_nfs_sc_with_all_options", func() { + nfsSCtemplate := generateNFSStorageClass(NFSStorageClassConfig{ + Name: nameForTestResource, + Host: server, + Share: share, + SubDir: subDir, + NFSVersion: nfsVer, + MountMode: mountMode, + Timeout: timeout, + Retransmissions: retransmissions, + ReadOnly: &readOnlyFalse, + ChmodPermissions: chmodPermissions, + ReclaimPolicy: string(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: string(v1.VolumeBindingWaitForFirstConsumer), + }) + + err := cl.Create(ctx, nfsSCtemplate) + Expect(err).NotTo(HaveOccurred()) + + nsc := &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(0)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(5)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer, mountMode, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyFalse))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, fmt.Sprintf("%s,%s,%s,%s,%s", mountOptForNFSVer, mountMode, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyFalse))) + + }) + + It("Update_nfs_sc_1", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc.Spec.MountOptions.MountMode = mountModeUpdated + nsc.Spec.MountOptions.ReadOnly = &readOnlyTrue + + err = cl.Update(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(5)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer, mountModeUpdated, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyTrue))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, fmt.Sprintf("%s,%s,%s,%s,%s", mountOptForNFSVer, mountModeUpdated, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyTrue))) + + }) + + It("Remove_mount_options_from_nfs_sc", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc.Spec.MountOptions = nil + + err = cl.Update(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(1)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, mountOptForNFSVer)) + + }) + + It("Add_partial_mount_options_to_nfs_sc", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc.Spec.MountOptions = &v1alpha1.NFSStorageClassMountOptions{ + MountMode: mountModeUpdated, + Retransmissions: retransmissions, + } + + err = cl.Update(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(3)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer, mountModeUpdated, mountOptForRetransmissions))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, fmt.Sprintf("%s,%s,%s", mountOptForNFSVer, mountModeUpdated, mountOptForRetransmissions))) + + }) + + It("Remove_nfs_sc", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Delete(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc = &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + }) + + It("Create_nfs_sc_when_sc_with_another_provisioner_exists", func() { + sc := &v1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: nameForTestResource, + }, + Provisioner: "test-provisioner", + } + + err := cl.Create(ctx, sc) + Expect(err).NotTo(HaveOccurred()) + + nfsSCtemplate := generateNFSStorageClass(NFSStorageClassConfig{ + Name: nameForTestResource, + Host: server, + Share: share, + NFSVersion: nfsVer, + MountMode: mountMode, + ReadOnly: &readOnlyFalse, + ReclaimPolicy: string(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: string(v1.VolumeBindingWaitForFirstConsumer), + }) + + err = cl.Create(ctx, nfsSCtemplate) + Expect(err).NotTo(HaveOccurred()) + + nsc := &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).To(HaveOccurred()) + Expect(shouldRequeue).To(BeTrue()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + Expect(sc.Provisioner).To(Equal("test-provisioner")) + Expect(sc.Finalizers).To(HaveLen(0)) + Expect(sc.Labels).To(HaveLen(0)) + }) + + It("Remove_nfs_sc_when_sc_with_another_provisioner_exists", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Delete(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc = &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + Expect(sc.Provisioner).To(Equal("test-provisioner")) + Expect(sc.Finalizers).To(HaveLen(0)) + Expect(sc.Labels).To(HaveLen(0)) + }) + + // TODO: "Create_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_does_not_exists", "Create_nfs_sc_when_sc_does_not_exists_and_secret_exists", "Create_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_exists", "Update_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_does_not_exists", "Remove_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_does_not_exists", "Remove_nfs_sc_when_sc_does_not_exists_and_secret_exists" + +}) + +type NFSStorageClassConfig struct { + Name string + Host string + Share string + SubDir string + NFSVersion string + MountMode string + Timeout int + Retransmissions int + ReadOnly *bool + ChmodPermissions string + ReclaimPolicy string + VolumeBindingMode string +} + +func generateNFSStorageClass(cfg NFSStorageClassConfig) *v1alpha1.NFSStorageClass { + return &v1alpha1.NFSStorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: cfg.Name, + }, + Spec: v1alpha1.NFSStorageClassSpec{ + Connection: &v1alpha1.NFSStorageClassConnection{ + Host: cfg.Host, + Share: cfg.Share, + SubDir: cfg.SubDir, + NFSVersion: cfg.NFSVersion, + }, + MountOptions: &v1alpha1.NFSStorageClassMountOptions{ + MountMode: cfg.MountMode, + Timeout: cfg.Timeout, + Retransmissions: cfg.Retransmissions, + ReadOnly: cfg.ReadOnly, + }, + ChmodPermissions: cfg.ChmodPermissions, + ReclaimPolicy: cfg.ReclaimPolicy, + VolumeBindingMode: cfg.VolumeBindingMode, + }, + } +} + +func BoolPtr(b bool) *bool { + return &b +} + +func performStandartChecksForSc(sc *v1.StorageClass, server, share, nameForTestResource, controllerNamespace string) { + Expect(sc).NotTo(BeNil()) + Expect(sc.Name).To(Equal(nameForTestResource)) + Expect(sc.Finalizers).To(HaveLen(1)) + Expect(sc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + Expect(sc.Provisioner).To(Equal(controller.NFSStorageClassProvisioner)) + Expect(*sc.ReclaimPolicy).To(Equal(corev1.PersistentVolumeReclaimDelete)) + Expect(*sc.VolumeBindingMode).To(Equal(v1.VolumeBindingWaitForFirstConsumer)) + Expect(sc.Parameters).To(HaveKeyWithValue("server", server)) + Expect(sc.Parameters).To(HaveKeyWithValue("share", share)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.StorageClassSecretNameKey, controller.SecretForMountOptionsPrefix+nameForTestResource)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.StorageClassSecretNSKey, controllerNamespace)) +} + +func performStandartChecksForSecret(secret *corev1.Secret, nameForTestResource, controllerNamespace string) { + Expect(secret).NotTo(BeNil()) + Expect(secret.Name).To(Equal(controller.SecretForMountOptionsPrefix + nameForTestResource)) + Expect(secret.Namespace).To(Equal(controllerNamespace)) + Expect(secret.Finalizers).To(HaveLen(1)) + Expect(secret.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) +} diff --git a/images/controller/pkg/kubutils/kubernetes.go b/images/controller/pkg/kubutils/kubernetes.go new file mode 100644 index 0000000..4714cfe --- /dev/null +++ b/images/controller/pkg/kubutils/kubernetes.go @@ -0,0 +1,35 @@ +/* +Copyright 2024 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubutils + +import ( + "fmt" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func KubernetesDefaultConfigCreate() (*rest.Config, error) { + //todo validate empty + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ) + + // Get a config to talk to API server + config, err := clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("config kubernetes error %w", err) + } + return config, nil +} diff --git a/images/controller/pkg/logger/logger.go b/images/controller/pkg/logger/logger.go new file mode 100644 index 0000000..345af2b --- /dev/null +++ b/images/controller/pkg/logger/logger.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "flag" + "fmt" + "github.com/go-logr/logr" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" +) + +const ( + ErrorLevel Verbosity = "0" + WarningLevel Verbosity = "1" + InfoLevel Verbosity = "2" + DebugLevel Verbosity = "3" + TraceLevel Verbosity = "4" +) + +const ( + warnLvl = iota + 1 + infoLvl + debugLvl + traceLvl +) + +type ( + Verbosity string +) + +type Logger struct { + log logr.Logger +} + +func NewLogger(level Verbosity) (*Logger, error) { + klog.InitFlags(nil) + if err := flag.Set("v", string(level)); err != nil { + return nil, err + } + flag.Parse() + + log := klogr.New().WithCallDepth(1) + + return &Logger{log: log}, nil +} + +func (l Logger) GetLogger() logr.Logger { + return l.log +} + +func (l Logger) Error(err error, message string, keysAndValues ...interface{}) { + l.log.Error(err, fmt.Sprintf("ERROR %s", message), keysAndValues...) +} + +func (l Logger) Warning(message string, keysAndValues ...interface{}) { + l.log.V(warnLvl).Info(fmt.Sprintf("WARNING %s", message), keysAndValues...) +} + +func (l Logger) Info(message string, keysAndValues ...interface{}) { + l.log.V(infoLvl).Info(fmt.Sprintf("INFO %s", message), keysAndValues...) +} + +func (l Logger) Debug(message string, keysAndValues ...interface{}) { + l.log.V(debugLvl).Info(fmt.Sprintf("DEBUG %s", message), keysAndValues...) +} + +func (l Logger) Trace(message string, keysAndValues ...interface{}) { + l.log.V(traceLvl).Info(fmt.Sprintf("TRACE %s", message), keysAndValues...) +} diff --git a/images/csi-nfs/werf.inc.yaml b/images/csi-nfs/werf.inc.yaml index f0b4f80..258530c 100644 --- a/images/csi-nfs/werf.inc.yaml +++ b/images/csi-nfs/werf.inc.yaml @@ -16,7 +16,7 @@ shell: install: - export GO_VERSION={{ env "GOLANG_VERSION" }} - export GOPROXY={{ env "GOPROXY" }} - - git clone --depth 1 --branch {{ env "SOURCE_REPO_TAG" }} {{ env "SOURCE_REPO" }}/kubernetes-csi/csi-driver-nfs.git /csi-driver-nfs + - git clone --depth 1 --branch v4.7.0 {{ env "SOURCE_REPO" }}/kubernetes-csi/csi-driver-nfs.git /csi-driver-nfs - cd /csi-driver-nfs/cmd/nfsplugin - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o /nfsplugin - chmod +x /nfsplugin diff --git a/images/webhooks/src/go.mod b/images/webhooks/src/go.mod new file mode 100644 index 0000000..039314d --- /dev/null +++ b/images/webhooks/src/go.mod @@ -0,0 +1,35 @@ +module webhooks + +go 1.22.1 + +toolchain go1.22.2 + +require ( + github.com/sirupsen/logrus v1.9.3 + github.com/slok/kubewebhook/v2 v2.6.0 + k8s.io/api v0.30.0 + k8s.io/apimachinery v0.30.0 + k8s.io/klog/v2 v2.120.1 +) + +require ( + github.com/go-logr/logr v1.4.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/client-go v0.30.0 // indirect + k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/images/webhooks/src/go.sum b/images/webhooks/src/go.sum new file mode 100644 index 0000000..5a26076 --- /dev/null +++ b/images/webhooks/src/go.sum @@ -0,0 +1,110 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slok/kubewebhook/v2 v2.6.0 h1:NMDDXx219OcNDc17ZYpqGXW81/jkBNmkdEwFDcZDVcA= +github.com/slok/kubewebhook/v2 v2.6.0/go.mod h1:EoPfBo8lzgU1lmI1DSY/Fpwu+cdr4lZnzY4Tmg5sHe0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= +k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 h1:ao5hUqGhsqdm+bYbjH/pRkCs0unBGe9UyDahzs9zQzQ= +k8s.io/utils v0.0.0-20240423183400-0849a56e8f22/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/images/webhooks/src/handlers/func.go b/images/webhooks/src/handlers/func.go new file mode 100644 index 0000000..f0c9246 --- /dev/null +++ b/images/webhooks/src/handlers/func.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handlers + +import ( + "context" + "net/http" + + "github.com/slok/kubewebhook/v2/pkg/log" + + kwhhttp "github.com/slok/kubewebhook/v2/pkg/http" + "github.com/slok/kubewebhook/v2/pkg/model" + kwhmutating "github.com/slok/kubewebhook/v2/pkg/webhook/mutating" + kwhvalidating "github.com/slok/kubewebhook/v2/pkg/webhook/validating" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetMutatingWebhookHandler(mutationFunc func(ctx context.Context, _ *model.AdmissionReview, obj metav1.Object) (*kwhmutating.MutatorResult, error), mutatorID string, obj metav1.Object, logger log.Logger) (http.Handler, error) { + mutatorFunc := kwhmutating.MutatorFunc(mutationFunc) + + mutatingWebhookConfig := kwhmutating.WebhookConfig{ + ID: mutatorID, + Obj: obj, + Mutator: mutatorFunc, + Logger: logger, + } + + mutationWebhook, err := kwhmutating.NewWebhook(mutatingWebhookConfig) + if err != nil { + return nil, err + } + + mutationWebhookHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: mutationWebhook, Logger: logger}) + + return mutationWebhookHandler, err + +} + +func GetValidatingWebhookHandler(validationFunc func(ctx context.Context, _ *model.AdmissionReview, obj metav1.Object) (*kwhvalidating.ValidatorResult, error), validatorID string, obj metav1.Object, logger log.Logger) (http.Handler, error) { + validatorFunc := kwhvalidating.ValidatorFunc(validationFunc) + + validatingWebhookConfig := kwhvalidating.WebhookConfig{ + ID: validatorID, + Obj: obj, + Validator: validatorFunc, + Logger: logger, + } + + mutationWebhook, err := kwhvalidating.NewWebhook(validatingWebhookConfig) + if err != nil { + return nil, err + } + + mutationWebhookHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: mutationWebhook, Logger: logger}) + + return mutationWebhookHandler, err + +} diff --git a/images/webhooks/src/handlers/scValidator.go b/images/webhooks/src/handlers/scValidator.go new file mode 100644 index 0000000..e236317 --- /dev/null +++ b/images/webhooks/src/handlers/scValidator.go @@ -0,0 +1,58 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handlers + +import ( + "context" + "fmt" + + "k8s.io/klog/v2" + + "github.com/slok/kubewebhook/v2/pkg/model" + kwhvalidating "github.com/slok/kubewebhook/v2/pkg/webhook/validating" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + NFSStorageClassProvisioner = "nfs.csi.k8s.io" + allowedUserName = "system:serviceaccount:d8-csi-nfs:controller" +) + +func SCValidate(ctx context.Context, arReview *model.AdmissionReview, obj metav1.Object) (*kwhvalidating.ValidatorResult, error) { + sc, ok := obj.(*storagev1.StorageClass) + if !ok { + // If not a storage class just continue the validation chain(if there is one) and do nothing. + return &kwhvalidating.ValidatorResult{}, nil + } + + if sc.Provisioner == NFSStorageClassProvisioner { + if arReview.UserInfo.Username == allowedUserName { + klog.Infof("User %s is allowed to manage storage classes with provisioner %s", arReview.UserInfo.Username, NFSStorageClassProvisioner) + return &kwhvalidating.ValidatorResult{Valid: true}, + nil + } else { + klog.Infof("User %s is not allowed to manage storage classes with provisioner %s", arReview.UserInfo.Username, NFSStorageClassProvisioner) + return &kwhvalidating.ValidatorResult{Valid: false, Message: fmt.Sprintf("Manual operations with the StorageClass that uses the %s provisioner are not allowed. Please use NFSStorageClass instead.", NFSStorageClassProvisioner)}, + nil + } + } else { + return &kwhvalidating.ValidatorResult{Valid: true}, + nil + } + +} diff --git a/images/webhooks/src/main.go b/images/webhooks/src/main.go new file mode 100644 index 0000000..6555aab --- /dev/null +++ b/images/webhooks/src/main.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "webhooks/handlers" + + "github.com/sirupsen/logrus" + kwhlogrus "github.com/slok/kubewebhook/v2/pkg/log/logrus" + storagev1 "k8s.io/api/storage/v1" +) + +type config struct { + certFile string + keyFile string +} + +//goland:noinspection SpellCheckingInspection +func httpHandlerHealthz(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "Ok.") +} + +func initFlags() config { + cfg := config{} + + fl := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + fl.StringVar(&cfg.certFile, "tls-cert-file", "", "TLS certificate file") + fl.StringVar(&cfg.keyFile, "tls-key-file", "", "TLS key file") + + fl.Parse(os.Args[1:]) + return cfg +} + +const ( + port = ":8443" + PodSchedulerMutatorID = "PodSchedulerMutation" + LSCValidatorId = "LSCValidator" + SCValidatorId = "SCValidator" +) + +func main() { + logrusLogEntry := logrus.NewEntry(logrus.New()) + logrusLogEntry.Logger.SetLevel(logrus.DebugLevel) + logger := kwhlogrus.NewLogrus(logrusLogEntry) + + cfg := initFlags() + + scValidatingWebhookHandler, err := handlers.GetValidatingWebhookHandler(handlers.SCValidate, SCValidatorId, &storagev1.StorageClass{}, logger) + if err != nil { + fmt.Fprintf(os.Stderr, "error creating scValidatingWebhookHandler: %s", err) + os.Exit(1) + } + + mux := http.NewServeMux() + mux.Handle("/sc-validate", scValidatingWebhookHandler) + mux.HandleFunc("/healthz", httpHandlerHealthz) + + logger.Infof("Listening on %s", port) + err = http.ListenAndServeTLS(port, cfg.certFile, cfg.keyFile, mux) + if err != nil { + fmt.Fprintf(os.Stderr, "error serving webhook: %s", err) + os.Exit(1) + } +} diff --git a/images/csi-nfs-controller/werf.inc.yaml b/images/webhooks/werf.inc.yaml similarity index 51% rename from images/csi-nfs-controller/werf.inc.yaml rename to images/webhooks/werf.inc.yaml index 28bf6b2..50aafaf 100644 --- a/images/csi-nfs-controller/werf.inc.yaml +++ b/images/webhooks/werf.inc.yaml @@ -1,21 +1,16 @@ --- -image: csi-nfs-controller +image: webhooks from: "registry.deckhouse.io/base_images/golang:1.22.1-alpine@sha256:0de6cf7cceab6ecbf0718bdfb675b08b78113c3709c5e4b99456cdb2ae8c2495" git: - - url: https://github.com/kubernetes-csi/csi-driver-nfs.git - tag: v4.6.0 - add: / - to: / + - add: /images/webhooks/src + to: /src stageDependencies: setup: - "**/*" shell: setup: - - cd /cmd/nfsplugin - - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o nfsplugin - - mv nfsplugin /nfsplugin - - chmod +x /nfsplugin -docker: - ENTRYPOINT: ["/nfsplugin"] + - cd /src + - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o webhooks + - mv webhooks /webhooks diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 75c7415..e8571b8 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -1,7 +1,12 @@ type: object properties: - replicas: - type: integer - description: | - replicas count. - default: 1 + logLevel: + type: string + enum: + - ERROR + - WARN + - INFO + - DEBUG + - TRACE + description: Module log level + default: DEBUG diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml new file mode 100644 index 0000000..c901909 --- /dev/null +++ b/openapi/doc-ru-config-values.yaml @@ -0,0 +1,4 @@ +type: object +properties: + logLevel: + description: Уровень логирования модуля. diff --git a/openapi/values.yaml b/openapi/values.yaml index 2322851..d1d8cb4 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -11,6 +11,23 @@ properties: default: [] items: type: string + customWebhookCert: + type: object + default: {} + x-required-for-helm: + - crt + - key + - ca + properties: + crt: + type: string + x-examples: ["YjY0ZW5jX3N0cmluZwo="] + key: + type: string + x-examples: ["YjY0ZW5jX3N0cmluZwo="] + ca: + type: string + x-examples: ["YjY0ZW5jX3N0cmluZwo="] registry: type: object description: "System field, overwritten by Deckhouse. Don't use" diff --git a/templates/controller/deployment.yaml b/templates/controller/deployment.yaml new file mode 100644 index 0000000..795cbd4 --- /dev/null +++ b/templates/controller/deployment.yaml @@ -0,0 +1,98 @@ +{{- define "controller_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: controller + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: "controller" + minAllowed: + {{- include "controller_resources" . | nindent 8 }} + maxAllowed: + cpu: 200m + memory: 100Mi +{{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller" )) | nindent 2 }} +spec: + minAvailable: {{ include "helm_lib_is_ha_to_value" (list . 1 0) }} + selector: + matchLabels: + app: controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +spec: + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" . | nindent 2 }} + revisionHistoryLimit: 2 + selector: + matchLabels: + app: controller + template: + metadata: + labels: + app: controller + spec: + {{- include "helm_lib_priority_class" (tuple . "cluster-medium") | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple . "system") | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "system") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_nobody" . | nindent 6 }} + imagePullSecrets: + - name: {{ .Chart.Name }}-module-registry + serviceAccountName: controller + containers: + - name: controller + image: {{ include "helm_lib_module_image" (list . "controller") }} + imagePullPolicy: IfNotPresent + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} +{{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "controller_resources" . | nindent 14 }} +{{- end }} + securityContext: + privileged: true + seLinuxOptions: + level: s0 + type: spc_t + env: + - name: LOG_LEVEL +{{- if eq .Values.csiNfs.logLevel "ERROR" }} + value: "0" +{{- else if eq .Values.csiNfs.logLevel "WARN" }} + value: "1" +{{- else if eq .Values.csiNfs.logLevel "INFO" }} + value: "2" +{{- else if eq .Values.csiNfs.logLevel "DEBUG" }} + value: "3" +{{- else if eq .Values.csiNfs.logLevel "TRACE" }} + value: "4" +{{- end }} + - name: CONTROLLER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/templates/controller/rbac-for-us.yaml b/templates/controller/rbac-for-us.yaml new file mode 100644 index 0000000..a900f22 --- /dev/null +++ b/templates/controller/rbac-for-us.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - list + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:controller + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +rules: + - apiGroups: + - storage.deckhouse.io + resources: + - nfsstorageclasses + - nfsstorageclasses/status + verbs: + - get + - list + - create + - delete + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - create + - delete + - list + - get + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: controller + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: Role + name: controller + apiGroup: rbac.authorization.k8s.io + + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:controller + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: controller + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:controller + apiGroup: rbac.authorization.k8s.io + + diff --git a/templates/csi/controller.yaml b/templates/csi/controller.yaml index 7cdc485..39a0422 100644 --- a/templates/csi/controller.yaml +++ b/templates/csi/controller.yaml @@ -47,6 +47,7 @@ {{- $_ := set $csiControllerConfig "snapshotterEnabled" true }} {{- $_ := set $csiControllerConfig "resizerEnabled" false }} {{- $_ := set $csiControllerConfig "snapshotterTimeout" "1200s" }} +{{- $_ := set $csiControllerConfig "extraCreateMetadataEnabled" true }} {{- $_ := set $csiControllerConfig "livenessProbePort" 29652 }} {{- $_ := set $csiControllerConfig "additionalControllerArgs" (include "csi_controller_args" . | fromYamlArray) }} {{- $_ := set $csiControllerConfig "additionalControllerEnvs" (include "csi_controller_envs" . | fromYamlArray) }} diff --git a/templates/webhooks/deployment.yaml b/templates/webhooks/deployment.yaml new file mode 100644 index 0000000..83bcd09 --- /dev/null +++ b/templates/webhooks/deployment.yaml @@ -0,0 +1,96 @@ +{{- define "webhooks_resources" }} +cpu: 10m +memory: 50Mi +{{- end }} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks" "workload-resource-policy.deckhouse.io" "master")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: webhooks + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: webhooks + minAllowed: + {{- include "webhooks_resources" . | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 100Mi +{{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks" )) | nindent 2 }} +spec: + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" . | nindent 2 }} + selector: + matchLabels: + app: webhooks + template: + metadata: + labels: + app: webhooks + spec: + {{- include "helm_lib_priority_class" (tuple . "system-cluster-critical") | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized" "with-cloud-provider-uninitialized") | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple . "master") | nindent 6 }} + {{- include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "webhooks")) | nindent 6 }} + containers: + - name: webhooks + command: + - /webhooks + - -tls-cert-file=/etc/webhook/certs/tls.crt + - -tls-key-file=/etc/webhook/certs/tls.key + image: {{ include "helm_lib_module_image" (list . "webhooks") }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + readinessProbe: + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 5 + failureThreshold: 2 + periodSeconds: 1 + livenessProbe: + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + periodSeconds: 1 + failureThreshold: 3 + ports: + - name: http + containerPort: 8443 + protocol: TCP + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }} +{{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "webhooks_resources" . | nindent 12 }} +{{- end }} + + imagePullSecrets: + - name: {{ .Chart.Name }}-module-registry + serviceAccount: webhooks + serviceAccountName: webhooks + volumes: + - name: webhook-certs + secret: + secretName: webhooks-https-certs \ No newline at end of file diff --git a/templates/webhooks/rbac-for-us.yaml b/templates/webhooks/rbac-for-us.yaml new file mode 100644 index 0000000..9e1970b --- /dev/null +++ b/templates/webhooks/rbac-for-us.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks")) | nindent 2 }} diff --git a/templates/webhooks/secret.yaml b/templates/webhooks/secret.yaml new file mode 100644 index 0000000..ee8ca4a --- /dev/null +++ b/templates/webhooks/secret.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: webhooks-https-certs + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks")) | nindent 2 }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.csiNfs.internal.customWebhookCert.ca }} + tls.crt: {{ .Values.csiNfs.internal.customWebhookCert.crt }} + tls.key: {{ .Values.csiNfs.internal.customWebhookCert.key }} diff --git a/templates/webhooks/service.yaml b/templates/webhooks/service.yaml new file mode 100644 index 0000000..dc01ddb --- /dev/null +++ b/templates/webhooks/service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks" )) | nindent 2 }} +spec: + type: ClusterIP + ports: + - port: 443 + targetPort: 8443 + protocol: TCP + name: http + selector: + app: webhooks \ No newline at end of file diff --git a/templates/webhooks/webhook.yaml b/templates/webhooks/webhook.yaml new file mode 100644 index 0000000..891515f --- /dev/null +++ b/templates/webhooks/webhook.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: "d8-{{ .Chart.Name }}-sc-validation" +webhooks: + - name: "d8-{{ .Chart.Name }}-sc-validation.deckhouse.io" + rules: + - apiGroups: ["storage.k8s.io"] + apiVersions: ["v1"] + operations: ["*"] + resources: ["storageclasses"] + scope: "Cluster" + clientConfig: + service: + namespace: "d8-{{ .Chart.Name }}" + name: "webhooks" + path: "/sc-validate" + caBundle: | + {{ .Values.csiNfs.internal.customWebhookCert.ca }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/werf-giterminism.yaml b/werf-giterminism.yaml index 23dc40a..8145506 100644 --- a/werf-giterminism.yaml +++ b/werf-giterminism.yaml @@ -1,7 +1,7 @@ giterminismConfigVersion: 1 config: goTemplateRendering: # The rules for the Go-template functions to be able to pass build context to the release - allowEnvVariables: [ /CI_.+/, MODULES_MODULE_TAG, GOLANG_VERSION, GOPROXY, SOURCE_REPO_TAG, SOURCE_REPO ] + allowEnvVariables: [ /CI_.+/, MODULES_MODULE_TAG, GOLANG_VERSION, GOPROXY, SOURCE_REPO ] stapel: mount: allowBuildDir: true