diff --git a/.github/workflows/build_dev.yml b/.github/workflows/build_dev.yml index 23366ab..0930de9 100644 --- a/.github/workflows/build_dev.yml +++ b/.github/workflows/build_dev.yml @@ -10,7 +10,6 @@ env: GOLANG_VERSION: ${{ vars.GOLANG_VERSION }} GOPROXY: ${{ secrets.GOPROXY }} SOURCE_REPO: ${{ secrets.SOURCE_REPO }} - SOURCE_REPO_TAG: ${{ vars.SOURCE_REPO_TAG }} on: pull_request: diff --git a/.github/workflows/build_prod.yml b/.github/workflows/build_prod.yml index 561e19b..fb19f1e 100644 --- a/.github/workflows/build_prod.yml +++ b/.github/workflows/build_prod.yml @@ -11,7 +11,6 @@ env: GOLANG_VERSION: ${{ vars.GOLANG_VERSION }} GOPROXY: ${{ secrets.GOPROXY }} SOURCE_REPO: ${{ secrets.SOURCE_REPO }} - SOURCE_REPO_TAG: ${{ vars.SOURCE_REPO_TAG }} on: push: diff --git a/.gitignore b/.gitignore index 4fd25ed..dcabfa3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ __pycache__/ *.py[cod] *$py.class .pytest_cache/ + +# dev +images/controller/Makefile diff --git a/.werf/bundle.yaml b/.werf/bundle.yaml index e876254..a2242b0 100644 --- a/.werf/bundle.yaml +++ b/.werf/bundle.yaml @@ -2,6 +2,7 @@ --- image: bundle from: registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc +fromCacheVersion: "2024-05-12.1" import: # Rendering .werf/images-digests.yaml is required! - image: images-digests diff --git a/charts/deckhouse_lib_helm-1.21.0.tgz b/charts/deckhouse_lib_helm-1.21.0.tgz deleted file mode 100644 index d00bd9f..0000000 Binary files a/charts/deckhouse_lib_helm-1.21.0.tgz and /dev/null differ diff --git a/charts/deckhouse_lib_helm-1.22.0.tgz b/charts/deckhouse_lib_helm-1.22.0.tgz new file mode 100644 index 0000000..7a6b077 Binary files /dev/null and b/charts/deckhouse_lib_helm-1.22.0.tgz differ diff --git a/crds/doc-ru-nfsstorageclass.yaml b/crds/doc-ru-nfsstorageclass.yaml index d35c40b..db1a64c 100644 --- a/crds/doc-ru-nfsstorageclass.yaml +++ b/crds/doc-ru-nfsstorageclass.yaml @@ -4,11 +4,11 @@ spec: schema: openAPIV3Schema: description: | - Интерфейс управления StorageСlass для CSI-драйвера nfs.csi.storage.deckhouse.io. Ручное создание StorageClass для данного драйвера запрещено. + Интерфейс управления StorageСlass для CSI-драйвера nfs.csi.k8s.io. Ручное создание StorageClass для данного драйвера запрещено. properties: spec: properties: - server: + connection: description: | Настройки сервера NFS properties: @@ -18,10 +18,16 @@ spec: share: description: | Путь к точке монтирования на NFS сервере + subDir: + description: | + Поддиректория в NFS разделе. Если поддиректория не существует, она будет создана. Если значение subDir содержит следующие строки, они будут преобразованы в соответствующее имя pv/pvc или пространство имен: + - ${pvc.metadata.name} + - ${pvc.metadata.namespace} + - ${pv.metadata.name} nfsVersion: description: | Версия NFS сервера - options: + mountOptions: description: | Опции монтирования properties: @@ -39,7 +45,7 @@ spec: Монтирование в режиме "только чтение" chmodPermissions: description: | - Права монтирования субдиректории в NFS разделе + Права для chmod, которые будут применены к субдиректории тома в NFS разделе reclaimPolicy: description: | Режим поведения при удалении PVC. Может быть: diff --git a/crds/nfsstorageclass.yaml b/crds/nfsstorageclass.yaml index fdac42a..87832b7 100644 --- a/crds/nfsstorageclass.yaml +++ b/crds/nfsstorageclass.yaml @@ -32,10 +32,13 @@ spec: description: | Defines a Kubernetes Storage class configuration. required: - - server + - connection properties: - server: + connection: type: object + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. description: | Defines a Kubernetes Storage class configuration. required: @@ -50,6 +53,7 @@ spec: message: Value is immutable. description: | NFS server host + minLength: 1 share: type: string x-kubernetes-validations: @@ -57,6 +61,18 @@ spec: message: Value is immutable. description: | NFS server share path + minLength: 1 + subDir: + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + description: | + Sub directory under nfs share. If sub directory does not exist, it will be created. If subDir value contains following strings, it would be converted into corresponding pv/pvc name or namespace: + - ${pvc.metadata.name} + - ${pvc.metadata.namespace} + - ${pv.metadata.name} + minLength: 1 nfsVersion: type: string x-kubernetes-validations: @@ -68,7 +84,7 @@ spec: - "3" - "4.1" - "4.2" - options: + mountOptions: type: object description: | Storage class mount options @@ -84,10 +100,12 @@ spec: type: integer description: | NFS server timeout + minimum: 1 retransmissions: type: integer description: | NFS retries before fail + minimum: 1 readOnly: type: boolean description: | @@ -96,6 +114,7 @@ spec: type: string description: | chmod rights for PVs subdirectory + pattern: '^[0-7]{3,4}$' reclaimPolicy: type: string x-kubernetes-validations: @@ -138,3 +157,15 @@ spec: Additional information about the current state of the Storage Class. subresources: status: {} + additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.reason + name: Reason + type: string + priority: 1 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + description: The age of this resource diff --git a/hooks/common.py b/hooks/common.py new file mode 100644 index 0000000..062e259 --- /dev/null +++ b/hooks/common.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from lib.module import module +from typing import Callable +import json +import os +import unittest + + +NAMESPACE = "d8-csi-nfs" +MODULE_NAME = "csiNfs" + +def json_load(path: str): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + return data + +def get_dir_path() -> str: + return os.path.dirname(os.path.abspath(__file__)) diff --git a/hooks/generate_webhook_certs.py b/hooks/generate_webhook_certs.py new file mode 100755 index 0000000..e144604 --- /dev/null +++ b/hooks/generate_webhook_certs.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from lib.hooks.internal_tls import GenerateCertificateHook, TlsSecret, default_sans +from lib.module import values as module_values +from deckhouse import hook +from typing import Callable +import common + +def main(): + hook = GenerateCertificateHook( + TlsSecret( + cn="webhooks", + name="webhooks-https-certs", + sansGenerator=default_sans([ + "webhooks", + f"webhooks.{common.NAMESPACE}", + f"webhooks.{common.NAMESPACE}.svc"]), + values_path_prefix=f"{common.MODULE_NAME}.internal.customWebhookCert" + ), + cn="csi-nfs-webhooks", + common_ca=True, + namespace=common.NAMESPACE) + + hook.run() + +if __name__ == "__main__": + main() diff --git a/hooks/lib/__init__.py b/hooks/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/certificate/__init__.py b/hooks/lib/certificate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/certificate/certificate.py b/hooks/lib/certificate/certificate.py new file mode 100644 index 0000000..10a0661 --- /dev/null +++ b/hooks/lib/certificate/certificate.py @@ -0,0 +1,265 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import re +from OpenSSL import crypto +from ipaddress import ip_address +from datetime import datetime, timedelta +from lib.certificate.parse import parse_certificate, get_certificate_san + +class Certificate: + def __init__(self, cn: str, expire: int, key_size: int, algo: str) -> None: + self.key = crypto.PKey() + self.__with_key(algo=algo, size=key_size) + self.cert = crypto.X509() + self.cert.set_version(version=2) + self.cert.get_subject().CN = cn + self.cert.set_serial_number(random.getrandbits(64)) + self.cert.gmtime_adj_notBefore(0) + self.cert.gmtime_adj_notAfter(expire) + + def get_subject(self) -> crypto.X509Name: + return self.cert.get_subject() + + def __with_key(self, algo: str, size: int) -> None: + if algo == "rsa": + self.key.generate_key(crypto.TYPE_RSA, size) + elif algo == "dsa": + self.key.generate_key(crypto.TYPE_DSA, size) + else: + raise Exception(f"Algo {algo} is not support. Only [rsa, dsa]") + + def with_metadata(self, country: str = None, + state: str = None, + locality: str = None, + organisation_name: str = None, + organisational_unit_name: str = None): + """ + Adds subjects to certificate. + + :param country: Optional. The country of the entity. + :type country: :py:class:`str` + + :param state: Optional. The state or province of the entity. + :type state: :py:class:`str` + + :param locality: Optional. The locality of the entity + :type locality: :py:class:`str` + + :param organisation_name: Optional. The organization name of the entity. + :type organisation_name: :py:class:`str` + + :param organisational_unit_name: Optional. The organizational unit of the entity. + :type organisational_unit_name: :py:class:`str` + """ + + if country is not None: + self.cert.get_subject().C = country + if state is not None: + self.cert.get_subject().ST = state + if locality is not None: + self.cert.get_subject().L = locality + if organisation_name is not None: + self.cert.get_subject().O = organisation_name + if organisational_unit_name is not None: + self.cert.get_subject().OU = organisational_unit_name + return self + + def add_extension(self, type_name: str, + critical: bool, + value: str, + subject: crypto.X509 = None, + issuer: crypto.X509 = None): + """ + Adds extensions to certificate. + :param type_name: The name of the type of extension_ to create. + :type type_name: :py:class:`str` + + :param critical: A flag indicating whether this is a critical + extension. + :type critical: :py:class:`bool` + + :param value: The OpenSSL textual representation of the extension's + value. + :type value: :py:class:`str` + + :param subject: Optional X509 certificate to use as subject. + :type subject: :py:class:`crypto.X509` + + :param issuer: Optional X509 certificate to use as issuer. + :type issuer: :py:class:`crypto.X509` + """ + ext = crypto.X509Extension(type_name=str.encode(type_name), + critical=critical, + value=str.encode(value), + subject=subject, + issuer=issuer) + self.cert.add_extensions(extensions=[ext]) + return self + + def generate(self) -> (bytes, bytes): + """ + Generate certificate. + :return: (certificate, key) + :rtype: (:py:data:`bytes`, :py:data:`bytes`) + """ + pub = crypto.dump_certificate(crypto.FILETYPE_PEM, self.cert) + priv = crypto.dump_privatekey(crypto.FILETYPE_PEM, self.key) + return pub, priv + + +class CACertificateGenerator(Certificate): + """ + A class representing a generator CA certificate. + """ + def __sign(self) -> None: + self.cert.set_issuer(self.get_subject()) + self.cert.set_pubkey(self.key) + self.cert.sign(self.key, 'sha256') + + def generate(self) -> (bytes, bytes): + """ + Generate CA certificate. + :return: (ca crt, ca key) + :rtype: (:py:data:`bytes`, :py:data:`bytes`) + """ + self.add_extension(type_name="subjectKeyIdentifier", + critical=False, value="hash", subject=self.cert) + self.add_extension(type_name="authorityKeyIdentifier", + critical=False, value="keyid:always", issuer=self.cert) + self.add_extension(type_name="basicConstraints", + critical=False, value="CA:TRUE") + self.add_extension(type_name="keyUsage", critical=False, + value="keyCertSign, cRLSign, keyEncipherment") + self.__sign() + return super().generate() + + +class CertificateGenerator(Certificate): + """ + A class representing a generator certificate. + """ + def with_hosts(self, *hosts: str): + """ + This function is used to add subject alternative names to a certificate. + It takes a variable number of hosts as parameters, and based on the type of host (IP or DNS). + + :param hosts: Variable number of hosts to be added as subject alternative names to the certificate. + :type hosts: :py:class:`tuple` + """ + alt_names = [] + for h in hosts: + try: + ip_address(h) + alt_names.append(f"IP:{h}") + except ValueError: + if not is_valid_hostname(h): + continue + alt_names.append(f"DNS:{h}") + self.add_extension("subjectAltName", False, ", ".join(alt_names)) + return self + + def __sign(self, ca_subj: crypto.X509Name, ca_key: crypto.PKey) -> None: + self.cert.set_issuer(ca_subj) + self.cert.set_pubkey(self.key) + self.cert.sign(ca_key, 'sha256') + + def generate(self, ca_subj: crypto.X509Name, ca_key: crypto.PKey) -> (bytes, bytes): + """ + Generate certificate. + :param ca_subj: CA subject. + :type ca_subj: :py:class:`crypto.X509Name` + :param ca_key: CA Key. + :type ca_key: :py:class:`crypto.PKey` + :return: (certificate, key) + :rtype: (:py:data:`bytes`, :py:data:`bytes`) + """ + self.__sign(ca_subj, ca_key) + return super().generate() + +def is_valid_hostname(hostname: str) -> bool: + if len(hostname) > 255: + return False + hostname.rstrip(".") + allowed = re.compile("(?!-)[A-Z\d-]{1,63}(? bool: + """ + Check certificate + :param crt: Certificate + :type crt: :py:class:`crypto.X509` + :param cert_outdated_duration: certificate outdated duration + :type cert_outdated_duration: :py:class:`timedelta` + :return: + if timeNow > expire - cert_outdated_duration: + return True + return False + :rtype: :py:class:`bool` + """ + not_after = datetime.strptime( + crt.get_notAfter().decode('ascii'), '%Y%m%d%H%M%SZ') + if datetime.now() > not_after - cert_outdated_duration: + return True + return False + + +def is_outdated_ca(ca: str, cert_outdated_duration: timedelta) -> bool: + """ + Issue a new certificate if there is no CA in the secret. Without CA it is not possible to validate the certificate. + Check CA duration. + :param ca: Raw CA + :type ca: :py:class:`str` + :param cert_outdated_duration: certificate outdated duration + :type cert_outdated_duration: :py:class:`timedelta` + :rtype: :py:class:`bool` + """ + if len(ca) == 0: + return True + crt = parse_certificate(ca) + return cert_renew_deadline_exceeded(crt, cert_outdated_duration) + + +def is_irrelevant_cert(crt_data: str, sans: list, cert_outdated_duration: timedelta) -> bool: + """ + Check certificate duration and SANs list + :param crt_data: Raw certificate + :type crt_data: :py:class:`str` + :param sans: List of sans. + :type sans: :py:class:`list` + :param cert_outdated_duration: certificate outdated duration + :type cert_outdated_duration: :py:class:`timedelta` + :rtype: :py:class:`bool` + """ + if len(crt_data) == 0: + return True + crt = parse_certificate(crt_data) + if cert_renew_deadline_exceeded(crt, cert_outdated_duration): + return True + alt_names = [] + for san in sans: + try: + ip_address(san) + alt_names.append(f"IP Address:{san}") + except ValueError: + alt_names.append(f"DNS:{san}") + cert_sans = get_certificate_san(crt) + cert_sans.sort() + alt_names.sort() + if cert_sans != alt_names: + return True + return False \ No newline at end of file diff --git a/hooks/lib/certificate/parse.py b/hooks/lib/certificate/parse.py new file mode 100644 index 0000000..fd88c0d --- /dev/null +++ b/hooks/lib/certificate/parse.py @@ -0,0 +1,31 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from OpenSSL import crypto +from pprint import pprint + + +def parse_certificate(crt: str) -> crypto.X509: + return crypto.load_certificate(crypto.FILETYPE_PEM, crt) + + +def get_certificate_san(crt: crypto.X509) -> list[str]: + san = '' + ext_count = crt.get_extension_count() + for i in range(0, ext_count): + ext = crt.get_extension(i) + if 'subjectAltName' in str(ext.get_short_name()): + san = ext.__str__() + return san.split(', ') diff --git a/hooks/lib/hooks/__init__.py b/hooks/lib/hooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/hooks/copy_custom_certificate.py b/hooks/lib/hooks/copy_custom_certificate.py new file mode 100644 index 0000000..b3f3e83 --- /dev/null +++ b/hooks/lib/hooks/copy_custom_certificate.py @@ -0,0 +1,84 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from typing import Callable +from lib.module import module +from lib.hooks.hook import Hook + +class CopyCustomCertificatesHook(Hook): + CUSTOM_CERTIFICATES_SNAPSHOT_NAME = "custom_certificates" + def __init__(self, + module_name: str = None): + super().__init__(module_name=module_name) + self.queue = f"/modules/{self.module_name}/copy-custom-certificates" + + def generate_config(self) -> dict: + return { + "configVersion": "v1", + "beforeHelm": 10, + "kubernetes": [ + { + "name": self.CUSTOM_CERTIFICATES_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Secret", + "labelSelector": { + "matchExpressions": [ + { + "key": "owner", + "operator": "NotIn", + "values": ["helm"] + } + ] + }, + "namespace": { + "nameSelector": { + "matchNames": ["d8-system"] + } + }, + "includeSnapshotsFrom": [self.CUSTOM_CERTIFICATES_SNAPSHOT_NAME], + "jqFilter": '{"name": .metadata.name, "data": .data}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + ] + } + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + custom_certificates = {} + for s in ctx.snapshots.get(self.CUSTOM_CERTIFICATES_SNAPSHOT_NAME, []): + custom_certificates[s["filterResult"]["name"]] = s["filterResult"]["data"] + if len(custom_certificates) == 0: + return + + https_mode = module.get_https_mode(module_name=self.module_name, + values=ctx.values) + path = f"{self.module_name}.internal.customCertificateData" + if https_mode != "CustomCertificate": + self.delete_value(path, ctx.values) + return + + raw_secret_name = module.get_values_first_defined(ctx.values, + f"{self.module_name}.https.customCertificate.secretName", + "global.modules.https.customCertificate.secretName") + secret_name = str(raw_secret_name or "") + secret_data = custom_certificates.get(secret_name) + if secret_data is None: + print( + f"Custom certificate secret name is configured, but secret d8-system/{secret_name} doesn't exist") + return + self.set_value(path, ctx.values, secret_data) + return r \ No newline at end of file diff --git a/hooks/lib/hooks/hook.py b/hooks/lib/hooks/hook.py new file mode 100644 index 0000000..e412fae --- /dev/null +++ b/hooks/lib/hooks/hook.py @@ -0,0 +1,56 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from typing import Callable +from lib.module import module +from lib.module import values as module_values +import yaml + +class Hook: + def __init__(self, module_name: str = None) -> None: + self.module_name = self.get_module_name(module_name) + + def generate_config(self): + pass + + @staticmethod + def get_value(path: str, values: dict, default=None): + return module_values.get_value(path, values, default) + + @staticmethod + def set_value(path: str, values: dict, value: str) -> None: + return module_values.set_value(path, values, value) + + @staticmethod + def delete_value(path: str, values: dict) -> None: + return module_values.delete_value(path, values) + + @staticmethod + def get_module_name(module_name: str) -> str: + if module_name is not None: + return module_name + return module.get_module_name() + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + pass + return r + + def run(self) -> None: + conf = self.generate_config() + if isinstance(conf, dict): + conf = yaml.dump(conf) + hook.run(func=self.reconcile(), config=conf) diff --git a/hooks/lib/hooks/internal_tls.py b/hooks/lib/hooks/internal_tls.py new file mode 100644 index 0000000..3b351c6 --- /dev/null +++ b/hooks/lib/hooks/internal_tls.py @@ -0,0 +1,434 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from datetime import timedelta +from OpenSSL import crypto +from typing import Callable +from lib.hooks.hook import Hook +import lib.utils as utils +import lib.certificate.certificate as certificate + +PUBLIC_DOMAIN_PREFIX = "%PUBLIC_DOMAIN%://" +CLUSTER_DOMAIN_PREFIX = "%CLUSTER_DOMAIN%://" + + +KEY_USAGES = { + 0: "digitalSignature", + 1: "nonRepudiation", + 2: "keyEncipherment", + 3: "dataEncipherment", + 4: "keyAgreement", + 5: "keyCertSign", + 6: "cRLSign", + 7: "encipherOnly", + 8: "decipherOnly" +} + +EXTENDED_KEY_USAGES = { + 0: "serverAuth", + 1: "clientAuth", + 2: "codeSigning", + 3: "emailProtection", + 4: "OCSPSigning" +} + +class TlsSecret: + def __init__(self, + cn: str, + name: str, + sansGenerator: Callable[[list[str]], Callable[[hook.Context], list[str]]], + values_path_prefix: str, + key_usages: list[str] = [KEY_USAGES[2], KEY_USAGES[5]], + extended_key_usages: list[str] = [EXTENDED_KEY_USAGES[0]]): + self.cn = cn + self.name = name + self.sansGenerator = sansGenerator + self.values_path_prefix = values_path_prefix + self.key_usages = key_usages + self.extended_key_usages = extended_key_usages + +class GenerateCertificateHook(Hook): + """ + Config for the hook that generates certificates. + """ + SNAPSHOT_SECRETS_NAME = "secrets" + SNAPSHOT_SECRETS_CHECK_NAME = "secretsCheck" + + def __init__(self, *tls_secrets: TlsSecret, + cn: str, + namespace: str, + module_name: str = None, + common_ca: bool = False, + before_hook_check: Callable[[hook.Context], bool] = None, + expire: int = 31536000, + key_size: int = 4096, + algo: str = "rsa", + cert_outdated_duration: timedelta = timedelta(days=30), + country: str = None, + state: str = None, + locality: str = None, + organisation_name: str = None, + organisational_unit_name: str = None) -> None: + super().__init__(module_name=module_name) + self.cn = cn + self.tls_secrets = tls_secrets + self.namespace = namespace + self.common_ca = common_ca + self.before_hook_check = before_hook_check + self.expire = expire + self.key_size = key_size + self.algo = algo + self.cert_outdated_duration = cert_outdated_duration + self.country = country + self.state = state + self.locality = locality + self.organisation_name = organisation_name + self.organisational_unit_name = organisational_unit_name + self.secret_names = [secret.name for secret in self.tls_secrets] + self.queue = f"/modules/{self.module_name}/generate-certs" + """ + :param module_name: Module name + :type module_name: :py:class:`str` + + :param cn: Certificate common Name. often it is module name + :type cn: :py:class:`str` + + :param sansGenerator: Function which returns list of domain to include into cert. Use default_sans + :type sansGenerator: :py:class:`function` + + :param namespace: Namespace for TLS secret. + :type namespace: :py:class:`str` + + :param tls_secret_name: TLS secret name. + Secret must be TLS secret type https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets. + CA certificate MUST set to ca.crt key. + :type tls_secret_name: :py:class:`str` + + :param values_path_prefix: Prefix full path to store CA certificate TLS private key and cert. + full paths will be + values_path_prefix + .`ca` - CA certificate + values_path_prefix + .`crt` - TLS private key + values_path_prefix + .`key` - TLS certificate + Example: values_path_prefix = 'virtualization.internal.dvcrCert' + Data in values store as plain text + :type values_path_prefix: :py:class:`str` + + :param key_usages: Optional. key_usages specifies valid usage contexts for keys. + :type key_usages: :py:class:`list` + + :param extended_key_usages: Optional. extended_key_usages specifies valid usage contexts for keys. + :type extended_key_usages: :py:class:`list` + + :param before_hook_check: Optional. Runs check function before hook execution. Function should return boolean 'continue' value + if return value is false - hook will stop its execution + if return value is true - hook will continue + :type before_hook_check: :py:class:`function` + + :param expire: Optional. Validity period of SSL certificates. + :type expire: :py:class:`int` + + :param key_size: Optional. Key Size. + :type key_size: :py:class:`int` + + :param algo: Optional. Key generation algorithm. Supports only rsa and dsa. + :type algo: :py:class:`str` + + :param cert_outdated_duration: Optional. (expire - cert_outdated_duration) is time to regenerate the certificate. + :type cert_outdated_duration: :py:class:`timedelta` + """ + + def generate_config(self) -> dict: + return { + "configVersion": "v1", + "beforeHelm": 5, + "kubernetes": [ + { + "name": self.SNAPSHOT_SECRETS_NAME, + "apiVersion": "v1", + "kind": "Secret", + "nameSelector": { + "matchNames": self.secret_names + }, + "namespace": { + "nameSelector": { + "matchNames": [self.namespace] + } + }, + "includeSnapshotsFrom": [self.SNAPSHOT_SECRETS_NAME], + "jqFilter": '{"name": .metadata.name, "data": .data}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + ], + "schedule": [ + { + "name": self.SNAPSHOT_SECRETS_CHECK_NAME, + "crontab": "42 4 * * *" + } + ] + } + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + if self.before_hook_check is not None: + passed = self.before_hook_check(ctx) + if not passed: + return + + regenerate_all = False + secrets_from_snaps = {} + diff_secrets = [] + if len(ctx.snapshots.get(self.SNAPSHOT_SECRETS_NAME, [])) == 0: + regenerate_all = True + else: + for snap in ctx.snapshots[self.SNAPSHOT_SECRETS_NAME]: + secrets_from_snaps[snap["filterResult"]["name"]] = snap["filterResult"]["data"] + for secret in self.tls_secrets: + if secrets_from_snaps.get(secret.name) is None: + diff_secrets.append(secret.name) + + if self.common_ca and not regenerate_all: + if len(diff_secrets) > 0: + regenerate_all = True + else: + for secret in self.tls_secrets: + data = secrets_from_snaps[secret.name] + if self.is_outdated_ca(utils.base64_decode(data.get("ca.crt", ""))): + regenerate_all = True + break + sans = secret.sansGenerator(ctx) + if self.is_irrelevant_cert(utils.base64_decode(data.get("tls.crt", "")), sans): + regenerate_all = True + break + + if regenerate_all: + if self.common_ca: + ca = self.__get_ca_generator() + ca_crt, _ = ca.generate() + for secret in self.tls_secrets: + sans = secret.sansGenerator(ctx) + print(f"Generate new certififcates for secret {secret.name}.") + tls_data = self.generate_selfsigned_tls_data_with_ca(cn=secret.cn, + ca=ca, + ca_crt=ca_crt, + sans=sans, + key_usages=secret.key_usages, + extended_key_usages=secret.extended_key_usages) + self.set_value(secret.values_path_prefix, ctx.values, tls_data) + return + + for secret in self.tls_secrets: + sans = secret.sansGenerator(ctx) + print(f"Generate new certififcates for secret {secret.name}.") + tls_data = self.generate_selfsigned_tls_data(cn=secret.cn, + sans=sans, + key_usages=secret.key_usages, + extended_key_usages=secret.extended_key_usages) + self.set_value(secret.values_path_prefix, ctx.values, tls_data) + return + + for secret in self.tls_secrets: + data = secrets_from_snaps[secret.name] + sans = secret.sansGenerator(ctx) + cert_outdated = self.is_irrelevant_cert( + utils.base64_decode(data.get("tls.crt", "")), sans) + + tls_data = {} + if cert_outdated or data.get("tls.key", "") == "": + print(f"Certificates from secret {secret.name} is invalid. Generate new certififcates.") + tls_data = self.generate_selfsigned_tls_data(cn=secret.cn, + sans=sans, + key_usages=secret.key_usages, + extended_key_usages=secret.extended_key_usages) + else: + tls_data = { + "ca": data["ca.crt"], + "crt": data["tls.crt"], + "key": data["tls.key"] + } + self.set_value(secret.values_path_prefix, ctx.values, tls_data) + return r + + def __get_ca_generator(self) -> certificate.CACertificateGenerator: + return certificate.CACertificateGenerator(cn=f"{self.cn}", + expire=self.expire, + key_size=self.key_size, + algo=self.algo) + + def generate_selfsigned_tls_data_with_ca(self, + cn: str, + ca: certificate.CACertificateGenerator, + ca_crt: bytes, + sans: list[str], + key_usages: list[str], + extended_key_usages: list[str]) -> dict[str, str]: + """ + Generate self signed certificate. + :param cn: certificate common name. + :param ca: Ca certificate generator. + :type ca: :py:class:`certificate.CACertificateGenerator` + :param ca_crt: bytes. + :type ca_crt: :py:class:`bytes` + :param sans: List of sans. + :type sans: :py:class:`list` + :param key_usages: List of key_usages. + :type key_usages: :py:class:`list` + :param extended_key_usages: List of extended_key_usages. + :type extended_key_usages: :py:class:`list` + Example: { + "ca": "encoded in base64", + "crt": "encoded in base64", + "key": "encoded in base64" + } + :rtype: :py:class:`dict[str, str]` + """ + cert = certificate.CertificateGenerator(cn=cn, + expire=self.expire, + key_size=self.key_size, + algo=self.algo) + if len(key_usages) > 0: + key_usages = ", ".join(key_usages) + cert.add_extension(type_name="keyUsage", + critical=False, value=key_usages) + if len(extended_key_usages) > 0: + extended_key_usages = ", ".join(extended_key_usages) + cert.add_extension(type_name="extendedKeyUsage", + critical=False, value=extended_key_usages) + crt, key = cert.with_metadata(country=self.country, + state=self.state, + locality=self.locality, + organisation_name=self.organisation_name, + organisational_unit_name=self.organisational_unit_name + ).with_hosts(*sans).generate(ca_subj=ca.get_subject(), + ca_key=ca.key) + return {"ca": utils.base64_encode(ca_crt), + "crt": utils.base64_encode(crt), + "key": utils.base64_encode(key)} + + def generate_selfsigned_tls_data(self, + cn: str, + sans: list[str], + key_usages: list[str], + extended_key_usages: list[str]) -> dict[str, str]: + """ + Generate self signed certificate. + :param cn: certificate common name. + :param sans: List of sans. + :type sans: :py:class:`list` + :param key_usages: List of key_usages. + :type key_usages: :py:class:`list` + :param extended_key_usages: List of extended_key_usages. + :type extended_key_usages: :py:class:`list` + Example: { + "ca": "encoded in base64", + "crt": "encoded in base64", + "key": "encoded in base64" + } + :rtype: :py:class:`dict[str, str]` + """ + ca = self.__get_ca_generator() + ca_crt, _ = ca.generate() + return self.generate_selfsigned_tls_data_with_ca(cn=cn, + ca=ca, + ca_crt=ca_crt, + sans=sans, + key_usages=key_usages, + extended_key_usages=extended_key_usages) + + def is_irrelevant_cert(self, crt_data: str, sans: list) -> bool: + """ + Check certificate duration and SANs list + :param crt_data: Raw certificate + :type crt_data: :py:class:`str` + :param sans: List of sans. + :type sans: :py:class:`list` + :rtype: :py:class:`bool` + """ + return certificate.is_irrelevant_cert(crt_data, sans, self.cert_outdated_duration) + + def is_outdated_ca(self, ca: str) -> bool: + """ + Issue a new certificate if there is no CA in the secret. Without CA it is not possible to validate the certificate. + Check CA duration. + :param ca: Raw CA + :type ca: :py:class:`str` + :rtype: :py:class:`bool` + """ + return certificate.is_outdated_ca(ca, self.cert_outdated_duration) + + def cert_renew_deadline_exceeded(self, crt: crypto.X509) -> bool: + """ + Check certificate + :param crt: Certificate + :type crt: :py:class:`crypto.X509` + :return: + if timeNow > expire - cert_outdated_duration: + return True + return False + :rtype: :py:class:`bool` + """ + return certificate.cert_renew_deadline_exceeded(crt, self.cert_outdated_duration) + +def default_sans(sans: list[str]) -> Callable[[hook.Context], list[str]]: + """ + Generate list of sans for certificate + :param sans: List of alt names. + :type sans: :py:class:`list[str]` + cluster_domain_san(san) to generate sans with respect of cluster domain (e.g.: "app.default.svc" with "cluster.local" value will give: app.default.svc.cluster.local + + public_domain_san(san) + """ + def generate_sans(ctx: hook.Context) -> list[str]: + res = ["localhost", "127.0.0.1"] + public_domain = str(ctx.values["global"]["modules"].get( + "publicDomainTemplate", "")) + cluster_domain = str( + ctx.values["global"]["discovery"].get("clusterDomain", "")) + for san in sans: + san.startswith(PUBLIC_DOMAIN_PREFIX) + if san.startswith(PUBLIC_DOMAIN_PREFIX) and public_domain != "": + san = get_public_domain_san(san, public_domain) + elif san.startswith(CLUSTER_DOMAIN_PREFIX) and cluster_domain != "": + san = get_cluster_domain_san(san, cluster_domain) + res.append(san) + return res + return generate_sans + + +def cluster_domain_san(san: str) -> str: + """ + Create template to enrich specified san with a cluster domain + :param san: San. + :type sans: :py:class:`str` + """ + return CLUSTER_DOMAIN_PREFIX + san.rstrip('.') + + +def public_domain_san(san: str) -> str: + """ + Create template to enrich specified san with a public domain + :param san: San. + :type sans: :py:class:`str` + """ + return PUBLIC_DOMAIN_PREFIX + san.rstrip('.') + + +def get_public_domain_san(san: str, public_domain: str) -> str: + return f"{san.lstrip(PUBLIC_DOMAIN_PREFIX)}.{public_domain}" + + +def get_cluster_domain_san(san: str, cluster_domain: str) -> str: + return f"{san.lstrip(CLUSTER_DOMAIN_PREFIX)}.{cluster_domain}" diff --git a/hooks/lib/hooks/manage_tenant_secrets.py b/hooks/lib/hooks/manage_tenant_secrets.py new file mode 100644 index 0000000..36d817b --- /dev/null +++ b/hooks/lib/hooks/manage_tenant_secrets.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +from typing import Callable +from lib.hooks.hook import Hook + +class ManageTenantSecretsHook(Hook): + POD_SNAPSHOT_NAME = "pods" + SECRETS_SNAPSHOT_NAME = "secrets" + NAMESPACE_SNAPSHOT_NAME = "namespaces" + + def __init__(self, + source_namespace: str, + source_secret_name: str, + pod_labels_to_follow: dict, + destination_secret_labels: dict = {}, + module_name: str = None): + super().__init__(module_name=module_name) + self.source_namespace = source_namespace + self.source_secret_name = source_secret_name + self.pod_labels_to_follow = pod_labels_to_follow + self.destination_secret_labels = destination_secret_labels + self.module_name = module_name + self.queue = f"/modules/{module_name}/manage-tenant-secrets" + + def generate_config(self) -> dict: + return { + "configVersion": "v1", + "kubernetes": [ + { + "name": self.POD_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Pod", + "includeSnapshotsFrom": [ + self.POD_SNAPSHOT_NAME, + self.SECRETS_SNAPSHOT_NAME, + self.NAMESPACE_SNAPSHOT_NAME + ], + "labelSelector": { + "matchLabels": self.pod_labels_to_follow + }, + "jqFilter": '{"namespace": .metadata.namespace}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + { + "name": self.SECRETS_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Secret", + "includeSnapshotsFrom": [ + self.POD_SNAPSHOT_NAME, + self.SECRETS_SNAPSHOT_NAME, + self.NAMESPACE_SNAPSHOT_NAME + ], + "nameSelector": { + "matchNames": [self.source_secret_name] + }, + "jqFilter": '{"data": .data, "namespace": .metadata.namespace, "type": .type}', + "queue": self.queue, + "keepFullObjectsInMemory": False + }, + { + "name": self.NAMESPACE_SNAPSHOT_NAME, + "apiVersion": "v1", + "kind": "Secret", + "includeSnapshotsFrom": [ + self.POD_SNAPSHOT_NAME, + self.SECRETS_SNAPSHOT_NAME, + self.NAMESPACE_SNAPSHOT_NAME + ], + "jqFilter": '{"name": .metadata.name, "isTerminating": any(.metadata; .deletionTimestamp != null)}', + "queue": self.queue, + "keepFullObjectsInMemory": False + } + ] + } + + def generate_secret(self, namespace: str, data: dict, secret_type: str) -> dict: + return { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": self.source_secret_name, + "namespace": namespace, + "labels": self.destination_secret_labels + }, + "data": data, + "type": secret_type + } + + def reconcile(self) -> Callable[[hook.Context], None]: + def r(ctx: hook.Context) -> None: + pod_namespaces = set([p["filterResult"]["namespace"] for p in ctx.snapshots.get(self.POD_SNAPSHOT_NAME, [])]) + secrets = ctx.snapshots.get(self.SECRETS_SNAPSHOT_NAME, []) + for ns in ctx.snapshots.get(self.NAMESPACE_SNAPSHOT_NAME, []): + if ns["filterResult"]["isTerminating"]: + pod_namespaces.discard(ns["filterResult"]["name"]) + data, secret_type, secrets_by_ns = "", "", {} + for s in secrets: + if s["filterResult"]["namespace"] == self.source_namespace: + data = s["filterResult"]["data"] + secret_type = s["filterResult"]["type"] + continue + secrets_by_ns[s["filterResult"]["namespace"]] = s["filterResult"]["data"] + + if len(data) == 0 or len(secret_type) == 0: + print(f"Registry secret {self.source_namespace}/{self.source_secret_name} not found. Skip") + return + + for ns in pod_namespaces: + secret_data = secrets_by_ns.get(ns, "") + if (secret_data != data) and (ns != self.source_namespace): + secret = self.generate_secret(namespace=ns, + data=data, + secret_type=secret_type) + print(f"Create secret {ns}/{self.source_secret_name}.") + ctx.kubernetes.create_or_update(secret) + for ns in secrets_by_ns: + if (ns in pod_namespaces) or (ns == self.source_namespace): + continue + print(f"Delete secret {ns}/{self.source_secret_name}.") + ctx.kubernetes.delete(kind="Secret", + namespace=ns, + name=self.source_secret_name) + return r + diff --git a/hooks/lib/module/__init__.py b/hooks/lib/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/module/module.py b/hooks/lib/module/module.py new file mode 100644 index 0000000..25e034d --- /dev/null +++ b/hooks/lib/module/module.py @@ -0,0 +1,59 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.module import values as module_values +import re +import os + +def get_values_first_defined(values: dict, *keys): + return _get_first_defined(values, keys) + +def _get_first_defined(values: dict, keys: tuple): + for i in range(len(keys)): + if (val := module_values.get_value(path=keys[i], values=values)) is not None: + return val + return + +def get_https_mode(module_name: str, values: dict) -> str: + module_path = f"{module_name}.https.mode" + global_path = "global.modules.https.mode" + https_mode = get_values_first_defined(values, module_path, global_path) + if https_mode is not None: + return str(https_mode) + raise Exception("https mode is not defined") + +def get_module_name() -> str: + module = "" + file_path = os.path.abspath(__file__) + external_modules_dir = os.getenv("EXTERNAL_MODULES_DIR") + for dir in os.getenv("MODULES_DIR").split(":"): + if dir.startswith(external_modules_dir): + dir = external_modules_dir + if file_path.startswith(dir): + module = re.sub(f"{dir}/?\d?\d?\d?-?", "", file_path, 1).split("/")[0] + # /deckhouse/external-modules/virtualization/mr/hooks/hook_name.py + # {-------------------------- file_path --------------------------} + # {------ MODULES_DIR ------}{---------- regexp result ----------}} + # virtualization/mr/hooks/hook_name.py + # {-module-name-}{---------------------} + # or + # /deckhouse/modules/900-virtualization/hooks/hook_name.py + # {---------------------- file_path ----------------------} + # {-- MODULES_DIR --}{---{-------- regexp result --------}} + # virtualization/hooks/hook_name.py + # {-module-name-}{-----------------} + + break + return module \ No newline at end of file diff --git a/hooks/lib/module/values.py b/hooks/lib/module/values.py new file mode 100644 index 0000000..449e444 --- /dev/null +++ b/hooks/lib/module/values.py @@ -0,0 +1,60 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def get_value(path: str, values: dict, default=None): + def get(keys: list, values: dict, default): + if len(keys) == 1: + if not isinstance(values, dict): + return default + return values.get(keys[0], default) + if not isinstance(values, dict) or values.get(keys[0]) is None: + return default + if values.get(keys[0]) is None: + return default + return get(keys[1:], values[keys[0]], default) + keys = path.lstrip(".").split(".") + return get(keys, values, default) + +def set_value(path: str, values: dict, value) -> None: + """ + Functions for save value to dict. + + Example: + path = "virtualization.internal.dvcr.cert" + values = {"virtualization": {"internal": {}}} + value = "{"ca": "ca", "crt"="tlscrt", "key"="tlskey"}" + + result values = {"virtualization": {"internal": {"dvcr": {"cert": {"ca": "ca", "crt":"tlscrt", "key":"tlskey"}}}}} + """ + def set(keys: list, values: dict, value): + if len(keys) == 1: + values[keys[0]] = value + return + if values.get(keys[0]) is None: + values[keys[0]] = {} + set(keys[1:], values[keys[0]], value) + keys = path.lstrip(".").split(".") + return set(keys, values, value) + +def delete_value(path: str, values: dict) -> None: + if get_value(path, values) is None: + return + keys = path.lstrip(".").split(".") + def delete(keys: list, values: dict) -> None: + if len(keys) == 1: + values.pop(keys[0]) + return + delete(keys[1:], values[keys[0]]) + return delete(keys, values) \ No newline at end of file diff --git a/hooks/lib/password_generator/__init__.py b/hooks/lib/password_generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/password_generator/password_generator.py b/hooks/lib/password_generator/password_generator.py new file mode 100644 index 0000000..df63ea8 --- /dev/null +++ b/hooks/lib/password_generator/password_generator.py @@ -0,0 +1,77 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import string + +def generate_random_string(length: int, letters: str) -> str: + return ''.join(random.choice(letters) for i in range(length)) + +SYMBOLS = "[]{}<>()=-_!@#$%^&*.," + +def num(length: int) -> str: + """ + Generates a random string of the given length out of numeric characters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.digits) + +def alpha(length: int) -> str: + """ + Generates a random string of the given length out of alphabetic characters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_letters) + +def symbols(length: int) -> str: + """ + Generates a random string of the given length out of symbols. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, SYMBOLS) + + +def alpha_num(length: int) -> str: + """ + Generates a random string of the given length out of alphanumeric characters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_letters + string.digits) + +def alpha_num_lower_case(length: int) -> str: + """ + Generates a random string of the given length out of alphanumeric characters without UpperCase letters. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_lowercase + string.digits) + +def alpha_num_symbols(length: int) -> str: + """ + Generates a random string of the given length out of alphanumeric characters and symbols. + :param length: length of generate string. + :type length: :py:class:`int` + :rtype: :py:class:`str` + """ + return generate_random_string(length, string.ascii_letters + string.digits + SYMBOLS) diff --git a/hooks/lib/tests/__init__.py b/hooks/lib/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hooks/lib/tests/test_copy_custom_certificate.py b/hooks/lib/tests/test_copy_custom_certificate.py new file mode 100644 index 0000000..c10e9c9 --- /dev/null +++ b/hooks/lib/tests/test_copy_custom_certificate.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.tests import testing +from lib.hooks.copy_custom_certificate import CopyCustomCertificatesHook + + +MODULE_NAME = "test" +SECRET_NAME = "secretName" +SECRET_DATA = { + "ca.crt": "CACRT", + "tls.crt": "TLSCRT", + "tls.key": "TLSKEY" + } + +hook = CopyCustomCertificatesHook(module_name=MODULE_NAME) + +binding_context = [ + { + "binding": "binding", + "snapshots": { + hook.CUSTOM_CERTIFICATES_SNAPSHOT_NAME: [ + { + "filterResult": { + "name": SECRET_NAME, + "data": SECRET_DATA + } + }, + { + "filterResult": { + "name": "test", + "data": {} + } + } + ] + } + } +] + +values_add = { + "global": { + "modules": { + "https": { + "mode": "CustomCertificate", + "customCertificate": { + "secretName": "test" + } + } + } + }, + MODULE_NAME: { + "https": { + "customCertificate": { + "secretName": SECRET_NAME + } + }, + "internal": {} + } +} + + +values_delete = { + "global": { + "modules": { + "https": { + "mode": "CertManager" + } + } + }, + MODULE_NAME: { + "internal": { + "customCertificateData": SECRET_DATA + } + } +} + + +class TestCopyCustomCertificateAdd(testing.TestHook): + def setUp(self): + self.func = hook.reconcile() + self.bindind_context = binding_context + self.values = values_add + def test_copy_custom_certificate_adding(self): + self.hook_run() + self.assertGreater(len(self.values[MODULE_NAME]["internal"].get("customCertificateData", {})), 0) + self.assertEqual(self.values[MODULE_NAME]["internal"]["customCertificateData"], SECRET_DATA) + +class TestCopyCustomCertificateDelete(testing.TestHook): + def setUp(self): + self.func = hook.reconcile() + self.bindind_context = binding_context + self.values = values_delete + def test_copy_custom_certificate_deleting(self): + self.hook_run() + self.assertEqual(len(self.values[MODULE_NAME]["internal"].get("customCertificateData", {})), 0) + + diff --git a/hooks/lib/tests/test_internal_tls_test.py b/hooks/lib/tests/test_internal_tls_test.py new file mode 100644 index 0000000..de842cd --- /dev/null +++ b/hooks/lib/tests/test_internal_tls_test.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.tests import testing +from lib.hooks.internal_tls import GenerateCertificateHook, default_sans, TlsSecret +from lib.certificate import parse +import lib.utils as utils +from OpenSSL import crypto +from ipaddress import ip_address + +NAME = "test" +MODULE_NAME = NAME +NAMESPACE = NAME +SANS = [ + NAME, + f"{NAME}.{NAMESPACE}", + f"{NAME}.{NAMESPACE}.svc" +] + +hook_generate = GenerateCertificateHook( + TlsSecret( + name=NAME, + sansGenerator=default_sans(SANS), + values_path_prefix=f"{MODULE_NAME}.internal.dvcr.cert"), + module_name=MODULE_NAME, + cn=NAME, + namespace=NAMESPACE) + +hook_regenerate = GenerateCertificateHook( + TlsSecret( + name=NAME, + sansGenerator=default_sans(SANS), + values_path_prefix=f"{MODULE_NAME}.internal.dvcr.cert"), + module_name=MODULE_NAME, + cn=NAME, + namespace=NAMESPACE, + expire=0) + +binding_context = [ + { + "binding": "binding", + "snapshots": {} + } +] + +values = { + "global": { + "modules": { + "publicDomainTemplate": "example.com" + }, + "discovery": { + "clusterDomain": "cluster.local" + } + }, + MODULE_NAME: { + "internal": {} + } +} + +class TestCertificate(testing.TestHook): + secret_data = {} + sans_default = SANS + ["localhost", "127.0.0.1"] + + @staticmethod + def parse_certificate(crt: str) -> crypto.X509: + return parse.parse_certificate(utils.base64_decode(crt)) + + def check_data(self): + self.assertGreater(len(self.values[MODULE_NAME]["internal"].get("dvcr", {}).get("cert", {})), 0) + self.secret_data = self.values[MODULE_NAME]["internal"]["dvcr"]["cert"] + self.assertTrue(utils.is_base64(self.secret_data.get("ca", ""))) + self.assertTrue(utils.is_base64(self.secret_data.get("crt", ""))) + self.assertTrue(utils.is_base64(self.secret_data.get("key", ""))) + + def check_sans(self, crt: crypto.X509) -> bool: + sans_from_cert = parse.get_certificate_san(crt) + sans = [] + for san in self.sans_default: + try: + ip_address(san) + sans.append(f"IP Address:{san}") + except ValueError: + sans.append(f"DNS:{san}") + sans_from_cert.sort() + sans.sort() + self.assertEqual(sans_from_cert, sans) + + def verify_certificate(self, ca: crypto.X509, crt: crypto.X509) -> crypto.X509StoreContextError: + store = crypto.X509Store() + store.add_cert(ca) + ctx = crypto.X509StoreContext(store, crt) + try: + ctx.verify_certificate() + return None + except crypto.X509StoreContextError as e: + return e + +class TestGenerateCertificate(TestCertificate): + def setUp(self): + self.func = hook_generate.reconcile() + self.bindind_context = binding_context + self.values = values + def test_generate_certificate(self): + self.hook_run() + self.check_data() + ca = self.parse_certificate(self.secret_data["ca"]) + crt = self.parse_certificate(self.secret_data["crt"]) + if (e := self.verify_certificate(ca, crt)) is not None: + self.fail(f"Certificate is not verify. Raised an exception: {e} ") + self.check_sans(crt) + +class TestReGenerateCertificate(TestCertificate): + def setUp(self): + self.func = hook_regenerate.reconcile() + self.bindind_context = binding_context + self.values = values + self.hook_run() + self.bindind_context[0]["snapshots"] = { + hook_regenerate.SNAPSHOT_SECRETS_NAME : [ + { + "filterResult": { + "data": { + "ca.crt" : self.values[MODULE_NAME]["internal"]["dvcr"]["cert"]["ca"], + "tls.crt": self.values[MODULE_NAME]["internal"]["dvcr"]["cert"]["crt"], + "key.crt": self.values[MODULE_NAME]["internal"]["dvcr"]["cert"]["key"] + }, + "name": NAME + } + } + ] + } + self.func = hook_generate.reconcile() + + def test_regenerate_certificate(self): + self.check_data() + ca = self.parse_certificate(self.secret_data["ca"]) + crt = self.parse_certificate(self.secret_data["crt"]) + if self.verify_certificate(ca, crt) is None: + self.fail(f"certificate has not expired") + self.hook_run() + self.check_data() + ca = self.parse_certificate(self.secret_data["ca"]) + crt = self.parse_certificate(self.secret_data["crt"]) + if (e := self.verify_certificate(ca, crt)) is not None: + self.fail(f"Certificate is not verify. Raised an exception: {e} ") + self.check_sans(crt) diff --git a/hooks/lib/tests/test_manage_tenant_secrets.py b/hooks/lib/tests/test_manage_tenant_secrets.py new file mode 100644 index 0000000..8e77059 --- /dev/null +++ b/hooks/lib/tests/test_manage_tenant_secrets.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lib.tests import testing +from lib.hooks.manage_tenant_secrets import ManageTenantSecretsHook + +hook = ManageTenantSecretsHook(source_namespace="source_namespace", + source_secret_name="secret_name", + pod_labels_to_follow={"app": "test"}, + destination_secret_labels={"test":"test"}, + module_name="test") + +binding_context = [ + { + "binding": "binding", + "snapshots": { + hook.POD_SNAPSHOT_NAME: [ + { + "filterResult": { + "namespace": "pod-namespace1" ## Create secret + } + }, + { + "filterResult": { + "namespace": "pod-namespace2" ## Don't create secret, because ns has deletionTimestamp + } + } + ], + hook.SECRETS_SNAPSHOT_NAME: [ + { + "filterResult": { + "data": {"test": "test"}, + "namespace": "source_namespace", + "type": "Opaque" + } + }, + { + "filterResult": { + "data": {"test": "test"}, + "namespace": "pod-namespace3", ## Delete secret, because namespace pod-namespace3 hasn't pods + "type": "Opaque" + } + }, + ], + hook.NAMESPACE_SNAPSHOT_NAME: [ + { + "filterResult": { + "name": "source_namespace", + "isTerminating": False + } + }, + { + "filterResult": { + "name": "pod-namespace1", + "isTerminating": False + } + }, + { + "filterResult": { + "name": "pod-namespace2", + "isTerminating": True + } + }, + { + "filterResult": { + "name": "pod-namespace3", + "isTerminating": False + } + }, + ] + } + } +] + +class TestManageSecrets(testing.TestHook): + def setUp(self): + self.func = hook.reconcile() + self.bindind_context = binding_context + self.values = {} + def test_manage_secrets(self): + self.hook_run() + self.assertEqual(len(self.kube_resources), 1) + self.assertEqual(self.kube_resources[0]["kind"], "Secret") + self.assertEqual(self.kube_resources[0]["metadata"]["name"], "secret_name") + self.assertEqual(self.kube_resources[0]["metadata"]["namespace"], "pod-namespace1") + self.assertEqual(self.kube_resources[0]["type"], "Opaque") + self.assertEqual(self.kube_resources[0]["data"], {'test': 'test'}) + self.assertEqual(self.kube_resources[0]["metadata"]["labels"], {'test': 'test'}) + diff --git a/hooks/lib/tests/testing.py b/hooks/lib/tests/testing.py new file mode 100644 index 0000000..f0257aa --- /dev/null +++ b/hooks/lib/tests/testing.py @@ -0,0 +1,57 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhouse import hook +import unittest +import jsonpatch +import kubernetes_validate +import jsonschema + +class TestHook(unittest.TestCase): + kube_resources = [] + kube_version = "1.28" + def setUp(self): + self.bindind_context = [] + self.values = {} + self.func = None + + def tearDown(self): + pass + + def hook_run(self, validate_kube_resources: bool = True) -> None: + out = hook.testrun(func=self.func, + binding_context=self.bindind_context, + initial_values=self.values) + for patch in out.values_patches.data: + self.values = jsonpatch.apply_patch(self.values, [patch]) + + deletes = ("Delete", "DeleteInBackground", "DeleteNonCascading") + for kube_operation in out.kube_operations.data: + if kube_operation["operation"] in deletes: + continue + obj = kube_operation["object"] + if validate_kube_resources: + try: + ## TODO Validate CRD + kubernetes_validate.validate(obj, self.kube_version, strict=True) + self.kube_resources.append(obj) + except (kubernetes_validate.SchemaNotFoundError, + kubernetes_validate.InvalidSchemaError, + kubernetes_validate.ValidationError, + jsonschema.RefResolutionError) as e: + self.fail(f"Object is not valid. Raised an exception: {e} ") + else: + self.kube_resources.append(obj) + diff --git a/hooks/lib/utils.py b/hooks/lib/utils.py new file mode 100644 index 0000000..a486efd --- /dev/null +++ b/hooks/lib/utils.py @@ -0,0 +1,54 @@ +# +# Copyright 2023 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import json + +def base64_encode(b: bytes) -> str: + return str(base64.b64encode(b), encoding='utf-8') + +def base64_decode(s: str) -> str: + return str(base64.b64decode(s), encoding="utf-8") + +def base64_encode_from_str(s: str) -> str: + return base64_encode(bytes(s, 'utf-8')) + +def json_load(path: str): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + return data + +def get_dir_path() -> str: + return os.path.dirname(os.path.abspath(__file__)) + +def is_base64(s): + try: + base64_decode(s) + return True + except base64.binascii.Error: + return False + +def check_elem_in_list(l: list, elem) -> bool: + for i in l: + if i == elem: + return True + return False + +def find_index_in_list(l: list, elem) -> int: + for i in range(len(l)): + if l[i] == elem: + return i + return \ No newline at end of file diff --git a/images/controller/Dockerfile b/images/controller/Dockerfile new file mode 100644 index 0000000..f98b4ad --- /dev/null +++ b/images/controller/Dockerfile @@ -0,0 +1,17 @@ +ARG BASE_ALPINE=registry.deckhouse.io/base_images/alpine:3.16.3@sha256:5548e9172c24a1b0ca9afdd2bf534e265c94b12b36b3e0c0302f5853eaf00abb +ARG BASE_GOLANG_21_ALPINE_BUILDER=registry.deckhouse.io/base_images/golang:1.21.4-alpine3.18@sha256:cf84f3d6882c49ea04b6478ac514a2582c8922d7e5848b43d2918fff8329f6e6 + +FROM $BASE_GOLANG_21_ALPINE_BUILDER as builder + +WORKDIR /go/src +ADD go.mod . +ADD go.sum . +RUN go mod download +COPY . . +WORKDIR /go/src/cmd +RUN GOOS=linux GOARCH=amd64 go build -o controller + +FROM --platform=linux/amd64 $BASE_ALPINE +COPY --from=builder /go/src/cmd/controller /go/src/cmd/controller + +ENTRYPOINT ["/go/src/cmd/controller"] diff --git a/images/controller/api/v1alpha1/nfs_storage_class.go b/images/controller/api/v1alpha1/nfs_storage_class.go new file mode 100644 index 0000000..afd2340 --- /dev/null +++ b/images/controller/api/v1alpha1/nfs_storage_class.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type NFSStorageClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec NFSStorageClassSpec `json:"spec"` + Status *NFSStorageClassStatus `json:"status,omitempty"` +} + +// NFSStorageClassList contains a list of empty block device +type NFSStorageClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []NFSStorageClass `json:"items"` +} + +type NFSStorageClassSpec struct { + Connection *NFSStorageClassConnection `json:"connection,omitempty"` + MountOptions *NFSStorageClassMountOptions `json:"mountOptions,omitempty"` + ChmodPermissions string `json:"chmodPermissions,omitempty"` + ReclaimPolicy string `json:"reclaimPolicy"` + VolumeBindingMode string `json:"volumeBindingMode"` +} + +type NFSStorageClassConnection struct { + Host string `json:"host"` + Share string `json:"share"` + NFSVersion string `json:"nfsVersion"` + SubDir string `json:"subDir,omitempty"` +} + +type NFSStorageClassMountOptions struct { + MountMode string `json:"mountMode,omitempty"` + Timeout int `json:"timeout,omitempty"` + Retransmissions int `json:"retransmissions,omitempty"` + ReadOnly *bool `json:"readOnly,omitempty"` +} + +type NFSStorageClassStatus struct { + Phase string `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` +} diff --git a/images/controller/api/v1alpha1/register.go b/images/controller/api/v1alpha1/register.go new file mode 100644 index 0000000..9e37972 --- /dev/null +++ b/images/controller/api/v1alpha1/register.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + NFSStorageClassKind = "NFSStorageClass" + APIGroup = "storage.deckhouse.io" + APIVersion = "v1alpha1" +) + +// SchemeGroupVersion is group version used to register these objects +var ( + SchemeGroupVersion = schema.GroupVersion{ + Group: APIGroup, + Version: APIVersion, + } + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +// Adds the list of known types to Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &NFSStorageClass{}, + &NFSStorageClassList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/images/controller/api/v1alpha1/zz_generated.deepcopy.go b/images/controller/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..d74af89 --- /dev/null +++ b/images/controller/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import "k8s.io/apimachinery/pkg/runtime" + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NFSStorageClass) DeepCopyInto(out *NFSStorageClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyBlockDevice. +func (in *NFSStorageClass) DeepCopy() *NFSStorageClass { + if in == nil { + return nil + } + out := new(NFSStorageClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NFSStorageClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NFSStorageClassList) DeepCopyInto(out *NFSStorageClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NFSStorageClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuestbookList. +func (in *NFSStorageClassList) DeepCopy() *NFSStorageClassList { + if in == nil { + return nil + } + out := new(NFSStorageClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NFSStorageClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/images/controller/cmd/main.go b/images/controller/cmd/main.go new file mode 100644 index 0000000..cbb8ed9 --- /dev/null +++ b/images/controller/cmd/main.go @@ -0,0 +1,131 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "d8-controller/api/v1alpha1" + "d8-controller/pkg/config" + "d8-controller/pkg/controller" + "d8-controller/pkg/kubutils" + "d8-controller/pkg/logger" + "fmt" + "os" + goruntime "runtime" + + "sigs.k8s.io/controller-runtime/pkg/cache" + + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + apiruntime "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var ( + resourcesSchemeFuncs = []func(*apiruntime.Scheme) error{ + v1alpha1.AddToScheme, + clientgoscheme.AddToScheme, + extv1.AddToScheme, + v1.AddToScheme, + sv1.AddToScheme, + } +) + +func main() { + ctx := context.Background() + cfgParams := config.NewConfig() + + log, err := logger.NewLogger(cfgParams.Loglevel) + if err != nil { + fmt.Println(fmt.Sprintf("unable to create NewLogger, err: %v", err)) + os.Exit(1) + } + + log.Info(fmt.Sprintf("[main] Go Version:%s ", goruntime.Version())) + log.Info(fmt.Sprintf("[main] OS/Arch:Go OS/Arch:%s/%s ", goruntime.GOOS, goruntime.GOARCH)) + + log.Info("[main] CfgParams has been successfully created") + log.Info(fmt.Sprintf("[main] %s = %s", config.LogLevelEnvName, cfgParams.Loglevel)) + log.Info(fmt.Sprintf("[main] RequeueStorageClassInterval = %d", cfgParams.RequeueStorageClassInterval)) + + kConfig, err := kubutils.KubernetesDefaultConfigCreate() + if err != nil { + log.Error(err, "[main] unable to KubernetesDefaultConfigCreate") + } + log.Info("[main] kubernetes config has been successfully created.") + + scheme := runtime.NewScheme() + for _, f := range resourcesSchemeFuncs { + err := f(scheme) + if err != nil { + log.Error(err, "[main] unable to add scheme to func") + os.Exit(1) + } + } + log.Info("[main] successfully read scheme CR") + + cacheOpt := cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + cfgParams.ControllerNamespace: {}, + }, + } + + managerOpts := manager.Options{ + Scheme: scheme, + Cache: cacheOpt, + //MetricsBindAddress: cfgParams.MetricsPort, + HealthProbeBindAddress: cfgParams.HealthProbeBindAddress, + LeaderElection: true, + LeaderElectionNamespace: cfgParams.ControllerNamespace, + LeaderElectionID: config.ControllerName, + Logger: log.GetLogger(), + } + + mgr, err := manager.New(kConfig, managerOpts) + if err != nil { + log.Error(err, "[main] unable to manager.New") + os.Exit(1) + } + log.Info("[main] successfully created kubernetes manager") + + if _, err = controller.RunNFSStorageClassWatcherController(mgr, *cfgParams, *log); err != nil { + log.Error(err, fmt.Sprintf("[main] unable to run %s", controller.NFSStorageClassCtrlName)) + os.Exit(1) + } + + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(err, "[main] unable to mgr.AddHealthzCheck") + os.Exit(1) + } + log.Info("[main] successfully AddHealthzCheck") + + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error(err, "[main] unable to mgr.AddReadyzCheck") + os.Exit(1) + } + log.Info("[main] successfully AddReadyzCheck") + + err = mgr.Start(ctx) + if err != nil { + log.Error(err, "[main] unable to mgr.Start") + os.Exit(1) + } +} diff --git a/images/controller/go.mod b/images/controller/go.mod new file mode 100644 index 0000000..8d4a6a9 --- /dev/null +++ b/images/controller/go.mod @@ -0,0 +1,71 @@ +module d8-controller + +go 1.21 + +require ( + github.com/go-logr/logr v1.4.1 + github.com/onsi/ginkgo/v2 v2.14.0 + github.com/onsi/gomega v1.30.0 + k8s.io/api v0.29.2 + k8s.io/apiextensions-apiserver v0.29.2 + k8s.io/apimachinery v0.29.2 + k8s.io/client-go v0.29.2 + k8s.io/klog/v2 v2.120.1 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e + sigs.k8s.io/controller-runtime v0.17.4 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.16.1 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-base v0.29.2 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/images/controller/go.sum b/images/controller/go.sum new file mode 100644 index 0000000..3473c3a --- /dev/null +++ b/images/controller/go.sum @@ -0,0 +1,204 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= +k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= +k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.4 h1:AMf1E0+93/jLQ13fb76S6Atwqp24EQFCmNbG84GJxew= +sigs.k8s.io/controller-runtime v0.17.4/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/images/controller/pkg/config/config.go b/images/controller/pkg/config/config.go new file mode 100644 index 0000000..abd10d0 --- /dev/null +++ b/images/controller/pkg/config/config.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "d8-controller/pkg/logger" + "log" + "os" + "time" +) + +const ( + LogLevelEnvName = "LOG_LEVEL" + ControllerNamespaceEnv = "CONTROLLER_NAMESPACE" + HardcodedControllerNS = "d8-csi-nfs" + ControllerName = "d8-controller" + DefaultHealthProbeBindAddressEnvName = "HEALTH_PROBE_BIND_ADDRESS" + DefaultHealthProbeBindAddress = ":8081" + DefaultRequeueStorageClassInterval = 10 +) + +type Options struct { + Loglevel logger.Verbosity + RequeueStorageClassInterval time.Duration + HealthProbeBindAddress string + ControllerNamespace string +} + +func NewConfig() *Options { + var opts Options + + loglevel := os.Getenv(LogLevelEnvName) + if loglevel == "" { + opts.Loglevel = logger.DebugLevel + } else { + opts.Loglevel = logger.Verbosity(loglevel) + } + + opts.HealthProbeBindAddress = os.Getenv(DefaultHealthProbeBindAddressEnvName) + if opts.HealthProbeBindAddress == "" { + opts.HealthProbeBindAddress = DefaultHealthProbeBindAddress + } + + opts.ControllerNamespace = os.Getenv(ControllerNamespaceEnv) + if opts.ControllerNamespace == "" { + + namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + log.Printf("Failed to get namespace from filesystem: %v", err) + log.Printf("Using hardcoded namespace: %s", HardcodedControllerNS) + opts.ControllerNamespace = HardcodedControllerNS + } else { + log.Printf("Got namespace from filesystem: %s", string(namespace)) + opts.ControllerNamespace = string(namespace) + } + } + + opts.RequeueStorageClassInterval = DefaultRequeueStorageClassInterval + + return &opts +} diff --git a/images/controller/pkg/controller/controller_suite_test.go b/images/controller/pkg/controller/controller_suite_test.go new file mode 100644 index 0000000..81e10e2 --- /dev/null +++ b/images/controller/pkg/controller/controller_suite_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller_test + +import ( + v1alpha1 "d8-controller/api/v1alpha1" + "fmt" + "os" + "testing" + + v1 "k8s.io/api/apps/v1" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + sv1 "k8s.io/api/storage/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + + apiruntime "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controller Suite") +} + +func NewFakeClient() client.Client { + resourcesSchemeFuncs := []func(*apiruntime.Scheme) error{ + v1alpha1.AddToScheme, + clientgoscheme.AddToScheme, + extv1.AddToScheme, + v1.AddToScheme, + sv1.AddToScheme, + } + scheme := apiruntime.NewScheme() + for _, f := range resourcesSchemeFuncs { + err := f(scheme) + if err != nil { + println(fmt.Sprintf("Error adding scheme: %s", err)) + os.Exit(1) + } + } + + // See https://github.com/kubernetes-sigs/controller-runtime/issues/2362#issuecomment-1837270195 + builder := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&v1alpha1.NFSStorageClass{}) + + cl := builder.Build() + return cl +} diff --git a/images/controller/pkg/controller/nfs_storage_class_watcher.go b/images/controller/pkg/controller/nfs_storage_class_watcher.go new file mode 100644 index 0000000..b5d46e7 --- /dev/null +++ b/images/controller/pkg/controller/nfs_storage_class_watcher.go @@ -0,0 +1,253 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + v1alpha1 "d8-controller/api/v1alpha1" + "d8-controller/pkg/config" + "d8-controller/pkg/logger" + "errors" + "fmt" + "reflect" + "time" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + NFSStorageClassCtrlName = "nfs-storage-class-controller" + + StorageClassKind = "StorageClass" + StorageClassAPIVersion = "storage.k8s.io/v1" + + NFSStorageClassProvisioner = "nfs.csi.k8s.io" + + NFSStorageClassFinalizerName = "storage.deckhouse.io/nfs-storage-class-controller" + NFSStorageClassManagedLabelKey = "storage.deckhouse.io/managed-by" + NFSStorageClassManagedLabelValue = "nfs-storage-class-controller" + + AllowVolumeExpansionDefaultValue = true + + FailedStatusPhase = "Failed" + CreatedStatusPhase = "Created" + + CreateReconcile = "Create" + UpdateReconcile = "Update" + DeleteReconcile = "Delete" + + serverParamKey = "server" + shareParamKey = "share" + MountPermissionsParamKey = "mountPermissions" + SubDirParamKey = "subdir" + MountOptionsSecretKey = "mountOptions" + + SecretForMountOptionsPrefix = "nfs-mount-options-for-" + StorageClassSecretNameKey = "csi.storage.k8s.io/provisioner-secret-name" + StorageClassSecretNSKey = "csi.storage.k8s.io/provisioner-secret-namespace" +) + +func RunNFSStorageClassWatcherController( + mgr manager.Manager, + cfg config.Options, + log logger.Logger, +) (controller.Controller, error) { + cl := mgr.GetClient() + + c, err := controller.New(NFSStorageClassCtrlName, mgr, controller.Options{ + Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log.Info(fmt.Sprintf("[NFSStorageClassReconciler] starts Reconcile for the NFSStorageClass %q", request.Name)) + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, request.NamespacedName, nsc) + if err != nil && !k8serr.IsNotFound(err) { + log.Error(err, fmt.Sprintf("[NFSStorageClassReconciler] unable to get NFSStorageClass, name: %s", request.Name)) + return reconcile.Result{}, err + } + + if nsc.Name == "" { + log.Info(fmt.Sprintf("[NFSStorageClassReconciler] seems like the NFSStorageClass for the request %s was deleted. Reconcile retrying will stop.", request.Name)) + return reconcile.Result{}, nil + } + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + if err != nil { + log.Error(err, "[NFSStorageClassReconciler] unable to list Storage Classes") + return reconcile.Result{}, err + } + + shouldRequeue, err := RunEventReconcile(ctx, cl, log, scList, nsc, cfg.ControllerNamespace) + if err != nil { + log.Error(err, fmt.Sprintf("[NFSStorageClassReconciler] an error occured while reconciles the NFSStorageClass, name: %s", nsc.Name)) + } + + if shouldRequeue { + log.Warning(fmt.Sprintf("[NFSStorageClassReconciler] Reconciler will requeue the request, name: %s", request.Name)) + return reconcile.Result{ + RequeueAfter: cfg.RequeueStorageClassInterval * time.Second, + }, nil + } + + log.Info(fmt.Sprintf("[NFSStorageClassReconciler] ends Reconcile for the NFSStorageClass %q", request.Name)) + return reconcile.Result{}, nil + }), + }) + if err != nil { + log.Error(err, "[RunNFSStorageClassWatcherController] unable to create controller") + return nil, err + } + + err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.NFSStorageClass{}), handler.Funcs{ + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[CreateFunc] get event for NFSStorageClass %q. Add to the queue", e.Object.GetName())) + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: e.Object.GetNamespace(), Name: e.Object.GetName()}} + q.Add(request) + }, + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[UpdateFunc] get event for NFSStorageClass %q. Check if it should be reconciled", e.ObjectNew.GetName())) + + oldLsc, ok := e.ObjectOld.(*v1alpha1.NFSStorageClass) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[UpdateFunc] an error occurred while handling create event") + return + } + newLsc, ok := e.ObjectNew.(*v1alpha1.NFSStorageClass) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[UpdateFunc] an error occurred while handling create event") + return + } + + if reflect.DeepEqual(oldLsc.Spec, newLsc.Spec) && newLsc.DeletionTimestamp == nil { + log.Info(fmt.Sprintf("[UpdateFunc] an update event for the NFSStorageClass %s has no Spec field updates. It will not be reconciled", newLsc.Name)) + return + } + + log.Info(fmt.Sprintf("[UpdateFunc] the NFSStorageClass %q will be reconciled. Add to the queue", newLsc.Name)) + request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: newLsc.Namespace, Name: newLsc.Name}} + q.Add(request) + }, + }) + if err != nil { + log.Error(err, "[RunNFSStorageClassWatcherController] unable to watch the events") + return nil, err + } + + return c, nil +} + +func RunEventReconcile(ctx context.Context, cl client.Client, log logger.Logger, scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (shouldRequeue bool, err error) { + added, err := addFinalizerIfNotExistsForNSC(ctx, cl, nsc) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to add a finalizer %s to the NFSStorageClass %s: %w", NFSStorageClassFinalizerName, nsc.Name, err) + return true, err + } + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] finalizer %s was added to the NFSStorageClass %s: %t", NFSStorageClassFinalizerName, nsc.Name, added)) + + reconcileTypeForStorageClass, err := IdentifyReconcileFuncForStorageClass(log, scList, nsc, controllerNamespace) + if err != nil { + err = fmt.Errorf("[runEventReconcile] error occured while identifying the reconcile function for StorageClass %s: %w", nsc.Name, err) + return true, err + } + + shouldRequeue = false + log.Debug(fmt.Sprintf("[runEventReconcile] reconcile operation for StorageClass %q: %q", nsc.Name, reconcileTypeForStorageClass)) + switch reconcileTypeForStorageClass { + case CreateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] CreateReconcile starts reconciliataion of StorageClass, name: %s", nsc.Name)) + shouldRequeue, err = ReconcileStorageClassCreateFunc(ctx, cl, log, scList, nsc, controllerNamespace) + case UpdateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] UpdateReconcile starts reconciliataion of StorageClass, name: %s", nsc.Name)) + shouldRequeue, err = reconcileStorageClassUpdateFunc(ctx, cl, log, scList, nsc, controllerNamespace) + case DeleteReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] DeleteReconcile starts reconciliataion of StorageClass, name: %s", nsc.Name)) + shouldRequeue, err = reconcileStorageClassDeleteFunc(ctx, cl, log, scList, nsc) + default: + log.Debug(fmt.Sprintf("[runEventReconcile] StorageClass for NFSStorageClass %s should not be reconciled", nsc.Name)) + } + log.Debug(fmt.Sprintf("[runEventReconcile] ends reconciliataion of StorageClass, name: %s, shouldRequeue: %t, err: %v", nsc.Name, shouldRequeue, err)) + + if err != nil || shouldRequeue { + return shouldRequeue, err + } + + secretList := &corev1.SecretList{} + err = cl.List(ctx, secretList, client.InNamespace(controllerNamespace)) + if err != nil { + err = fmt.Errorf("[runEventReconcile] unable to list Secrets: %w", err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + reconcileTypeForSecret, err := IdentifyReconcileFuncForSecret(log, secretList, nsc, controllerNamespace) + if err != nil { + log.Error(err, fmt.Sprintf("[runEventReconcile] error occured while identifying the reconcile function for the Secret %q", SecretForMountOptionsPrefix+nsc.Name)) + return true, err + } + + log.Debug(fmt.Sprintf("[runEventReconcile] reconcile operation for Secret %q: %q", SecretForMountOptionsPrefix+nsc.Name, reconcileTypeForSecret)) + switch reconcileTypeForSecret { + case CreateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] CreateReconcile starts reconciliataion of Secret, name: %s", SecretForMountOptionsPrefix+nsc.Name)) + shouldRequeue, err = ReconcileSecretCreateFunc(ctx, cl, log, nsc, controllerNamespace) + case UpdateReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] UpdateReconcile starts reconciliataion of Secret, name: %s", SecretForMountOptionsPrefix+nsc.Name)) + shouldRequeue, err = reconcileSecretUpdateFunc(ctx, cl, log, secretList, nsc, controllerNamespace) + case DeleteReconcile: + log.Debug(fmt.Sprintf("[runEventReconcile] DeleteReconcile starts reconciliataion of Secret, name: %s", SecretForMountOptionsPrefix+nsc.Name)) + shouldRequeue, err = reconcileSecretDeleteFunc(ctx, cl, log, secretList, nsc) + default: + log.Debug(fmt.Sprintf("[runEventReconcile] Secret %q should not be reconciled", SecretForMountOptionsPrefix+nsc.Name)) + } + + log.Debug(fmt.Sprintf("[runEventReconcile] ends reconciliataion of Secret, name: %s, shouldRequeue: %t, err: %v", SecretForMountOptionsPrefix+nsc.Name, shouldRequeue, err)) + + if err != nil || shouldRequeue { + return shouldRequeue, err + } + + log.Debug(fmt.Sprintf("[runEventReconcile] Finish all reconciliations for NFSStorageClass %q.", nsc.Name)) + + if reconcileTypeForSecret != DeleteReconcile { + err = updateNFSStorageClassPhase(ctx, cl, nsc, CreatedStatusPhase, "") + if err != nil { + err = fmt.Errorf("[runEventReconcile] unable to update the NFSStorageClass %s: %w", nsc.Name, err) + return true, err + } + log.Debug(fmt.Sprintf("[runEventReconcile] successfully updated the NFSStorageClass %s status", nsc.Name)) + } + + return false, nil + +} diff --git a/images/controller/pkg/controller/nfs_storage_class_watcher_func.go b/images/controller/pkg/controller/nfs_storage_class_watcher_func.go new file mode 100644 index 0000000..43af91c --- /dev/null +++ b/images/controller/pkg/controller/nfs_storage_class_watcher_func.go @@ -0,0 +1,720 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + v1alpha1 "d8-controller/api/v1alpha1" + "d8-controller/pkg/logger" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/strings/slices" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ReconcileStorageClassCreateFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + scList *v1.StorageClassList, + nsc *v1alpha1.NFSStorageClass, + controllerNamespace string, +) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] starts for NFSStorageClass %q", nsc.Name)) + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] starts storage class configuration for the NFSStorageClass, name: %s", nsc.Name)) + newSC, err := ConfigureStorageClass(nsc, controllerNamespace) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to configure a Storage Class for the NFSStorageClass %s: %w", nsc.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return false, err + } + + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] successfully configurated storage class for the NFSStorageClass, name: %s", nsc.Name)) + log.Trace(fmt.Sprintf("[reconcileStorageClassCreateFunc] storage class: %+v", newSC)) + + created, err := createStorageClassIfNotExists(ctx, cl, scList, newSC) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to create a Storage Class %s: %w", newSC.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + log.Debug(fmt.Sprintf("[reconcileStorageClassCreateFunc] a storage class %s was created: %t", newSC.Name, created)) + if created { + log.Info(fmt.Sprintf("[reconcileStorageClassCreateFunc] successfully create storage class, name: %s", newSC.Name)) + } else { + log.Warning(fmt.Sprintf("[reconcileLSCCreateFunc] Storage class %s already exists. Adding event to requeue.", newSC.Name)) + return true, nil + } + + return false, nil +} + +func reconcileStorageClassUpdateFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + scList *v1.StorageClassList, + nsc *v1alpha1.NFSStorageClass, + controllerNamespace string, +) (bool, error) { + + log.Debug(fmt.Sprintf("[reconcileStorageClassUpdateFunc] starts for NFSStorageClass %q", nsc.Name)) + + var oldSC *v1.StorageClass + for _, s := range scList.Items { + if s.Name == nsc.Name { + oldSC = &s + break + } + } + + if oldSC == nil { + err := fmt.Errorf("a storage class %s does not exist", nsc.Name) + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to find a storage class for the NFSStorageClass %s: %w", nsc.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Debug(fmt.Sprintf("[reconcileStorageClassUpdateFunc] successfully found a storage class for the NFSStorageClass, name: %s", nsc.Name)) + + log.Trace(fmt.Sprintf("[reconcileStorageClassUpdateFunc] storage class: %+v", oldSC)) + newSC, err := ConfigureStorageClass(nsc, controllerNamespace) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to configure a Storage Class for the NFSStorageClass %s: %w", nsc.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return false, err + } + + diff, err := GetSCDiff(oldSC, newSC) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] error occured while identifying the difference between the existed StorageClass %s and the new one: %w", newSC.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + if diff != "" { + log.Info(fmt.Sprintf("[reconcileStorageClassUpdateFunc] current Storage Class LVMVolumeGroups do not match NFSStorageClass ones. The Storage Class %s will be recreated with new ones", nsc.Name)) + + err = recreateStorageClass(ctx, cl, oldSC, newSC) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to recreate a Storage Class %s: %w", newSC.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileStorageClassUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Info(fmt.Sprintf("[reconcileStorageClassUpdateFunc] a Storage Class %s was successfully recreated", newSC.Name)) + } + + return false, nil +} + +func reconcileStorageClassDeleteFunc( + ctx context.Context, + cl client.Client, + log logger.Logger, + scList *v1.StorageClassList, + nsc *v1alpha1.NFSStorageClass, +) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileStorageClassDeleteFunc] tries to find a storage class for the NFSStorageClass %s", nsc.Name)) + var sc *v1.StorageClass + for _, s := range scList.Items { + if s.Name == nsc.Name { + sc = &s + break + } + } + if sc == nil { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] no storage class found for the NFSStorageClass, name: %s", nsc.Name)) + } + + if sc != nil { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] successfully found a storage class for the NFSStorageClass %s", nsc.Name)) + log.Debug(fmt.Sprintf("[reconcileStorageClassDeleteFunc] starts identifing a provisioner for the storage class %s", sc.Name)) + + if sc.Provisioner != NFSStorageClassProvisioner { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] the storage class %s does not belongs to %s provisioner. It will not be deleted", sc.Name, NFSStorageClassProvisioner)) + } else { + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] the storage class %s belongs to %s provisioner. It will be deleted", sc.Name, NFSStorageClassProvisioner)) + + err := deleteStorageClass(ctx, cl, sc) + if err != nil { + err = fmt.Errorf("[reconcileStorageClassDeleteFunc] unable to delete a storage class %s: %w", sc.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to delete a storage class, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileStorageClassDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + log.Info(fmt.Sprintf("[reconcileStorageClassDeleteFunc] successfully deleted a storage class, name: %s", sc.Name)) + } + } + + log.Debug("[reconcileStorageClassDeleteFunc] ends the reconciliation") + return false, nil +} + +func ReconcileSecretCreateFunc(ctx context.Context, cl client.Client, log logger.Logger, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileSecretCreateFunc] starts for NFSStorageClass %q", nsc.Name)) + + newSecret := configureSecret(nsc, controllerNamespace) + log.Debug(fmt.Sprintf("[reconcileSecretCreateFunc] successfully configurated secret for the NFSStorageClass, name: %s", nsc.Name)) + log.Trace(fmt.Sprintf("[reconcileSecretCreateFunc] secret: %+v", newSecret)) + + err := cl.Create(ctx, newSecret) + if err != nil { + err = fmt.Errorf("[reconcileSecretCreateFunc] unable to create a Secret %s: %w", newSecret.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileSecretCreateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + return false, nil +} + +func reconcileSecretUpdateFunc(ctx context.Context, cl client.Client, log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileSecretUpdateFunc] starts for secret %q", SecretForMountOptionsPrefix+nsc.Name)) + + var oldSecret *corev1.Secret + for _, s := range secretList.Items { + if s.Name == SecretForMountOptionsPrefix+nsc.Name { + oldSecret = &s + break + } + } + + if oldSecret == nil { + err := fmt.Errorf("[reconcileSecretUpdateFunc] unable to find a secret %s for the NFSStorageClass, name: %s", SecretForMountOptionsPrefix+nsc.Name, nsc.Name) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileSecretUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Debug(fmt.Sprintf("[reconcileSecretUpdateFunc] successfully found a secret %q for the NFSStorageClass, name: %q", oldSecret.Name, nsc.Name)) + + newSecret := configureSecret(nsc, controllerNamespace) + + log.Trace(fmt.Sprintf("[reconcileSecretUpdateFunc] old secret: %+v", oldSecret)) + log.Trace(fmt.Sprintf("[reconcileSecretUpdateFunc] new secret: %+v", newSecret)) + + err := cl.Update(ctx, newSecret) + if err != nil { + err = fmt.Errorf("[reconcileSecretUpdateFunc] unable to update a Secret %s: %w", newSecret.Name, err) + upError := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, err.Error()) + if upError != nil { + upError = fmt.Errorf("[reconcileSecretUpdateFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upError) + err = errors.Join(err, upError) + } + return true, err + } + + log.Info(fmt.Sprintf("[reconcileSecretUpdateFunc] ends the reconciliation for Secret %q", newSecret.Name)) + + return false, nil +} + +func reconcileSecretDeleteFunc(ctx context.Context, cl client.Client, log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass) (bool, error) { + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] tries to find a secret for the NFSStorageClass %q with name %q", nsc.Name, SecretForMountOptionsPrefix+nsc.Name)) + var secret *corev1.Secret + for _, s := range secretList.Items { + if s.Name == SecretForMountOptionsPrefix+nsc.Name { + secret = &s + break + } + } + if secret == nil { + log.Info(fmt.Sprintf("[reconcileSecretDeleteFunc] no secret found for the NFSStorageClass, name: %s", nsc.Name)) + } + + if secret != nil { + log.Info(fmt.Sprintf("[reconcileSecretDeleteFunc] successfully found a secret for the NFSStorageClass %s", nsc.Name)) + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] starts removing a finalizer %s from the Secret, name: %s", NFSStorageClassFinalizerName, secret.Name)) + _, err := removeFinalizerIfExists(ctx, cl, secret, NFSStorageClassFinalizerName) + if err != nil { + err = fmt.Errorf("[reconcileSecretDeleteFunc] unable to remove a finalizer %s from the Secret %s: %w", NFSStorageClassFinalizerName, secret.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to remove a finalizer, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileSecretDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + + err = cl.Delete(ctx, secret) + if err != nil { + err = fmt.Errorf("[reconcileSecretDeleteFunc] unable to delete a secret %s: %w", secret.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to delete a secret, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileSecretDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + } + + log.Info(fmt.Sprintf("[reconcileSecretDeleteFunc] ends the reconciliation for Secret %q", SecretForMountOptionsPrefix+nsc.Name)) + + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] starts removing a finalizer %s from the NFSStorageClass, name: %s", NFSStorageClassFinalizerName, nsc.Name)) + removed, err := removeFinalizerIfExists(ctx, cl, nsc, NFSStorageClassFinalizerName) + if err != nil { + err = fmt.Errorf("[reconcileSecretDeleteFunc] unable to remove a finalizer %s from the NFSStorageClass %s: %w", NFSStorageClassFinalizerName, nsc.Name, err) + upErr := updateNFSStorageClassPhase(ctx, cl, nsc, FailedStatusPhase, fmt.Sprintf("Unable to remove a finalizer, err: %s", err.Error())) + if upErr != nil { + upErr = fmt.Errorf("[reconcileSecretDeleteFunc] unable to update the NFSStorageClass %s: %w", nsc.Name, upErr) + err = errors.Join(err, upErr) + } + return true, err + } + log.Debug(fmt.Sprintf("[reconcileSecretDeleteFunc] the NFSStorageClass %s finalizer %s was removed: %t", nsc.Name, NFSStorageClassFinalizerName, removed)) + + return false, nil +} + +func IdentifyReconcileFuncForStorageClass(log logger.Logger, scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (reconcileType string, err error) { + if shouldReconcileByDeleteFunc(nsc) { + return DeleteReconcile, nil + } + + if shouldReconcileStorageClassByCreateFunc(scList, nsc) { + return CreateReconcile, nil + } + + should, err := shouldReconcileStorageClassByUpdateFunc(log, scList, nsc, controllerNamespace) + if err != nil { + return "", err + } + if should { + return UpdateReconcile, nil + } + + return "", nil +} + +func shouldReconcileStorageClassByCreateFunc(scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass) bool { + if nsc.DeletionTimestamp != nil { + return false + } + + for _, sc := range scList.Items { + if sc.Name == nsc.Name { + return false + } + } + + return true +} + +func shouldReconcileStorageClassByUpdateFunc(log logger.Logger, scList *v1.StorageClassList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + if nsc.DeletionTimestamp != nil { + return false, nil + } + + for _, oldSC := range scList.Items { + if oldSC.Name == nsc.Name { + if oldSC.Provisioner == NFSStorageClassProvisioner { + newSC, err := ConfigureStorageClass(nsc, controllerNamespace) + if err != nil { + return false, err + } + + diff, err := GetSCDiff(&oldSC, newSC) + if err != nil { + return false, err + } + + if diff != "" { + log.Debug(fmt.Sprintf("[shouldReconcileStorageClassByUpdateFunc] a storage class %s should be updated. Diff: %s", oldSC.Name, diff)) + return true, nil + } + + if nsc.Status != nil && nsc.Status.Phase == FailedStatusPhase { + return true, nil + } + + return false, nil + + } else { + err := fmt.Errorf("a storage class %s does not belong to %s provisioner", oldSC.Name, NFSStorageClassProvisioner) + return false, err + } + } + } + + err := fmt.Errorf("a storage class %s does not exist", nsc.Name) + return false, err +} + +func shouldReconcileByDeleteFunc(nsc *v1alpha1.NFSStorageClass) bool { + if nsc.DeletionTimestamp != nil { + return true + } + + return false +} + +func removeFinalizerIfExists(ctx context.Context, cl client.Client, obj metav1.Object, finalizerName string) (bool, error) { + removed := false + finalizers := obj.GetFinalizers() + for i, f := range finalizers { + if f == finalizerName { + finalizers = append(finalizers[:i], finalizers[i+1:]...) + removed = true + break + } + } + + if removed { + obj.SetFinalizers(finalizers) + err := cl.Update(ctx, obj.(client.Object)) + if err != nil { + return false, err + } + } + + return removed, nil +} + +func GetSCDiff(oldSC, newSC *v1.StorageClass) (string, error) { + + if oldSC.Provisioner != newSC.Provisioner { + err := fmt.Errorf("NFSStorageClass %q: the provisioner field is different in the StorageClass %q", newSC.Name, oldSC.Name) + return "", err + } + + if *oldSC.ReclaimPolicy != *newSC.ReclaimPolicy { + diff := fmt.Sprintf("ReclaimPolicy: %q -> %q", *oldSC.ReclaimPolicy, *newSC.ReclaimPolicy) + return diff, nil + } + + if *oldSC.VolumeBindingMode != *newSC.VolumeBindingMode { + diff := fmt.Sprintf("VolumeBindingMode: %q -> %q", *oldSC.VolumeBindingMode, *newSC.VolumeBindingMode) + return diff, nil + } + + if *oldSC.AllowVolumeExpansion != *newSC.AllowVolumeExpansion { + diff := fmt.Sprintf("AllowVolumeExpansion: %t -> %t", *oldSC.AllowVolumeExpansion, *newSC.AllowVolumeExpansion) + return diff, nil + } + + if !reflect.DeepEqual(oldSC.Parameters, newSC.Parameters) { + diff := fmt.Sprintf("Parameters: %+v -> %+v", oldSC.Parameters, newSC.Parameters) + return diff, nil + } + + if !reflect.DeepEqual(oldSC.MountOptions, newSC.MountOptions) { + diff := fmt.Sprintf("MountOptions: %v -> %v", oldSC.MountOptions, newSC.MountOptions) + return diff, nil + } + + return "", nil +} + +func createStorageClassIfNotExists(ctx context.Context, cl client.Client, scList *v1.StorageClassList, sc *v1.StorageClass) (bool, error) { + for _, s := range scList.Items { + if s.Name == sc.Name { + return false, nil + } + } + + err := cl.Create(ctx, sc) + if err != nil { + return false, err + } + + return true, err +} + +func addFinalizerIfNotExistsForNSC(ctx context.Context, cl client.Client, nsc *v1alpha1.NFSStorageClass) (bool, error) { + if !slices.Contains(nsc.Finalizers, NFSStorageClassFinalizerName) { + nsc.Finalizers = append(nsc.Finalizers, NFSStorageClassFinalizerName) + } + + err := cl.Update(ctx, nsc) + if err != nil { + return false, err + } + + return true, nil +} + +func ConfigureStorageClass(nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (*v1.StorageClass, error) { + if nsc.Spec.ReclaimPolicy == "" { + err := fmt.Errorf("NFSStorageClass %q: the ReclaimPolicy field is empty", nsc.Name) + return nil, err + } + if nsc.Spec.VolumeBindingMode == "" { + err := fmt.Errorf("NFSStorageClass %q: the VolumeBindingMode field is empty", nsc.Name) + return nil, err + } + + reclaimPolicy := corev1.PersistentVolumeReclaimPolicy(nsc.Spec.ReclaimPolicy) + volumeBindingMode := v1.VolumeBindingMode(nsc.Spec.VolumeBindingMode) + AllowVolumeExpansion := AllowVolumeExpansionDefaultValue + + sc := &v1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: StorageClassKind, + APIVersion: StorageClassAPIVersion, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nsc.Name, + Namespace: nsc.Namespace, + Finalizers: []string{NFSStorageClassFinalizerName}, + }, + Parameters: GetSCParams(nsc, controllerNamespace), + MountOptions: GetSCMountOptions(nsc), + Provisioner: NFSStorageClassProvisioner, + ReclaimPolicy: &reclaimPolicy, + VolumeBindingMode: &volumeBindingMode, + AllowVolumeExpansion: &AllowVolumeExpansion, + } + + return sc, nil +} + +func updateNFSStorageClassPhase(ctx context.Context, cl client.Client, nsc *v1alpha1.NFSStorageClass, phase, reason string) error { + if nsc.Status == nil { + nsc.Status = &v1alpha1.NFSStorageClassStatus{} + } + nsc.Status.Phase = phase + nsc.Status.Reason = reason + + // TODO: add retry logic + err := cl.Status().Update(ctx, nsc) + if err != nil { + return err + } + + return nil +} + +func recreateStorageClass(ctx context.Context, cl client.Client, oldSC, newSC *v1.StorageClass) error { + // It is necessary to pass the original StorageClass to the delete operation because + // the deletion will not succeed if the fields in the StorageClass provided to delete + // differ from those currently in the cluster. + err := deleteStorageClass(ctx, cl, oldSC) + if err != nil { + err = fmt.Errorf("[recreateStorageClass] unable to delete a storage class %s: %s", oldSC.Name, err.Error()) + return err + } + + err = cl.Create(ctx, newSC) + if err != nil { + err = fmt.Errorf("[recreateStorageClass] unable to create a storage class %s: %s", newSC.Name, err.Error()) + return err + } + + return nil +} + +func deleteStorageClass(ctx context.Context, cl client.Client, sc *v1.StorageClass) error { + if sc.Provisioner != NFSStorageClassProvisioner { + return fmt.Errorf("a storage class %s does not belong to %s provisioner", sc.Name, NFSStorageClassProvisioner) + } + + _, err := removeFinalizerIfExists(ctx, cl, sc, NFSStorageClassFinalizerName) + if err != nil { + return err + } + + err = cl.Delete(ctx, sc) + if err != nil { + return err + } + + return nil +} + +func GetSCMountOptions(nsc *v1alpha1.NFSStorageClass) []string { + mountOptions := []string{} + + if nsc.Spec.Connection.NFSVersion != "" { + mountOptions = append(mountOptions, "nfsvers="+nsc.Spec.Connection.NFSVersion) + } + + if nsc.Spec.MountOptions != nil { + + if nsc.Spec.MountOptions.MountMode != "" { + mountOptions = append(mountOptions, nsc.Spec.MountOptions.MountMode) + } + + if nsc.Spec.MountOptions.Timeout > 0 { + mountOptions = append(mountOptions, "timeo="+strconv.Itoa(nsc.Spec.MountOptions.Timeout)) + } + + if nsc.Spec.MountOptions.Retransmissions > 0 { + mountOptions = append(mountOptions, "retrans="+strconv.Itoa(nsc.Spec.MountOptions.Retransmissions)) + } + + if nsc.Spec.MountOptions.ReadOnly != nil { + if *nsc.Spec.MountOptions.ReadOnly { + mountOptions = append(mountOptions, "ro") + } else { + mountOptions = append(mountOptions, "rw") + } + } + } + + return mountOptions +} + +func GetSCParams(nsc *v1alpha1.NFSStorageClass, controllerNamespace string) map[string]string { + params := make(map[string]string) + + params[serverParamKey] = nsc.Spec.Connection.Host + params[shareParamKey] = nsc.Spec.Connection.Share + params[StorageClassSecretNameKey] = SecretForMountOptionsPrefix + nsc.Name + params[StorageClassSecretNSKey] = controllerNamespace + + if nsc.Spec.ChmodPermissions != "" { + params[MountPermissionsParamKey] = nsc.Spec.ChmodPermissions + } + + if nsc.Spec.Connection.SubDir != "" { + params[SubDirParamKey] = nsc.Spec.Connection.SubDir + } + + return params +} + +func IdentifyReconcileFuncForSecret(log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (reconcileType string, err error) { + if shouldReconcileByDeleteFunc(nsc) { + return DeleteReconcile, nil + } + + if shouldReconcileSecretByCreateFunc(secretList, nsc) { + return CreateReconcile, nil + } + + should, err := shouldReconcileSecretByUpdateFunc(log, secretList, nsc, controllerNamespace) + if err != nil { + return "", err + } + if should { + return UpdateReconcile, nil + } + + return "", nil +} + +func shouldReconcileSecretByCreateFunc(secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass) bool { + if nsc.DeletionTimestamp != nil { + return false + } + + for _, s := range secretList.Items { + if s.Name == SecretForMountOptionsPrefix+nsc.Name { + return false + } + } + + return true +} + +func shouldReconcileSecretByUpdateFunc(log logger.Logger, secretList *corev1.SecretList, nsc *v1alpha1.NFSStorageClass, controllerNamespace string) (bool, error) { + if nsc.DeletionTimestamp != nil { + return false, nil + } + + secretSelector := labels.Set(map[string]string{ + NFSStorageClassManagedLabelKey: NFSStorageClassManagedLabelValue, + }) + + for _, oldSecret := range secretList.Items { + if oldSecret.Name == SecretForMountOptionsPrefix+nsc.Name { + newSecret := configureSecret(nsc, controllerNamespace) + if !reflect.DeepEqual(oldSecret.StringData, newSecret.StringData) { + log.Debug(fmt.Sprintf("[shouldReconcileSecretByUpdateFunc] a secret %s should be updated", oldSecret.Name)) + if !labels.Set(oldSecret.Labels).AsSelector().Matches(secretSelector) { + err := fmt.Errorf("a secret %q does not have a label %s=%s", oldSecret.Name, NFSStorageClassManagedLabelKey, NFSStorageClassManagedLabelValue) + return false, err + } + return true, nil + } + + if !labels.Set(oldSecret.Labels).AsSelector().Matches(secretSelector) { + log.Debug(fmt.Sprintf("[shouldReconcileSecretByUpdateFunc] a secret %s should be updated. The label %s=%s is missing", oldSecret.Name, NFSStorageClassManagedLabelKey, NFSStorageClassManagedLabelValue)) + return true, nil + } + + return false, nil + } + } + + return true, nil +} + +func configureSecret(nsc *v1alpha1.NFSStorageClass, controllerNamespace string) *corev1.Secret { + mountOptions := GetSCMountOptions(nsc) + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: SecretForMountOptionsPrefix + nsc.Name, + Namespace: controllerNamespace, + Labels: map[string]string{ + NFSStorageClassManagedLabelKey: NFSStorageClassManagedLabelValue, + }, + Finalizers: []string{NFSStorageClassFinalizerName}, + }, + StringData: map[string]string{ + MountOptionsSecretKey: strings.Join(mountOptions, ","), + }, + } + + return secret +} diff --git a/images/controller/pkg/controller/nfs_storage_class_watcher_test.go b/images/controller/pkg/controller/nfs_storage_class_watcher_test.go new file mode 100644 index 0000000..7e2d25d --- /dev/null +++ b/images/controller/pkg/controller/nfs_storage_class_watcher_test.go @@ -0,0 +1,452 @@ +/* +Copyright 2023 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller_test + +import ( + "context" + v1alpha1 "d8-controller/api/v1alpha1" + "d8-controller/pkg/controller" + "d8-controller/pkg/logger" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe(controller.NFSStorageClassCtrlName, func() { + const ( + controllerNamespace = "test-namespace" + nameForTestResource = "example" + ) + var ( + ctx = context.Background() + cl = NewFakeClient() + log = logger.Logger{} + + server = "192.168.1.100" + share = "/data" + subDir = "${pvc.metadata.namespace}/${pvc.metadata.name}" + nfsVer = "4.1" + mountOptForNFSVer = fmt.Sprintf("nfsvers=%s", nfsVer) + mountMode = "hard" + mountModeUpdated = "soft" + timeout = 10 + mountOptForTimeout = "timeo=10" + retransmissions = 3 + mountOptForRetransmissions = "retrans=3" + readOnlyFalse = false + mountOptForReadOnlyFalse = "rw" + readOnlyTrue = true + mountOptForReadOnlyTrue = "ro" + chmodPermissions = "0777" + ) + + It("Create_nfs_sc_with_all_options", func() { + nfsSCtemplate := generateNFSStorageClass(NFSStorageClassConfig{ + Name: nameForTestResource, + Host: server, + Share: share, + SubDir: subDir, + NFSVersion: nfsVer, + MountMode: mountMode, + Timeout: timeout, + Retransmissions: retransmissions, + ReadOnly: &readOnlyFalse, + ChmodPermissions: chmodPermissions, + ReclaimPolicy: string(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: string(v1.VolumeBindingWaitForFirstConsumer), + }) + + err := cl.Create(ctx, nfsSCtemplate) + Expect(err).NotTo(HaveOccurred()) + + nsc := &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(0)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(5)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer, mountMode, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyFalse))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, fmt.Sprintf("%s,%s,%s,%s,%s", mountOptForNFSVer, mountMode, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyFalse))) + + }) + + It("Update_nfs_sc_1", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc.Spec.MountOptions.MountMode = mountModeUpdated + nsc.Spec.MountOptions.ReadOnly = &readOnlyTrue + + err = cl.Update(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(5)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer, mountModeUpdated, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyTrue))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, fmt.Sprintf("%s,%s,%s,%s,%s", mountOptForNFSVer, mountModeUpdated, mountOptForTimeout, mountOptForRetransmissions, mountOptForReadOnlyTrue))) + + }) + + It("Remove_mount_options_from_nfs_sc", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc.Spec.MountOptions = nil + + err = cl.Update(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(1)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, mountOptForNFSVer)) + + }) + + It("Add_partial_mount_options_to_nfs_sc", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc.Spec.MountOptions = &v1alpha1.NFSStorageClassMountOptions{ + MountMode: mountModeUpdated, + Retransmissions: retransmissions, + } + + err = cl.Update(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + Expect(nsc).NotTo(BeNil()) + Expect(nsc.Name).To(Equal(nameForTestResource)) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSc(sc, server, share, nameForTestResource, controllerNamespace) + Expect(sc.MountOptions).To(HaveLen(3)) + Expect(sc.MountOptions).To((ContainElements(mountOptForNFSVer, mountModeUpdated, mountOptForRetransmissions))) + Expect(sc.Parameters).To(HaveLen(6)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.MountPermissionsParamKey, chmodPermissions)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.SubDirParamKey, subDir)) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(err).NotTo(HaveOccurred()) + performStandartChecksForSecret(secret, nameForTestResource, controllerNamespace) + + Expect(secret.StringData).To(HaveKeyWithValue(controller.MountOptionsSecretKey, fmt.Sprintf("%s,%s,%s", mountOptForNFSVer, mountModeUpdated, mountOptForRetransmissions))) + + }) + + It("Remove_nfs_sc", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Delete(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc = &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + secret := &corev1.Secret{} + err = cl.Get(ctx, client.ObjectKey{Name: controller.SecretForMountOptionsPrefix + nameForTestResource, Namespace: controllerNamespace}, secret) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + }) + + It("Create_nfs_sc_when_sc_with_another_provisioner_exists", func() { + sc := &v1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: nameForTestResource, + }, + Provisioner: "test-provisioner", + } + + err := cl.Create(ctx, sc) + Expect(err).NotTo(HaveOccurred()) + + nfsSCtemplate := generateNFSStorageClass(NFSStorageClassConfig{ + Name: nameForTestResource, + Host: server, + Share: share, + NFSVersion: nfsVer, + MountMode: mountMode, + ReadOnly: &readOnlyFalse, + ReclaimPolicy: string(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: string(v1.VolumeBindingWaitForFirstConsumer), + }) + + err = cl.Create(ctx, nfsSCtemplate) + Expect(err).NotTo(HaveOccurred()) + + nsc := &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).To(HaveOccurred()) + Expect(shouldRequeue).To(BeTrue()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + Expect(sc.Provisioner).To(Equal("test-provisioner")) + Expect(sc.Finalizers).To(HaveLen(0)) + Expect(sc.Labels).To(HaveLen(0)) + }) + + It("Remove_nfs_sc_when_sc_with_another_provisioner_exists", func() { + nsc := &v1alpha1.NFSStorageClass{} + err := cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Delete(ctx, nsc) + Expect(err).NotTo(HaveOccurred()) + + nsc = &v1alpha1.NFSStorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(err).NotTo(HaveOccurred()) + Expect(nsc.Finalizers).To(HaveLen(1)) + Expect(nsc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + + scList := &v1.StorageClassList{} + err = cl.List(ctx, scList) + Expect(err).NotTo(HaveOccurred()) + + shouldRequeue, err := controller.RunEventReconcile(ctx, cl, log, scList, nsc, controllerNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse()) + + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, nsc) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + + sc := &v1.StorageClass{} + err = cl.Get(ctx, client.ObjectKey{Name: nameForTestResource}, sc) + Expect(err).NotTo(HaveOccurred()) + Expect(sc.Provisioner).To(Equal("test-provisioner")) + Expect(sc.Finalizers).To(HaveLen(0)) + Expect(sc.Labels).To(HaveLen(0)) + }) + + // TODO: "Create_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_does_not_exists", "Create_nfs_sc_when_sc_does_not_exists_and_secret_exists", "Create_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_exists", "Update_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_does_not_exists", "Remove_nfs_sc_when_sc_with_nfs_provisioner_exists_and_secret_does_not_exists", "Remove_nfs_sc_when_sc_does_not_exists_and_secret_exists" + +}) + +type NFSStorageClassConfig struct { + Name string + Host string + Share string + SubDir string + NFSVersion string + MountMode string + Timeout int + Retransmissions int + ReadOnly *bool + ChmodPermissions string + ReclaimPolicy string + VolumeBindingMode string +} + +func generateNFSStorageClass(cfg NFSStorageClassConfig) *v1alpha1.NFSStorageClass { + return &v1alpha1.NFSStorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: cfg.Name, + }, + Spec: v1alpha1.NFSStorageClassSpec{ + Connection: &v1alpha1.NFSStorageClassConnection{ + Host: cfg.Host, + Share: cfg.Share, + SubDir: cfg.SubDir, + NFSVersion: cfg.NFSVersion, + }, + MountOptions: &v1alpha1.NFSStorageClassMountOptions{ + MountMode: cfg.MountMode, + Timeout: cfg.Timeout, + Retransmissions: cfg.Retransmissions, + ReadOnly: cfg.ReadOnly, + }, + ChmodPermissions: cfg.ChmodPermissions, + ReclaimPolicy: cfg.ReclaimPolicy, + VolumeBindingMode: cfg.VolumeBindingMode, + }, + } +} + +func BoolPtr(b bool) *bool { + return &b +} + +func performStandartChecksForSc(sc *v1.StorageClass, server, share, nameForTestResource, controllerNamespace string) { + Expect(sc).NotTo(BeNil()) + Expect(sc.Name).To(Equal(nameForTestResource)) + Expect(sc.Finalizers).To(HaveLen(1)) + Expect(sc.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) + Expect(sc.Provisioner).To(Equal(controller.NFSStorageClassProvisioner)) + Expect(*sc.ReclaimPolicy).To(Equal(corev1.PersistentVolumeReclaimDelete)) + Expect(*sc.VolumeBindingMode).To(Equal(v1.VolumeBindingWaitForFirstConsumer)) + Expect(sc.Parameters).To(HaveKeyWithValue("server", server)) + Expect(sc.Parameters).To(HaveKeyWithValue("share", share)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.StorageClassSecretNameKey, controller.SecretForMountOptionsPrefix+nameForTestResource)) + Expect(sc.Parameters).To(HaveKeyWithValue(controller.StorageClassSecretNSKey, controllerNamespace)) +} + +func performStandartChecksForSecret(secret *corev1.Secret, nameForTestResource, controllerNamespace string) { + Expect(secret).NotTo(BeNil()) + Expect(secret.Name).To(Equal(controller.SecretForMountOptionsPrefix + nameForTestResource)) + Expect(secret.Namespace).To(Equal(controllerNamespace)) + Expect(secret.Finalizers).To(HaveLen(1)) + Expect(secret.Finalizers).To(ContainElement(controller.NFSStorageClassFinalizerName)) +} diff --git a/images/controller/pkg/kubutils/kubernetes.go b/images/controller/pkg/kubutils/kubernetes.go new file mode 100644 index 0000000..4714cfe --- /dev/null +++ b/images/controller/pkg/kubutils/kubernetes.go @@ -0,0 +1,35 @@ +/* +Copyright 2024 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubutils + +import ( + "fmt" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func KubernetesDefaultConfigCreate() (*rest.Config, error) { + //todo validate empty + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ) + + // Get a config to talk to API server + config, err := clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("config kubernetes error %w", err) + } + return config, nil +} diff --git a/images/controller/pkg/logger/logger.go b/images/controller/pkg/logger/logger.go new file mode 100644 index 0000000..345af2b --- /dev/null +++ b/images/controller/pkg/logger/logger.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "flag" + "fmt" + "github.com/go-logr/logr" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" +) + +const ( + ErrorLevel Verbosity = "0" + WarningLevel Verbosity = "1" + InfoLevel Verbosity = "2" + DebugLevel Verbosity = "3" + TraceLevel Verbosity = "4" +) + +const ( + warnLvl = iota + 1 + infoLvl + debugLvl + traceLvl +) + +type ( + Verbosity string +) + +type Logger struct { + log logr.Logger +} + +func NewLogger(level Verbosity) (*Logger, error) { + klog.InitFlags(nil) + if err := flag.Set("v", string(level)); err != nil { + return nil, err + } + flag.Parse() + + log := klogr.New().WithCallDepth(1) + + return &Logger{log: log}, nil +} + +func (l Logger) GetLogger() logr.Logger { + return l.log +} + +func (l Logger) Error(err error, message string, keysAndValues ...interface{}) { + l.log.Error(err, fmt.Sprintf("ERROR %s", message), keysAndValues...) +} + +func (l Logger) Warning(message string, keysAndValues ...interface{}) { + l.log.V(warnLvl).Info(fmt.Sprintf("WARNING %s", message), keysAndValues...) +} + +func (l Logger) Info(message string, keysAndValues ...interface{}) { + l.log.V(infoLvl).Info(fmt.Sprintf("INFO %s", message), keysAndValues...) +} + +func (l Logger) Debug(message string, keysAndValues ...interface{}) { + l.log.V(debugLvl).Info(fmt.Sprintf("DEBUG %s", message), keysAndValues...) +} + +func (l Logger) Trace(message string, keysAndValues ...interface{}) { + l.log.V(traceLvl).Info(fmt.Sprintf("TRACE %s", message), keysAndValues...) +} diff --git a/images/csi-nfs/werf.inc.yaml b/images/csi-nfs/werf.inc.yaml index f0b4f80..258530c 100644 --- a/images/csi-nfs/werf.inc.yaml +++ b/images/csi-nfs/werf.inc.yaml @@ -16,7 +16,7 @@ shell: install: - export GO_VERSION={{ env "GOLANG_VERSION" }} - export GOPROXY={{ env "GOPROXY" }} - - git clone --depth 1 --branch {{ env "SOURCE_REPO_TAG" }} {{ env "SOURCE_REPO" }}/kubernetes-csi/csi-driver-nfs.git /csi-driver-nfs + - git clone --depth 1 --branch v4.7.0 {{ env "SOURCE_REPO" }}/kubernetes-csi/csi-driver-nfs.git /csi-driver-nfs - cd /csi-driver-nfs/cmd/nfsplugin - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o /nfsplugin - chmod +x /nfsplugin diff --git a/images/webhooks/src/go.mod b/images/webhooks/src/go.mod new file mode 100644 index 0000000..039314d --- /dev/null +++ b/images/webhooks/src/go.mod @@ -0,0 +1,35 @@ +module webhooks + +go 1.22.1 + +toolchain go1.22.2 + +require ( + github.com/sirupsen/logrus v1.9.3 + github.com/slok/kubewebhook/v2 v2.6.0 + k8s.io/api v0.30.0 + k8s.io/apimachinery v0.30.0 + k8s.io/klog/v2 v2.120.1 +) + +require ( + github.com/go-logr/logr v1.4.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/client-go v0.30.0 // indirect + k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/images/webhooks/src/go.sum b/images/webhooks/src/go.sum new file mode 100644 index 0000000..5a26076 --- /dev/null +++ b/images/webhooks/src/go.sum @@ -0,0 +1,110 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slok/kubewebhook/v2 v2.6.0 h1:NMDDXx219OcNDc17ZYpqGXW81/jkBNmkdEwFDcZDVcA= +github.com/slok/kubewebhook/v2 v2.6.0/go.mod h1:EoPfBo8lzgU1lmI1DSY/Fpwu+cdr4lZnzY4Tmg5sHe0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= +k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 h1:ao5hUqGhsqdm+bYbjH/pRkCs0unBGe9UyDahzs9zQzQ= +k8s.io/utils v0.0.0-20240423183400-0849a56e8f22/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/images/webhooks/src/handlers/func.go b/images/webhooks/src/handlers/func.go new file mode 100644 index 0000000..f0c9246 --- /dev/null +++ b/images/webhooks/src/handlers/func.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handlers + +import ( + "context" + "net/http" + + "github.com/slok/kubewebhook/v2/pkg/log" + + kwhhttp "github.com/slok/kubewebhook/v2/pkg/http" + "github.com/slok/kubewebhook/v2/pkg/model" + kwhmutating "github.com/slok/kubewebhook/v2/pkg/webhook/mutating" + kwhvalidating "github.com/slok/kubewebhook/v2/pkg/webhook/validating" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetMutatingWebhookHandler(mutationFunc func(ctx context.Context, _ *model.AdmissionReview, obj metav1.Object) (*kwhmutating.MutatorResult, error), mutatorID string, obj metav1.Object, logger log.Logger) (http.Handler, error) { + mutatorFunc := kwhmutating.MutatorFunc(mutationFunc) + + mutatingWebhookConfig := kwhmutating.WebhookConfig{ + ID: mutatorID, + Obj: obj, + Mutator: mutatorFunc, + Logger: logger, + } + + mutationWebhook, err := kwhmutating.NewWebhook(mutatingWebhookConfig) + if err != nil { + return nil, err + } + + mutationWebhookHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: mutationWebhook, Logger: logger}) + + return mutationWebhookHandler, err + +} + +func GetValidatingWebhookHandler(validationFunc func(ctx context.Context, _ *model.AdmissionReview, obj metav1.Object) (*kwhvalidating.ValidatorResult, error), validatorID string, obj metav1.Object, logger log.Logger) (http.Handler, error) { + validatorFunc := kwhvalidating.ValidatorFunc(validationFunc) + + validatingWebhookConfig := kwhvalidating.WebhookConfig{ + ID: validatorID, + Obj: obj, + Validator: validatorFunc, + Logger: logger, + } + + mutationWebhook, err := kwhvalidating.NewWebhook(validatingWebhookConfig) + if err != nil { + return nil, err + } + + mutationWebhookHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: mutationWebhook, Logger: logger}) + + return mutationWebhookHandler, err + +} diff --git a/images/webhooks/src/handlers/scValidator.go b/images/webhooks/src/handlers/scValidator.go new file mode 100644 index 0000000..e236317 --- /dev/null +++ b/images/webhooks/src/handlers/scValidator.go @@ -0,0 +1,58 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handlers + +import ( + "context" + "fmt" + + "k8s.io/klog/v2" + + "github.com/slok/kubewebhook/v2/pkg/model" + kwhvalidating "github.com/slok/kubewebhook/v2/pkg/webhook/validating" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + NFSStorageClassProvisioner = "nfs.csi.k8s.io" + allowedUserName = "system:serviceaccount:d8-csi-nfs:controller" +) + +func SCValidate(ctx context.Context, arReview *model.AdmissionReview, obj metav1.Object) (*kwhvalidating.ValidatorResult, error) { + sc, ok := obj.(*storagev1.StorageClass) + if !ok { + // If not a storage class just continue the validation chain(if there is one) and do nothing. + return &kwhvalidating.ValidatorResult{}, nil + } + + if sc.Provisioner == NFSStorageClassProvisioner { + if arReview.UserInfo.Username == allowedUserName { + klog.Infof("User %s is allowed to manage storage classes with provisioner %s", arReview.UserInfo.Username, NFSStorageClassProvisioner) + return &kwhvalidating.ValidatorResult{Valid: true}, + nil + } else { + klog.Infof("User %s is not allowed to manage storage classes with provisioner %s", arReview.UserInfo.Username, NFSStorageClassProvisioner) + return &kwhvalidating.ValidatorResult{Valid: false, Message: fmt.Sprintf("Manual operations with the StorageClass that uses the %s provisioner are not allowed. Please use NFSStorageClass instead.", NFSStorageClassProvisioner)}, + nil + } + } else { + return &kwhvalidating.ValidatorResult{Valid: true}, + nil + } + +} diff --git a/images/webhooks/src/main.go b/images/webhooks/src/main.go new file mode 100644 index 0000000..6555aab --- /dev/null +++ b/images/webhooks/src/main.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "webhooks/handlers" + + "github.com/sirupsen/logrus" + kwhlogrus "github.com/slok/kubewebhook/v2/pkg/log/logrus" + storagev1 "k8s.io/api/storage/v1" +) + +type config struct { + certFile string + keyFile string +} + +//goland:noinspection SpellCheckingInspection +func httpHandlerHealthz(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "Ok.") +} + +func initFlags() config { + cfg := config{} + + fl := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + fl.StringVar(&cfg.certFile, "tls-cert-file", "", "TLS certificate file") + fl.StringVar(&cfg.keyFile, "tls-key-file", "", "TLS key file") + + fl.Parse(os.Args[1:]) + return cfg +} + +const ( + port = ":8443" + PodSchedulerMutatorID = "PodSchedulerMutation" + LSCValidatorId = "LSCValidator" + SCValidatorId = "SCValidator" +) + +func main() { + logrusLogEntry := logrus.NewEntry(logrus.New()) + logrusLogEntry.Logger.SetLevel(logrus.DebugLevel) + logger := kwhlogrus.NewLogrus(logrusLogEntry) + + cfg := initFlags() + + scValidatingWebhookHandler, err := handlers.GetValidatingWebhookHandler(handlers.SCValidate, SCValidatorId, &storagev1.StorageClass{}, logger) + if err != nil { + fmt.Fprintf(os.Stderr, "error creating scValidatingWebhookHandler: %s", err) + os.Exit(1) + } + + mux := http.NewServeMux() + mux.Handle("/sc-validate", scValidatingWebhookHandler) + mux.HandleFunc("/healthz", httpHandlerHealthz) + + logger.Infof("Listening on %s", port) + err = http.ListenAndServeTLS(port, cfg.certFile, cfg.keyFile, mux) + if err != nil { + fmt.Fprintf(os.Stderr, "error serving webhook: %s", err) + os.Exit(1) + } +} diff --git a/images/csi-nfs-controller/werf.inc.yaml b/images/webhooks/werf.inc.yaml similarity index 51% rename from images/csi-nfs-controller/werf.inc.yaml rename to images/webhooks/werf.inc.yaml index 28bf6b2..50aafaf 100644 --- a/images/csi-nfs-controller/werf.inc.yaml +++ b/images/webhooks/werf.inc.yaml @@ -1,21 +1,16 @@ --- -image: csi-nfs-controller +image: webhooks from: "registry.deckhouse.io/base_images/golang:1.22.1-alpine@sha256:0de6cf7cceab6ecbf0718bdfb675b08b78113c3709c5e4b99456cdb2ae8c2495" git: - - url: https://github.com/kubernetes-csi/csi-driver-nfs.git - tag: v4.6.0 - add: / - to: / + - add: /images/webhooks/src + to: /src stageDependencies: setup: - "**/*" shell: setup: - - cd /cmd/nfsplugin - - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o nfsplugin - - mv nfsplugin /nfsplugin - - chmod +x /nfsplugin -docker: - ENTRYPOINT: ["/nfsplugin"] + - cd /src + - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o webhooks + - mv webhooks /webhooks diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 75c7415..e8571b8 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -1,7 +1,12 @@ type: object properties: - replicas: - type: integer - description: | - replicas count. - default: 1 + logLevel: + type: string + enum: + - ERROR + - WARN + - INFO + - DEBUG + - TRACE + description: Module log level + default: DEBUG diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml new file mode 100644 index 0000000..c901909 --- /dev/null +++ b/openapi/doc-ru-config-values.yaml @@ -0,0 +1,4 @@ +type: object +properties: + logLevel: + description: Уровень логирования модуля. diff --git a/openapi/values.yaml b/openapi/values.yaml index 2322851..d1d8cb4 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -11,6 +11,23 @@ properties: default: [] items: type: string + customWebhookCert: + type: object + default: {} + x-required-for-helm: + - crt + - key + - ca + properties: + crt: + type: string + x-examples: ["YjY0ZW5jX3N0cmluZwo="] + key: + type: string + x-examples: ["YjY0ZW5jX3N0cmluZwo="] + ca: + type: string + x-examples: ["YjY0ZW5jX3N0cmluZwo="] registry: type: object description: "System field, overwritten by Deckhouse. Don't use" diff --git a/templates/controller/deployment.yaml b/templates/controller/deployment.yaml new file mode 100644 index 0000000..795cbd4 --- /dev/null +++ b/templates/controller/deployment.yaml @@ -0,0 +1,98 @@ +{{- define "controller_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: controller + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: "controller" + minAllowed: + {{- include "controller_resources" . | nindent 8 }} + maxAllowed: + cpu: 200m + memory: 100Mi +{{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller" )) | nindent 2 }} +spec: + minAvailable: {{ include "helm_lib_is_ha_to_value" (list . 1 0) }} + selector: + matchLabels: + app: controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +spec: + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" . | nindent 2 }} + revisionHistoryLimit: 2 + selector: + matchLabels: + app: controller + template: + metadata: + labels: + app: controller + spec: + {{- include "helm_lib_priority_class" (tuple . "cluster-medium") | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple . "system") | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "system") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_nobody" . | nindent 6 }} + imagePullSecrets: + - name: {{ .Chart.Name }}-module-registry + serviceAccountName: controller + containers: + - name: controller + image: {{ include "helm_lib_module_image" (list . "controller") }} + imagePullPolicy: IfNotPresent + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} +{{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "controller_resources" . | nindent 14 }} +{{- end }} + securityContext: + privileged: true + seLinuxOptions: + level: s0 + type: spc_t + env: + - name: LOG_LEVEL +{{- if eq .Values.csiNfs.logLevel "ERROR" }} + value: "0" +{{- else if eq .Values.csiNfs.logLevel "WARN" }} + value: "1" +{{- else if eq .Values.csiNfs.logLevel "INFO" }} + value: "2" +{{- else if eq .Values.csiNfs.logLevel "DEBUG" }} + value: "3" +{{- else if eq .Values.csiNfs.logLevel "TRACE" }} + value: "4" +{{- end }} + - name: CONTROLLER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/templates/controller/rbac-for-us.yaml b/templates/controller/rbac-for-us.yaml new file mode 100644 index 0000000..a900f22 --- /dev/null +++ b/templates/controller/rbac-for-us.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - list + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:controller + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +rules: + - apiGroups: + - storage.deckhouse.io + resources: + - nfsstorageclasses + - nfsstorageclasses/status + verbs: + - get + - list + - create + - delete + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - create + - delete + - list + - get + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: controller + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: controller + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: Role + name: controller + apiGroup: rbac.authorization.k8s.io + + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:controller + {{- include "helm_lib_module_labels" (list . (dict "app" "controller")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: controller + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:controller + apiGroup: rbac.authorization.k8s.io + + diff --git a/templates/csi/controller.yaml b/templates/csi/controller.yaml index 7cdc485..39a0422 100644 --- a/templates/csi/controller.yaml +++ b/templates/csi/controller.yaml @@ -47,6 +47,7 @@ {{- $_ := set $csiControllerConfig "snapshotterEnabled" true }} {{- $_ := set $csiControllerConfig "resizerEnabled" false }} {{- $_ := set $csiControllerConfig "snapshotterTimeout" "1200s" }} +{{- $_ := set $csiControllerConfig "extraCreateMetadataEnabled" true }} {{- $_ := set $csiControllerConfig "livenessProbePort" 29652 }} {{- $_ := set $csiControllerConfig "additionalControllerArgs" (include "csi_controller_args" . | fromYamlArray) }} {{- $_ := set $csiControllerConfig "additionalControllerEnvs" (include "csi_controller_envs" . | fromYamlArray) }} diff --git a/templates/webhooks/deployment.yaml b/templates/webhooks/deployment.yaml new file mode 100644 index 0000000..83bcd09 --- /dev/null +++ b/templates/webhooks/deployment.yaml @@ -0,0 +1,96 @@ +{{- define "webhooks_resources" }} +cpu: 10m +memory: 50Mi +{{- end }} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks" "workload-resource-policy.deckhouse.io" "master")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: webhooks + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: webhooks + minAllowed: + {{- include "webhooks_resources" . | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 100Mi +{{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks" )) | nindent 2 }} +spec: + {{- include "helm_lib_deployment_on_master_strategy_and_replicas_for_ha" . | nindent 2 }} + selector: + matchLabels: + app: webhooks + template: + metadata: + labels: + app: webhooks + spec: + {{- include "helm_lib_priority_class" (tuple . "system-cluster-critical") | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "any-node" "with-uninitialized" "with-cloud-provider-uninitialized") | nindent 6 }} + {{- include "helm_lib_node_selector" (tuple . "master") | nindent 6 }} + {{- include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "webhooks")) | nindent 6 }} + containers: + - name: webhooks + command: + - /webhooks + - -tls-cert-file=/etc/webhook/certs/tls.crt + - -tls-key-file=/etc/webhook/certs/tls.key + image: {{ include "helm_lib_module_image" (list . "webhooks") }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + readinessProbe: + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 5 + failureThreshold: 2 + periodSeconds: 1 + livenessProbe: + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + periodSeconds: 1 + failureThreshold: 3 + ports: + - name: http + containerPort: 8443 + protocol: TCP + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }} +{{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "webhooks_resources" . | nindent 12 }} +{{- end }} + + imagePullSecrets: + - name: {{ .Chart.Name }}-module-registry + serviceAccount: webhooks + serviceAccountName: webhooks + volumes: + - name: webhook-certs + secret: + secretName: webhooks-https-certs \ No newline at end of file diff --git a/templates/webhooks/rbac-for-us.yaml b/templates/webhooks/rbac-for-us.yaml new file mode 100644 index 0000000..9e1970b --- /dev/null +++ b/templates/webhooks/rbac-for-us.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks")) | nindent 2 }} diff --git a/templates/webhooks/secret.yaml b/templates/webhooks/secret.yaml new file mode 100644 index 0000000..ee8ca4a --- /dev/null +++ b/templates/webhooks/secret.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: webhooks-https-certs + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks")) | nindent 2 }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.csiNfs.internal.customWebhookCert.ca }} + tls.crt: {{ .Values.csiNfs.internal.customWebhookCert.crt }} + tls.key: {{ .Values.csiNfs.internal.customWebhookCert.key }} diff --git a/templates/webhooks/service.yaml b/templates/webhooks/service.yaml new file mode 100644 index 0000000..dc01ddb --- /dev/null +++ b/templates/webhooks/service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: webhooks + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks" )) | nindent 2 }} +spec: + type: ClusterIP + ports: + - port: 443 + targetPort: 8443 + protocol: TCP + name: http + selector: + app: webhooks \ No newline at end of file diff --git a/templates/webhooks/webhook.yaml b/templates/webhooks/webhook.yaml new file mode 100644 index 0000000..891515f --- /dev/null +++ b/templates/webhooks/webhook.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: "d8-{{ .Chart.Name }}-sc-validation" +webhooks: + - name: "d8-{{ .Chart.Name }}-sc-validation.deckhouse.io" + rules: + - apiGroups: ["storage.k8s.io"] + apiVersions: ["v1"] + operations: ["*"] + resources: ["storageclasses"] + scope: "Cluster" + clientConfig: + service: + namespace: "d8-{{ .Chart.Name }}" + name: "webhooks" + path: "/sc-validate" + caBundle: | + {{ .Values.csiNfs.internal.customWebhookCert.ca }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/werf-giterminism.yaml b/werf-giterminism.yaml index 23dc40a..8145506 100644 --- a/werf-giterminism.yaml +++ b/werf-giterminism.yaml @@ -1,7 +1,7 @@ giterminismConfigVersion: 1 config: goTemplateRendering: # The rules for the Go-template functions to be able to pass build context to the release - allowEnvVariables: [ /CI_.+/, MODULES_MODULE_TAG, GOLANG_VERSION, GOPROXY, SOURCE_REPO_TAG, SOURCE_REPO ] + allowEnvVariables: [ /CI_.+/, MODULES_MODULE_TAG, GOLANG_VERSION, GOPROXY, SOURCE_REPO ] stapel: mount: allowBuildDir: true