Implemented "transform" of "required-envs" for build, template, deplo…

…y, run and system-test

README.md

@@ -24,7 +24,7 @@ It builds on well-known and field-proven tools:
 
 ## Current state of development
 
-`kubedev` is in early development and used internally by Gira.
+`kubedev` is in early development and used internally at Gira.
 
 ## Synopsis
 
@@ -77,6 +77,10 @@ Schema of kubedev.json:
             "required-envs": {
                 "MYDEPLOY_FLASK_ENV": {
                     "documentation": "..."
+                },
+                "MYDEPLOY_COMPLEX": {
+                  "documentation": "This is a variable with content that can not be passed to the helm chart on the command line and must be base64-encoded.",
+                  "transform": "base64"
                 }
             },
             # Defines the system test for this app. System tests must be defined in the directory ./systemTests/<app-name>/ and must include a Dockerfile:
@@ -150,6 +154,20 @@ When kubedev needs to access docker registries, it writes the content of `${DOCK
 - The environment variable `${CI}` is set.
 - The environment variable `${DOCKER_AUTH_CONFIG}` is set.
 
+## Environment Variable Transformation
+
+Environment Variables are passed to `helm` and `docker` by shell expansion. This has some limitations of values that are not "shell-safe", such as when they contain double-quotes or special characters. To make it safe and possible to pass these values, kubedev provides a few transformations.
+
+A transformation will take the content of the variable from the environment where kubedev is called, and pass it in a transformed way into `helm --set` and `docker --build-arg / --env`.
+
+Available transformations are:
+
+|Transformation|Description|
+|--------------|-----------|
+|base64|Base64 encodes the value|
+
+You can enable a transformation by setting the attribute `transform` in `required-envs` to the desired transformation, e.g. `base64`.
+
 ## kubedev generate
 
 Creates artifacts that kick-starts your microservice development.
@@ -234,7 +252,7 @@ The following parameters are passed to `docker run`, some of them can be configu
 
 ## kubedev deploy
 
-Reads a kube config from the env var $KUBEDEV_KUBECONFIG (required) and optionally a context from $KUBEDEV_KUBECONTEXT and then runs `helm upgrade --install` with appropriate arguments and all env vars from `kubedev.json`.
+Reads a kube config from the env var $KUBEDEV_KUBECONFIG (required), writes it to *\<temp-kubeconfig-file\>* and optionally a context from $KUBEDEV_KUBECONTEXT and then runs `helm upgrade --install --kube-config "<temp-kubeconfig-file>` with appropriate arguments and all required env vars from `kubedev.json`.
 
 See [Naming Conventions](#naming-conventions).
 

kubedev/kubedev.py

@@ -74,10 +74,10 @@ def execute(self, commandWithArgs: list, envVars: dict = dict(), piped_input: st
                           check    = check
                           ).returncode
 
-  def get_output(self, commandWithArgs, check=True):
+  def get_output(self, commandWithArgs, envVars: dict = dict(), check=True):
     cmds = [cmd for cmd in commandWithArgs if cmd is not None]
     print(f'{colorama.Fore.CYAN}➡️   Executing "{" ".join(cmds)}"')
-    cmdResult = subprocess.run(cmds, check=check, stdout=subprocess.PIPE, encoding='utf-8')
+    cmdResult = subprocess.run(cmds, check=check, env=envVars, stdout=subprocess.PIPE, encoding='utf-8')
     if cmdResult.returncode == 0:
       return cmdResult.stdout
     else:
@@ -364,34 +364,36 @@ def _get_kubecontext_arg(self, env_accessor):
   def _template(self, kubedev, shell_executor, env_accessor, file_accessor, get_output=False):
     variables = KubedevConfig.get_global_variables(kubedev)
     tag = KubedevConfig.get_tag(env_accessor)
+    envs = KubedevConfig.get_helm_set_env_args(kubedev, env_accessor)
     command = [
         '/bin/sh',
         '-c',
         f'helm template ./helm-chart/ ' +
         f'--set KUBEDEV_TAG="{tag}"' +
-        KubedevConfig.get_helm_set_env_args(kubedev)
+        envs['cmdline']
     ]
     if not get_output:
-      return shell_executor.execute(command, variables)
+      return shell_executor.execute(command, {**variables, **envs['envs']})
     else:
-      return shell_executor.get_output(command)
+      return shell_executor.get_output(command, {**variables, **envs['envs']})
 
   def _deploy(self, kubedev, release_name, shell_executor, env_accessor, file_accessor, get_output=False):
     variables = KubedevConfig.get_global_variables(kubedev)
     tag = KubedevConfig.get_tag(env_accessor)
     kubeconfig = KubedevConfig.get_kubeconfig_path(env_accessor, file_accessor)
+    envs = KubedevConfig.get_helm_set_env_args(kubedev, env_accessor)
     command = [
         '/bin/sh',
         '-c',
         f'helm upgrade {release_name} ./helm-chart/ --install --wait ' +
         f'--kubeconfig {kubeconfig} {self._get_kubecontext_arg(env_accessor)} ' +
         f'--set KUBEDEV_TAG="{tag}"' +
-        KubedevConfig.get_helm_set_env_args(kubedev)
+        envs['cmdline']
     ]
     if not get_output:
-      return shell_executor.execute(command, variables)
+      return shell_executor.execute(command, {**variables, **envs['envs']})
     else:
-      return shell_executor.get_output(command)
+      return shell_executor.get_output(command, {**variables, **envs['envs']})
 
   def template(self, configFileName, shell_executor=RealShellExecutor(), env_accessor=RealEnvAccessor(), file_accessor=RealFileAccessor()):
     return self.template_from_config(
@@ -448,14 +450,15 @@ def build_from_config(self, kubedev, container, force_tag, file_accessor, shell_
         imageTag = image['imageName']
       else:
         imageTag = f"{image['imageNameTagless']}:{force_tag}"
+      (argsCmdLine, extraEnv) = KubedevConfig.get_docker_build_args(image, env_accessor=env_accessor)
       call = [
           '/bin/sh',
           '-c',
           f"docker build -t {imageTag} " +
-          KubedevConfig.get_docker_build_args(image) +
+          argsCmdLine +
           f"{image['buildPath']}"
       ]
-      return shell_executor.execute(call, check=False)
+      return shell_executor.execute(call, envVars=extraEnv, check=False)
 
   def push(self, configFileName, container, file_accessor=RealFileAccessor(), shell_executor=RealShellExecutor(), env_accessor=RealEnvAccessor()):
     return self.push_from_config(
@@ -580,8 +583,8 @@ def is_command(cmd):
             '❌ Required field "imagePullSecrets" is missing in kubedev.json', True)
         result = False
 
-    envs = KubedevConfig.get_all_env_names(kubedev, build=is_command(
-        'build'), container=is_command('deploy') or is_command('template'))
+    envs = KubedevConfig.get_all_envs(kubedev, build=is_command(
+        'build'), container=is_command('deploy') or is_command('template')).keys()
     for env in sorted(envs):
       if isinstance(env_accessor.getenv(env), type(None)):
         printer.print(
@@ -621,16 +624,18 @@ def run_from_config(self,
       if buildResult != 0:
         return buildResult
       else:
+        (runEnvArgs, extraEnvs) = KubedevConfig.get_docker_run_envs(image, env_accessor=env_accessor)
+
         command = [
           '/bin/sh',
           '-c',
           f"docker run --interactive {interactive_flags}--rm " +
           KubedevConfig.get_docker_run_volumes(image, file_accessor, shell_executor) +
           KubedevConfig.get_docker_run_ports(image) +
-          KubedevConfig.get_docker_run_envs(image) +
+          runEnvArgs +
           f"{image['imageNameTagless']}:{currentTag}"
         ]
-        return shell_executor.execute(command, check=False)
+        return shell_executor.execute(command, envVars=extraEnvs, check=False)
 
   @staticmethod
   def _run_docker_detached(network: str, name: str, ports: list, rawImage: str, images: dict, variables: dict, shell_executor: object) -> (str, bool):
@@ -675,7 +680,7 @@ def _build_image(name: str, images: dict) -> (str, bool):
       if len(name) > 3 and name[0] == '{' and name[-1] == '}':
           appName = name[1:-1]
           if appName in images:
-              return (images[appName]['imageName'], images[appName]['containerEnvs'], True)
+              return (images[appName]['imageName'], images[appName]['containerEnvs'].keys(), True)
           else:
               raise Exception(f'App "{appName}" is referenced by the system test service {name}, but is not defined in kubedev config')
       return (name, dict(), False)
@@ -742,7 +747,7 @@ def system_test_from_config(self, kubedev, appName: str, file_accessor, env_acce
       buildArgs = self._field_optional(testContainer, "buildArgs", dict())
       variables = {**globalVariables, **self._field_optional(testContainer, "variables", dict())}
       requiredEnvs = images[appName]['containerEnvs']
-      filteredRequiredEnvs = sorted([env for env in requiredEnvs if env not in variables])
+      (filteredRequiredEnvs, additionalEnvs) = KubedevConfig.prepare_envs(requiredEnvs, env_accessor)
 
       containerDir = f"./systemTests/{appName}/"
       uuid = tag_generator.tag()
@@ -795,10 +800,10 @@ def system_test_from_config(self, kubedev, appName: str, file_accessor, env_acce
               "--network", network,
               "--name", f"{appName}-system-tests-{uuid}",
               "--interactive"] + \
-              functools.reduce(operator.concat, [["--env", f'{envName}="${{{envName}}}"'] for envName in filteredRequiredEnvs], []) + \
+              functools.reduce(operator.concat, [["--env", f'{envName}="${{{attribs["targetName"]}}}"'] for envName, attribs in filteredRequiredEnvs.items()], []) + \
               functools.reduce(operator.concat, [["--env", f'{varName}="{varValue}"'] for varName, varValue in variables.items()], []) + \
               [tag])]
-          if shell_executor.execute(cmdRunSystemTests, check=False) == 0:
+          if shell_executor.execute(cmdRunSystemTests, envVars=additionalEnvs, check=False) == 0:
               result = True
           else:
               print()

kubedev/utils/kubedev_config.py

@@ -1,4 +1,5 @@
 import os
+from base64 import b64encode
 from string import Template
 
 kubeconfig_temp_path = os.path.join('.kubedev', 'kube_config_tmp')
@@ -15,15 +16,14 @@ def get_global_variables(kubedev):
     }
 
   @staticmethod
-  def get_all_env_names(kubedev, build, container):
-    envs = set(KubedevConfig.load_envs(
-        kubedev, build=build, container=container).keys())
+  def get_all_envs(kubedev, build, container):
+    envs = KubedevConfig.load_envs(
+        kubedev, build=build, container=container)
     if 'deployments' in kubedev:
       for (_, deployment) in kubedev['deployments'].items():
         deploymentEnvs = KubedevConfig.load_envs(
             deployment, build=build, container=container)
-        envs = {
-            *envs, *set(deploymentEnvs.keys())}
+        envs = {**envs, **deploymentEnvs}
     return envs
 
   @staticmethod
@@ -39,17 +39,40 @@ def if_exists(obj: dict, field: str) -> dict:
     }
 
   @staticmethod
-  def get_helm_set_env_args(kubedev):
+  def prepare_envs(envs: dict, env_accessor: object) -> tuple:
+    def is_base64(attribs: dict) -> bool:
+      return 'transform' in attribs and attribs['transform'] == 'base64'
+
+    def env_name(name: str, attribs: dict) -> str:
+      if 'transform' in attribs and attribs['transform'] == 'base64':
+        return f'{name}_AS_BASE64'
+      else:
+        return name
+
+    sortedEnvs = dict(sorted(envs.items()))
+    return (
+      {name: {**attribs, **{'targetName': env_name(name, attribs)}} for name, attribs in sortedEnvs.items()},
+      {env_name(name, attribs): b64encode(env_accessor.getenv(name, default="").encode('utf-8')) for name, attribs in sortedEnvs.items() if is_base64(attribs)}
+    )
+
+  @staticmethod
+  def get_helm_set_env_args(kubedev: dict, env_accessor: object) -> dict:
     '''
     Returns shell parameters for helm commands in the form of ``--set <variable>="${<variable>}" ...''
     from a kubedev config.
     '''
-    envs = KubedevConfig.get_all_env_names(kubedev, False, True)
+    (envs, extraEnvs) = KubedevConfig.prepare_envs(KubedevConfig.get_all_envs(kubedev, False, True), env_accessor=env_accessor)
 
     if len(envs) > 0:
-      return ' ' + ' '.join([f'--set {e}="${{{e}}}"' for e in sorted(envs)])
+      return {
+        'cmdline': ' ' + ' '.join([f'--set {name}="${{{attribs["targetName"]}}}"' for name, attribs in envs.items()]),
+        'envs': extraEnvs
+      }
     else:
-      return ''
+      return {
+        'cmdline': '',
+        'envs': dict()
+      }
 
   @staticmethod
   def get_kubeconfig_path(env_accessor, file_accessor):
@@ -106,8 +129,8 @@ def get_images(kubedev, env_accessor):
             "imageNameTagless": f"{imageRegistry}/{finalDeploymentName}",
             "buildPath": KubedevConfig.get_buildpath(name, deploymentKey),
             "ports": deployment['ports'] if 'ports' in deployment else dict(),
-            "buildEnvs": {*globalBuildEnvs, *KubedevConfig.load_envs(deployment, True, False)},
-            "containerEnvs": {*globalContainerEnvs, *KubedevConfig.load_envs(deployment, False, True)},
+            "buildEnvs": {**globalBuildEnvs, **KubedevConfig.load_envs(deployment, True, False)},
+            "containerEnvs": {**globalContainerEnvs, **KubedevConfig.load_envs(deployment, False, True)},
             "volumes": deployment["volumes"]["dev"] if "volumes" in deployment and "dev" in deployment["volumes"] else dict(),
             "usedFrameworks": usedFrameworks
         }
@@ -137,24 +160,30 @@ def collapse_names(first, second):
       return f'{first}-{second}'
 
   @staticmethod
-  def get_docker_build_args(image):
+  def get_docker_build_args(image: dict, env_accessor: object) -> tuple:
     """
     Returns a string with all "--build-arg ..." parameters to the "docker build ..." call.
 
     :param image: One entry returned from KubedevConfig.get_images()
     """
-    envs = image['buildEnvs']
-    return " ".join([f'--build-arg {env}="${{{env}}}"' for env in sorted(envs)]) + " "
+    (envs, extraEnvs) = KubedevConfig.prepare_envs(image['buildEnvs'], env_accessor=env_accessor)
+    return (
+      " ".join([f'--build-arg {name}="${{{attribs["targetName"]}}}"' for name, attribs in sorted(envs.items())]) + " ",
+      extraEnvs
+    )
 
   @staticmethod
-  def get_docker_run_envs(image):
+  def get_docker_run_envs(image: dict, env_accessor: object) -> tuple:
     """
     Returns a string with all "--env ..." parameters to the "docker run ..." call.
 
     :param image: One entry returned from KubedevConfig.get_images()
     """
-    envs = image['containerEnvs']
-    return " ".join([f'--env {env}="${{{env}}}"' for env in sorted(envs)]) + " "
+    (envs, extraEnvs) = KubedevConfig.prepare_envs(image['containerEnvs'], env_accessor)
+    return (
+      " ".join([f'--env {name}="${{{attribs["targetName"]}}}"' for name, attribs in sorted(envs.items())]) + " ",
+      extraEnvs
+    )
 
   @staticmethod
   def get_docker_run_volumes(image, file_accessor, shell_executor):

test_utils/__init__.py

@@ -1,4 +1,5 @@
-from .configs import (testDeploymentConfig, testMixedSubProjectsConfig,
+from .configs import (testDeploymentBase64EnvConfig, testDeploymentConfig,
+                      testGlobalBase64EnvConfig, testMixedSubProjectsConfig,
                       testMultiDeploymentsConfig)
 from .mocks import (DownloadMock, EnvMock, FileMock, OutputMock,
                     ShellExecutorMock, SleepMock, TagGeneratorMock,

test_utils/configs.py

@@ -222,3 +222,63 @@
       # TODO
     }
 }
+
+testGlobalBase64EnvConfig = {
+    "name": "foo-service",
+    "description": "This is a sample service generated by kubedev.",
+    "imagePullSecrets": "foo-creds",
+    "imageRegistry": "foo-registry",
+    "required-envs": {
+        "FOO_SERVICE_GLOBAL_ENV1": {
+            "documentation": "Test env var #1 (global)",
+            "transform": "base64"
+        },
+    },
+    "deployments": {
+        "foo-deploy": {
+            "systemTest": {
+                "testContainer": {
+                },
+                "services": {
+                    "{foo-deploy}": {
+                        "hostname": "foo-deploy-test",
+                        "ports": [1234],
+                        "variables": {
+                            "FOO_SERVICE_DEPLOY_ENV1": "fixed-value"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+testDeploymentBase64EnvConfig = {
+    "name": "foo-service",
+    "description": "This is a sample service generated by kubedev.",
+    "imagePullSecrets": "foo-creds",
+    "imageRegistry": "foo-registry",
+    "deployments": {
+        "foo-deploy": {
+            "required-envs": {
+                "FOO_SERVICE_GLOBAL_ENV1": {
+                    "documentation": "Test env var #1 (global)",
+                    "transform": "base64"
+                },
+            },
+            "systemTest": {
+                "testContainer": {
+                },
+                "services": {
+                    "{foo-deploy}": {
+                        "hostname": "foo-deploy-test",
+                        "ports": [1234],
+                        "variables": {
+                            "FOO_SERVICE_DEPLOY_ENV1": "fixed-value"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

test_utils/mocks.py

@@ -15,8 +15,8 @@ def execute(self, commandWithArgs, envVars=dict(), piped_input: str = None, chec
     self._calls.append({'cmd': [cmd for cmd in commandWithArgs if cmd is not None], 'env': envVars, 'withOutput': True})
     return 0
 
-  def get_output(self, commandWithArgs, check=False):
-    self._calls.append({'cmd': [cmd for cmd in commandWithArgs if cmd is not None], 'env': dict(), 'withOutput': True})
+  def get_output(self, commandWithArgs, envVars: dict = dict(), check=False):
+    self._calls.append({'cmd': [cmd for cmd in commandWithArgs if cmd is not None], 'env': envVars, 'withOutput': True})
     if len(self._cmd_output) > 0:
       result = self._cmd_output[0]
       self._cmd_output = self._cmd_output[1:]