diff --git a/README.rst b/README.rst index 8aebd42b..b14262b5 100755 --- a/README.rst +++ b/README.rst @@ -350,6 +350,60 @@ shows the different configuration options available: # image is passed to subsequent steps. import: path/to/image/archive.tar + # Specify the secrets that should be used when building your image, + # similar to the --secret option used by Docker + # More info about secrets: https://docs.docker.com/build/building/secrets/ + secrets: + # Example of a secret that is a file + - id=secret1,src= + # Example of a secret that is an environment variable + - id=secret2,env= + +.. _Build Secrets: + +Build Secrets +============= + +Buildrunner supports specifying secrets that should be used when building your image, +similar to the --secret option used by Docker. This is done by adding the ``secrets`` +section to the ``build`` section. This is a list of secrets that should be used when +building the image. The string should be in the format of ``id=secret1,src=`` +when the secret is a file or ``id=secret2,env=`` when the secret is an environment variable. +This syntax is the same as the syntax used by Docker to build with secrets. +More info about building with secrets in docker and the syntax of the secret string +see https://docs.docker.com/build/building/secrets/. + +In order to use secrets in buildrunner, you need to do the following: + +#. Update the buildrunner configuration file + * Set ``use-legacy-builder`` to ``false`` or add ``platforms`` to the ``build`` section + * Add the secrets to the ``secrets`` section in the ``build`` section +#. Update the Dockerfile to use the secrets + * Add the ``--mount`` at the beginning of each RUN command that needs the secret + +.. code:: yaml + + use-legacy-builder: false + steps: + build-my-container: + build: + dockerfile: | + FROM alpine:latest + # Using secrets inline + RUN --mount=type=secret,id=secret1 \ + --mount=type=secret,id=secret2 \ + echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2) + # Using secrets in environment variables + RUN --mount=type=secret,id=secret1 \ + --mount=type=secret,id=secret2 \ + SECRET1_FILE=/run/secrets/secret1 \ + SECRET2_VARIABLE=$(cat /run/secrets/secret2) \ + && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE + secrets: + # Example of a secret that is a file + - id=secret1,src=examples/build/secrets/secret1.txt + # Example of a secret that is an environment variable + - id=secret2,env=SECRET2 .. _Running Containers: diff --git a/buildrunner/config/models.py b/buildrunner/config/models.py index e2e1d466..403fb8eb 100644 --- a/buildrunner/config/models.py +++ b/buildrunner/config/models.py @@ -150,7 +150,7 @@ class Config(BaseModel, extra="forbid"): @field_validator("steps") @classmethod - def validate_steps(cls, vals) -> None: + def validate_steps(cls, vals, info) -> None: """ Validate the config file @@ -161,13 +161,26 @@ def validate_steps(cls, vals) -> None: if not vals: raise ValueError('The "steps" configuration was not provided') - # Checks to see if there is a mutli-platform build step in the config + # Checks steps for mutli-platform or secrets has_multi_platform_build = False + + # Check for multi-platform builds and secrets validation for step in vals.values(): has_multi_platform_build = ( has_multi_platform_build or step.is_multi_platform() ) + # If the step has secrets and the builder is legacy or no platforms are set for the step, raise an error + if ( + step.has_secrets() + and not step.is_multi_platform() + and info.data.get("use_legacy_builder") + ): + raise ValueError( + "Build secrets are not supported with the legacy builder. Please set use-legacy-builder to false" + " or add platforms to the build section in order to use secrets in your build." + ) + if has_multi_platform_build: mp_push_tags = set() validate_multiplatform_build(vals, mp_push_tags) diff --git a/buildrunner/config/models_step.py b/buildrunner/config/models_step.py index a81c08f0..35149869 100644 --- a/buildrunner/config/models_step.py +++ b/buildrunner/config/models_step.py @@ -80,6 +80,7 @@ class StepBuild(StepTask): cache_to: Optional[Union[str, Dict[str, str]]] = None # import is a python reserved keyword so we need to alias it import_param: Optional[str] = Field(alias="import", default=None) + secrets: Optional[List[str]] = None class RunAndServicesBase(StepTask): @@ -242,3 +243,9 @@ def is_multi_platform(self): Check if the step is a multi-platform build step """ return self.build and self.build.platforms is not None + + def has_secrets(self): + """ + Check if the step has secrets + """ + return self.build and self.build.secrets is not None diff --git a/buildrunner/docker/multiplatform_image_builder.py b/buildrunner/docker/multiplatform_image_builder.py index 3f2bbdac..c114b3ac 100644 --- a/buildrunner/docker/multiplatform_image_builder.py +++ b/buildrunner/docker/multiplatform_image_builder.py @@ -80,6 +80,7 @@ def __init__( cache_builders: Optional[List[str]] = None, cache_from: Optional[Union[dict, str]] = None, cache_to: Optional[Union[dict, str]] = None, + secrets: Optional[List[str]] = None, ): self._docker_registry = docker_registry self._build_registry = build_registry @@ -89,6 +90,7 @@ def __init__( self._cache_builders = set(cache_builders if cache_builders else []) self._cache_from = cache_from self._cache_to = cache_to + self._secrets = secrets if self._cache_from or self._cache_to: LOGGER.info( f'Configuring multiplatform builds to cache from {cache_from} and to {cache_to} ' @@ -196,14 +198,11 @@ def _build_with_inject( self, inject: dict, image_ref: str, - platform: str, path: str, + platform: str, dockerfile: str, - target: str, build_args: dict, - builder: Optional[str], - cache: bool = False, - pull: bool = False, + build_kwargs: dict, ) -> None: if not path or not os.path.isdir(path): LOGGER.warning( @@ -256,16 +255,13 @@ def _build_with_inject( logs_itr = docker.buildx.build( context_dir, - tags=[image_ref], - platforms=[platform], - load=True, - target=target, - builder=builder, build_args=build_args, - cache=cache, - pull=pull, + load=True, + platforms=[platform], stream_logs=True, - **self._get_build_cache_options(builder), + tags=[image_ref], + **build_kwargs, + **self._get_build_cache_options(build_kwargs.get("builder")), ) self._log_buildx(logs_itr, platform) @@ -294,6 +290,7 @@ def _build_single_image( inject: dict, cache: bool = False, pull: bool = False, + secrets: Optional[List[str]] = None, ) -> None: """ Builds a single image for the given platform. @@ -307,6 +304,7 @@ def _build_single_image( target (str): The name of the stage to build in a multi-stage Dockerfile build_args (dict): The build args to pass to docker. inject (dict): The files to inject into the build context. + secrets (List[str]): The secrets to pass to docker. """ assert os.path.isdir(path) and os.path.exists(dockerfile), ( f"Either path {path} ({os.path.isdir(path)}) or file " @@ -321,18 +319,28 @@ def _build_single_image( f"Building image for platform {platform} with {builder or 'default'} builder" ) + # Build kwargs for the buildx build command + build_kwargs = {} + if builder: + build_kwargs["builder"] = builder + if cache: + build_kwargs["cache"] = cache + if pull: + build_kwargs["pull"] = pull + if secrets: + build_kwargs["secrets"] = secrets + if target: + build_kwargs["target"] = target + if inject and isinstance(inject, dict): self._build_with_inject( + path=path, inject=inject, image_ref=image_ref, platform=platform, - path=path, dockerfile=dockerfile, - target=target, build_args=build_args, - builder=builder, - cache=cache, - pull=pull, + build_kwargs=build_kwargs, ) else: logs_itr = docker.buildx.build( @@ -341,12 +349,9 @@ def _build_single_image( platforms=[platform], load=True, file=dockerfile, - target=target, build_args=build_args, - builder=builder, - cache=cache, - pull=pull, stream_logs=True, + **build_kwargs, **self._get_build_cache_options(builder), ) self._log_buildx(logs_itr, platform) @@ -403,6 +408,7 @@ def build_multiple_images( inject: dict = None, cache: bool = False, pull: bool = False, + secrets: Optional[List[str]] = None, ) -> BuiltImageInfo: """ Builds multiple images for the given platforms. One image will be built for each platform. @@ -498,6 +504,7 @@ def build_multiple_images( inject, cache, pull, + secrets, ) LOGGER.debug(f"Building {repo} for {platform}") if use_threading: diff --git a/buildrunner/steprunner/tasks/build.py b/buildrunner/steprunner/tasks/build.py index cc9c308f..eccda2e4 100644 --- a/buildrunner/steprunner/tasks/build.py +++ b/buildrunner/steprunner/tasks/build.py @@ -238,6 +238,7 @@ def run(self, context): inject=self.to_inject, cache=not self.nocache, pull=self.pull, + secrets=self.step.secrets, ) # Set expected number of platforms diff --git a/examples/build/secrets/buildrunner.yaml b/examples/build/secrets/buildrunner.yaml new file mode 100644 index 00000000..a42c8028 --- /dev/null +++ b/examples/build/secrets/buildrunner.yaml @@ -0,0 +1,27 @@ +# In order to use secrets, you need to set use-legacy-builder to false in the config file +# To run this example, you need to set the SECRET_PASSWORD environment variable +# and run the example with the following command: +# SECRET2=my_secret ./run-buildrunner.sh -f examples/build/secrets/buildrunner.yaml +# More info about secrets: https://docs.docker.com/build/building/secrets/ +use-legacy-builder: false +steps: + simple-build-step: + build: + no-cache: true + dockerfile: | + FROM alpine:latest + # Using secrets inline + RUN --mount=type=secret,id=secret1 \ + --mount=type=secret,id=secret2 \ + echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2) + # Using secrets in environment variables + RUN --mount=type=secret,id=secret1 \ + --mount=type=secret,id=secret2 \ + SECRET1_FILE=/run/secrets/secret1 \ + SECRET2_VARIABLE=$(cat /run/secrets/secret2) \ + && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE + secrets: + # Example of a secret that is a file + - id=secret1,src=examples/build/secrets/secret1.txt + # Example of a secret that is an environment variable + - id=secret2,env=SECRET2 diff --git a/examples/build/secrets/platforms-buildrunner.yaml b/examples/build/secrets/platforms-buildrunner.yaml new file mode 100644 index 00000000..9a627ac5 --- /dev/null +++ b/examples/build/secrets/platforms-buildrunner.yaml @@ -0,0 +1,29 @@ +# In order to use secrets, you need to set use-legacy-builder to false in the config file OR +# add platforms to the build section +# To run this example, you need to set the SECRET_PASSWORD environment variable +# and run the example with the following command: +# SECRET2=my_secret ./run-buildrunner.sh -f examples/build/secrets/platforms-buildrunner.yaml +# More info about secrets: https://docs.docker.com/build/building/secrets/ +steps: + simple-build-step: + build: + dockerfile: | + FROM alpine:latest + # Using secrets inline + RUN --mount=type=secret,id=secret1 \ + --mount=type=secret,id=secret2 \ + echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2) + # Using secrets in environment variables + RUN --mount=type=secret,id=secret1 \ + --mount=type=secret,id=secret2 \ + SECRET1_FILE=/run/secrets/secret1 \ + SECRET2_VARIABLE=$(cat /run/secrets/secret2) \ + && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE + secrets: + # Example of a secret that is a file + - id=secret1,src=examples/build/secrets/secret1.txt + # Example of a secret that is an environment variable + - id=secret2,env=SECRET2 + platforms: + - linux/amd64 + - linux/arm64 diff --git a/examples/build/secrets/secret1.txt b/examples/build/secrets/secret1.txt new file mode 100644 index 00000000..27deaa73 --- /dev/null +++ b/examples/build/secrets/secret1.txt @@ -0,0 +1 @@ +testuser123 \ No newline at end of file diff --git a/tests/test_buildrunner_files.py b/tests/test_buildrunner_files.py index 1cf5f622..88951444 100644 --- a/tests/test_buildrunner_files.py +++ b/tests/test_buildrunner_files.py @@ -124,6 +124,8 @@ def _get_example_runs(test_dir: str) -> List[Tuple[str, str, Optional[List[str]] excluded_example_files = [ "examples/build/import/buildrunner.yaml", "examples/run/caches/buildrunner.yaml", + # This file is not supported in the github actions runner + "examples/build/secrets/platforms-buildrunner.yaml", ] # Walk through the examples directory and find all files ending with buildrunner.yaml diff --git a/tests/test_multiplatform.py b/tests/test_multiplatform.py index 1258218c..3d6b758c 100644 --- a/tests/test_multiplatform.py +++ b/tests/test_multiplatform.py @@ -478,12 +478,8 @@ def test_build_multiple_builds( load=True, file=f"{test_path}/Dockerfile", build_args={"DOCKER_REGISTRY": None}, - builder=None, - cache=False, cache_from=None, cache_to=None, - pull=False, - target=None, stream_logs=True, ), call( @@ -493,12 +489,8 @@ def test_build_multiple_builds( load=True, file=f"{test_path}/Dockerfile", build_args={"DOCKER_REGISTRY": None}, - builder=None, - cache=False, cache_from=None, cache_to=None, - pull=False, - target=None, stream_logs=True, ), call( @@ -508,12 +500,8 @@ def test_build_multiple_builds( load=True, file=f"{test_path}/Dockerfile", build_args={"DOCKER_REGISTRY": None}, - builder=None, - cache=False, cache_from=None, cache_to=None, - pull=False, - target=None, stream_logs=True, ), call( @@ -523,12 +511,8 @@ def test_build_multiple_builds( load=True, file=f"{test_path}/Dockerfile", build_args={"DOCKER_REGISTRY": None}, - builder=None, - cache=False, cache_from=None, cache_to=None, - pull=False, - target=None, stream_logs=True, ), ]