Skip to content

Commit

Permalink
feat(ci): Build multi-arch Docker images (amd64, arm64) (#496)
Browse files Browse the repository at this point in the history
* build ARM images
* Fix GH API rate limits
* `docker buildx` currently does not support `load` and multi-arch at the same time. 
  And used Github Action does not support output=type=oci

---------

Co-authored-by: Maksym Vlasov <MaxymVlasov@users.noreply.github.com>
Co-authored-by: George L. Yermulnik <yz@yz.kiev.ua>
3 people authored Apr 28, 2023
1 parent f766cba commit 923c2c6
Showing 4 changed files with 75 additions and 28 deletions.
28 changes: 27 additions & 1 deletion .github/workflows/build-image-test.yaml
Original file line number Diff line number Diff line change
@@ -22,17 +22,25 @@ jobs:
.dockerignore
tools/entrypoint.sh
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0

- name: Build if Dockerfile changed
if: steps.changed-files-specific.outputs.any_changed == 'true'
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
with:
context: .
build-args: |
INSTALL_ALL=true
platforms: linux/amd64
platforms: linux/amd64 # Only one allowed here, see https://github.com/docker/buildx/issues/59#issuecomment-1433097926
push: false
load: true
tags: |
ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }}
# Fix multi-platform: https://github.com/docker/buildx/issues/1533
provenance: false
secrets: |
"github_token=${{ secrets.GITHUB_TOKEN }}"
- name: Run structure tests
if: steps.changed-files-specific.outputs.any_changed == 'true'
@@ -48,3 +56,21 @@ jobs:
image: ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }}
config-file: ${{ github.workspace }}/.github/.dive-ci.yaml
github-token: ${{ secrets.GITHUB_TOKEN }}

# Can't build both platforms and use --load at the same time
# https://github.com/docker/buildx/issues/59#issuecomment-1433097926
- name: Build Multi-arch docker-image
if: steps.changed-files-specific.outputs.any_changed == 'true'
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
with:
context: .
build-args: |
INSTALL_ALL=true
platforms: linux/amd64,linux/arm64
push: false
tags: |
ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }}
# Fix multi-platform: https://github.com/docker/buildx/issues/1533
provenance: false
secrets: |
"github_token=${{ secrets.GITHUB_TOKEN }}"
17 changes: 15 additions & 2 deletions .github/workflows/build-image.yaml
Original file line number Diff line number Diff line change
@@ -27,26 +27,39 @@ jobs:
- name: Set tag for image
run: |
echo IMAGE_TAG=$([ ${{ github.ref_type }} == 'tag' ] && echo ${{ github.ref_name }} || echo 'latest') >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0

- name: Build and Push release
if: github.event_name != 'schedule'
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
with:
context: .
build-args: |
INSTALL_ALL=true
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
push: true
tags: |
ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }}
ghcr.io/${{ github.repository }}:latest
# Fix multi-platform: https://github.com/docker/buildx/issues/1533
provenance: false
secrets: |
"github_token=${{ secrets.GITHUB_TOKEN }}"
- name: Build and Push nightly
if: github.event_name == 'schedule'
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
with:
context: .
build-args: |
INSTALL_ALL=true
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
push: true
tags: |
ghcr.io/${{ github.repository }}:nightly
# Fix multi-platform: https://github.com/docker/buildx/issues/1533
provenance: false
secrets: |
"github_token=${{ secrets.GITHUB_TOKEN }}"
41 changes: 23 additions & 18 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
ARG TAG=3.11.1-alpine3.17@sha256:d8b0703ce84fe5a52d485f212e9d852bcdb8606798064f5f21af57325a7cf73f
FROM python:${TAG} as builder
ARG TARGETOS
ARG TARGETARCH

WORKDIR /bin_dir

@@ -22,7 +24,7 @@ RUN [ ${PRE_COMMIT_VERSION} = "latest" ] && pip3 install --no-cache-dir pre-comm
RUN if [ "${TERRAFORM_VERSION}" = "latest" ]; then \
TERRAFORM_VERSION="$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | grep -o -E -m 1 "[0-9.]+")" \
; fi && \
curl -L "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" > terraform.zip && \
curl -L "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip" > terraform.zip && \
unzip terraform.zip terraform && rm terraform.zip

#
@@ -74,18 +76,18 @@ RUN . /.env && \
if [ "$INFRACOST_VERSION" != "false" ]; then \
( \
INFRACOST_RELEASES="https://api.github.com/repos/infracost/infracost/releases" && \
[ "$INFRACOST_VERSION" = "latest" ] && curl -L "$(curl -s ${INFRACOST_RELEASES}/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz \
|| curl -L "$(curl -s ${INFRACOST_RELEASES} | grep -o -E "https://.+?v${INFRACOST_VERSION}/infracost-linux-amd64.tar.gz")" > infracost.tgz \
) && tar -xzf infracost.tgz && rm infracost.tgz && mv infracost-linux-amd64 infracost \
[ "$INFRACOST_VERSION" = "latest" ] && curl -L "$(curl -s ${INFRACOST_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz \
|| curl -L "$(curl -s ${INFRACOST_RELEASES} | grep -o -E "https://.+?v${INFRACOST_VERSION}/infracost-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz \
) && tar -xzf infracost.tgz && rm infracost.tgz && mv infracost-${TARGETOS}-${TARGETARCH} infracost \
; fi

# Terraform docs
RUN . /.env && \
if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then \
( \
TERRAFORM_DOCS_RELEASES="https://api.github.com/repos/terraform-docs/terraform-docs/releases" && \
[ "$TERRAFORM_DOCS_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > terraform-docs.tgz \
|| curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-linux-amd64.tar.gz")" > terraform-docs.tgz \
[ "$TERRAFORM_DOCS_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz \
|| curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz \
) && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs \
; fi

@@ -94,19 +96,22 @@ RUN . /.env \
&& if [ "$TERRAGRUNT_VERSION" != "false" ]; then \
( \
TERRAGRUNT_RELEASES="https://api.github.com/repos/gruntwork-io/terragrunt/releases" && \
[ "$TERRAGRUNT_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_linux_amd64")" > terragrunt \
|| curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_linux_amd64")" > terragrunt \
[ "$TERRAGRUNT_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt \
|| curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt \
) && chmod +x terragrunt \
; fi


# Terrascan
RUN . /.env && \
if [ "$TERRASCAN_VERSION" != "false" ]; then \
if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="x86_64"; fi; \
# Convert the first letter to Uppercase
OS="$(echo ${TARGETOS} | cut -c1 | tr '[:lower:]' '[:upper:]' | xargs echo -n; echo ${TARGETOS} | cut -c2-)"; \
( \
TERRASCAN_RELEASES="https://api.github.com/repos/tenable/terrascan/releases" && \
[ "$TERRASCAN_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz \
|| curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_Linux_x86_64.tar.gz")" > terrascan.tar.gz \
[ "$TERRASCAN_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz \
|| curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz \
) && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && \
./terrascan init \
; fi
@@ -116,8 +121,8 @@ RUN . /.env && \
if [ "$TFLINT_VERSION" != "false" ]; then \
( \
TFLINT_RELEASES="https://api.github.com/repos/terraform-linters/tflint/releases" && \
[ "$TFLINT_VERSION" = "latest" ] && curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_linux_amd64.zip")" > tflint.zip \
|| curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_linux_amd64.zip")" > tflint.zip \
[ "$TFLINT_VERSION" = "latest" ] && curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip \
|| curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip \
) && unzip tflint.zip && rm tflint.zip \
; fi

@@ -126,8 +131,8 @@ RUN . /.env && \
if [ "$TFSEC_VERSION" != "false" ]; then \
( \
TFSEC_RELEASES="https://api.github.com/repos/aquasecurity/tfsec/releases" && \
[ "$TFSEC_VERSION" = "latest" ] && curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-linux-amd64")" > tfsec \
|| curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-linux-amd64")" > tfsec \
[ "$TFSEC_VERSION" = "latest" ] && curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec \
|| curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec \
) && chmod +x tfsec \
; fi

@@ -136,8 +141,8 @@ RUN . /.env && \
if [ "$TFUPDATE_VERSION" != "false" ]; then \
( \
TFUPDATE_RELEASES="https://api.github.com/repos/minamijoyo/tfupdate/releases" && \
[ "$TFUPDATE_VERSION" = "latest" ] && curl -L "$(curl -s ${TFUPDATE_RELEASES}/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > tfupdate.tgz \
|| curl -L "$(curl -s ${TFUPDATE_RELEASES} | grep -o -E -m 1 "https://.+?${TFUPDATE_VERSION}_linux_amd64.tar.gz")" > tfupdate.tgz \
[ "$TFUPDATE_VERSION" = "latest" ] && curl -L "$(curl -s ${TFUPDATE_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz \
|| curl -L "$(curl -s ${TFUPDATE_RELEASES} | grep -o -E -m 1 "https://.+?${TFUPDATE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz \
) && tar -xzf tfupdate.tgz tfupdate && rm tfupdate.tgz \
; fi

@@ -146,8 +151,8 @@ RUN . /.env && \
if [ "$HCLEDIT_VERSION" != "false" ]; then \
( \
HCLEDIT_RELEASES="https://api.github.com/repos/minamijoyo/hcledit/releases" && \
[ "$HCLEDIT_VERSION" = "latest" ] && curl -L "$(curl -s ${HCLEDIT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > hcledit.tgz \
|| curl -L "$(curl -s ${HCLEDIT_RELEASES} | grep -o -E -m 1 "https://.+?${HCLEDIT_VERSION}_linux_amd64.tar.gz")" > hcledit.tgz \
[ "$HCLEDIT_VERSION" = "latest" ] && curl -L "$(curl -s ${HCLEDIT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz \
|| curl -L "$(curl -s ${HCLEDIT_RELEASES} | grep -o -E -m 1 "https://.+?${HCLEDIT_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz \
) && tar -xzf hcledit.tgz hcledit && rm hcledit.tgz \
; fi

17 changes: 10 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -97,7 +97,10 @@ All available tags [here](https://github.com/antonbabenko/pre-commit-terraform/p

**Build from scratch**:

When `--build-arg` is not specified, the latest version of `pre-commit` and `terraform` will be only installed.
> **Note**: To build image you need to have [`docker buildx`](https://docs.docker.com/build/install-buildx/) enabled as default builder.
> Otherwise - provide `TARGETOS` and `TARGETARCH` as additional `--build-arg`'s to `docker build`.
When hooks-related `--build-arg`s are not specified, only the latest version of `pre-commit` and `terraform` will be installed.

```bash
git clone git@github.com:antonbabenko/pre-commit-terraform.git
@@ -184,7 +187,7 @@ curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/late

We highly recommend using [WSL/WSL2](https://docs.microsoft.com/en-us/windows/wsl/install) with Ubuntu and following the Ubuntu installation guide. Or use Docker.

> Note: We won't be able to help with issues that can't be reproduced in Linux/Mac.
> **Note**: We won't be able to help with issues that can't be reproduced in Linux/Mac.
> So, try to find a working solution and send PR before open an issue.
Otherwise, you can follow [this gist](https://gist.github.com/etiennejeanneaurevolve/1ed387dc73c5d4cb53ab313049587d09):
@@ -204,7 +207,7 @@ E.g. `C:\Users\USERNAME\AppData\Local\Programs\Python\Python39\Lib\site-packages

### 2. Install the pre-commit hook globally

> Note: not needed if you use the Docker image
> **Note**: not needed if you use the Docker image
```bash
DIR=~/.git-template
@@ -238,7 +241,7 @@ pre-commit run -a

Or, using Docker ([available tags](https://github.com/antonbabenko/pre-commit-terraform/pkgs/container/pre-commit-terraform/versions)):

> Note: This command uses your user id and group id for the docker container to use to access the local files. If the files are owned by another user, update the `USERID` environment variable. See [File Permissions section](#docker-usage-file-permissions) for more information.
> **Note**: This command uses your user id and group id for the docker container to use to access the local files. If the files are owned by another user, update the `USERID` environment variable. See [File Permissions section](#docker-usage-file-permissions) for more information.
```bash
TAG=latest
@@ -686,7 +689,7 @@ To replicate functionality in `terraform_docs` hook:
- --hook-config=--retry-once-with-cleanup=true # Boolean. true or false
```

> Note: The flag requires additional dependency to be installed: `jq`.
> **Note**: The flag requires additional dependency to be installed: `jq`.

If `--retry-once-with-cleanup=true`, then in each failed directory the cached modules and providers from the `.terraform` directory will be deleted, before retrying once more. To avoid unnecessary deletion of this directory, the cleanup and retry will only happen if Terraform produces any of the following error messages:

@@ -696,7 +699,7 @@ To replicate functionality in `terraform_docs` hook:
* "Module not installed"
* "Could not load plugin"

**Warning:** When using `--retry-once-with-cleanup=true`, problematic `.terraform/modules/` and `.terraform/providers/` directories will be recursively deleted without prompting for consent. Other files and directories will not be affected, such as the `.terraform/environment` file.
**Warning**: When using `--retry-once-with-cleanup=true`, problematic `.terraform/modules/` and `.terraform/providers/` directories will be recursively deleted without prompting for consent. Other files and directories will not be affected, such as the `.terraform/environment` file.

**Option 2**

@@ -714,7 +717,7 @@ To replicate functionality in `terraform_docs` hook:

`terraform_validate` hook will try to reinitialize them before running the `terraform validate` command.

**Warning:** If you use Terraform workspaces, DO NOT use this option ([details](https://github.com/antonbabenko/pre-commit-terraform/issues/203#issuecomment-918791847)). Consider the first option, or wait for [`force-init`](https://github.com/antonbabenko/pre-commit-terraform/issues/224) option implementation.
**Warning**: If you use Terraform workspaces, DO NOT use this option ([details](https://github.com/antonbabenko/pre-commit-terraform/issues/203#issuecomment-918791847)). Consider the first option, or wait for [`force-init`](https://github.com/antonbabenko/pre-commit-terraform/issues/224) option implementation.

4. `terraform_validate` in a repo with Terraform module, written using Terraform 0.15+ and which uses provider `configuration_aliases` ([Provider Aliases Within Modules](https://www.terraform.io/language/modules/develop/providers#provider-aliases-within-modules)), errors out.

0 comments on commit 923c2c6

Please sign in to comment.