From 86d832b620762a6f316ffccfd12ffeb6f46f20e5 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 19 Sep 2024 14:01:58 +0200 Subject: [PATCH 01/15] Skip AWS auth if Gitops aws configuration empty in atmos settings --- action.yml | 117 +++++++++++++++++++++++++++++++++---- tests/terraform/atmos.yaml | 7 +++ 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/action.yml b/action.yml index a3969b6..dfdc1e2 100644 --- a/action.yml +++ b/action.yml @@ -78,12 +78,23 @@ runs: echo "opentofu-version=$(atmos describe config -f json | jq -r '.integrations.github.gitops["opentofu-version"]')" >> $GITHUB_OUTPUT echo "terraform-version=$(atmos describe config -f json | jq -r '.integrations.github.gitops["terraform-version"]')" >> $GITHUB_OUTPUT echo "enable-infracost=$(atmos describe config -f json | jq -r '.integrations.github.gitops["infracost-enabled"]')" >> $GITHUB_OUTPUT + # AWS IAM role for Terraform plan + echo "terraform-plan-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.plan')" >> $GITHUB_OUTPUT + echo "terraform-apply-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.apply')" >> $GITHUB_OUTPUT + # AWS plan storage settings echo "aws-region=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].region')" >> $GITHUB_OUTPUT echo "terraform-state-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].role')" >> $GITHUB_OUTPUT echo "terraform-state-table=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].table')" >> $GITHUB_OUTPUT echo "terraform-state-bucket=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].bucket')" >> $GITHUB_OUTPUT - echo "terraform-plan-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.plan')" >> $GITHUB_OUTPUT - echo "terraform-apply-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.apply')" >> $GITHUB_OUTPUT + # Azure plan storage settings + echo "plan-repository-type=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["plan-repository-type"]')" >> $GITHUB_OUTPUT + echo "blob-account-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["blob-account-name"]')" >> $GITHUB_OUTPUT + echo "blob-container-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["blob-container-name"]')" >> $GITHUB_OUTPUT + echo "metadata-repository-type=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["metadata-repository-type"]')" >> $GITHUB_OUTPUT + echo "cosmos-container-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["cosmos-container-name"]')" >> $GITHUB_OUTPUT + echo "cosmos-database-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["cosmos-database-name"]')" >> $GITHUB_OUTPUT + echo "cosmos-endpoint=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["cosmos-endpoint"]')" >> $GITHUB_OUTPUT + - name: Install Terraform if: ${{ steps.config.outputs.terraform-version != '' && steps.config.outputs.terraform-version != 'null' }} @@ -104,7 +115,11 @@ runs: terraform-docs/terraform-docs: v0.18.0 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@v4 + if: ${{ steps.config.outputs.aws-region != '' && + steps.config.outputs.aws-region != 'null' && + steps.config.outputs.terraform-apply-role != '' && + steps.config.outputs.terraform-apply-role != 'null' }} with: aws-region: ${{ steps.config.outputs.aws-region }} role-to-assume: ${{ steps.config.outputs.terraform-apply-role }} @@ -179,16 +194,24 @@ runs: echo "lock_file=$LOCK_FILE" >> $GITHUB_OUTPUT - name: Configure State AWS Credentials - if: env.ACTIONS_ENABLED == 'true' - uses: aws-actions/configure-aws-credentials@v4.0.2 + if: ${{ env.ACTIONS_ENABLED == 'true' && + steps.config.outputs.aws-region != '' && + steps.config.outputs.aws-region != 'null' && + steps.config.outputs.terraform-state-role != '' && + steps.config.outputs.terraform-state-role != 'null' }} + uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ steps.config.outputs.aws-region }} role-to-assume: ${{ steps.config.outputs.terraform-state-role }} role-session-name: "atmos-terraform-state-gitops" mask-aws-account-id: "no" - - name: Retrieve Plan - if: env.ACTIONS_ENABLED == 'true' + - name: Retrieve Plan (AWS) + if: ${{ env.ACTIONS_ENABLED == 'true' && + steps.config.outputs.terraform-state-table != '' && + steps.config.outputs.terraform-state-table != 'null' && + steps.config.outputs.terraform-state-bucket != '' && + steps.config.outputs.terraform-state-bucket != 'null' }} uses: cloudposse/github-action-terraform-plan-storage@v1 id: retrieve-plan continue-on-error: true @@ -201,8 +224,12 @@ runs: tableName: ${{ steps.config.outputs.terraform-state-table }} bucketName: ${{ steps.config.outputs.terraform-state-bucket }} - - name: Retrieve Lockfile - if: env.ACTIONS_ENABLED == 'true' + - name: Retrieve Lockfile (AWS) + if: ${{ env.ACTIONS_ENABLED == 'true' && + steps.config.outputs.terraform-state-table != '' && + steps.config.outputs.terraform-state-table != 'null' && + steps.config.outputs.terraform-state-bucket != '' && + steps.config.outputs.terraform-state-bucket != 'null' }} uses: cloudposse/github-action-terraform-plan-storage@v1 continue-on-error: true with: @@ -214,8 +241,78 @@ runs: tableName: ${{ steps.config.outputs.terraform-state-table }} bucketName: ${{ steps.config.outputs.terraform-state-bucket }} + - name: Retrieve Plan (Azure) + if: ${{ env.ACTIONS_ENABLED == 'true' && + steps.config.outputs.plan-repository-type != '' && + steps.config.outputs.plan-repository-type != 'null' && + steps.config.outputs.blob-account-name != '' && + steps.config.outputs.blob-account-name != 'null' && + steps.config.outputs.blob-container-name != '' && + steps.config.outputs.blob-container-name != 'null' && + steps.config.outputs.metadata-repository-type != '' && + steps.config.outputs.metadata-repository-type != 'null' && + steps.config.outputs.cosmos-container-name != '' && + steps.config.outputs.cosmos-container-name != 'null' && + steps.config.outputs.cosmos-database-name != '' && + steps.config.outputs.cosmos-database-name != 'null' && + steps.config.outputs.cosmos-endpoint != '' && + steps.config.outputs.cosmos-endpoint != 'null' }} + uses: cloudposse/github-action-terraform-plan-storage@v1 + id: retrieve-plan + continue-on-error: true + with: + action: getPlan + planPath: ${{ steps.vars.outputs.plan_file }} + commitSHA: ${{ inputs.sha }} + component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + # Azure settings + planRepositoryType: ${{ steps.config.outputs.plan-repository-type }} + blobAccountName: ${{ steps.config.outputs.blob-account-name }} + blobContainerName: ${{ steps.config.outputs.blob-container-name }} + metadataRepositoryType: ${{ steps.config.outputs.metadata-repository-type }} + cosmosContainerName: ${{ steps.config.outputs.cosmos-container-name }} + cosmosDatabaseName: ${{ steps.config.outputs.cosmos-database-name }} + cosmosEndpoint: ${{ steps.config.outputs.cosmos-endpoint }} + + - name: Retrieve Lockfile (Azure) + if: ${{ env.ACTIONS_ENABLED == 'true' && + steps.config.outputs.plan-repository-type != '' && + steps.config.outputs.plan-repository-type != 'null' && + steps.config.outputs.blob-account-name != '' && + steps.config.outputs.blob-account-name != 'null' && + steps.config.outputs.blob-container-name != '' && + steps.config.outputs.blob-container-name != 'null' && + steps.config.outputs.metadata-repository-type != '' && + steps.config.outputs.metadata-repository-type != 'null' && + steps.config.outputs.cosmos-container-name != '' && + steps.config.outputs.cosmos-container-name != 'null' && + steps.config.outputs.cosmos-database-name != '' && + steps.config.outputs.cosmos-database-name != 'null' && + steps.config.outputs.cosmos-endpoint != '' && + steps.config.outputs.cosmos-endpoint != 'null' }} + uses: cloudposse/github-action-terraform-plan-storage@v1 + continue-on-error: true + with: + action: getPlan + planPath: ${{ steps.vars.outputs.lock_file }} + commitSHA: ${{ inputs.sha }} + component: ${{ inputs.component }} + stack: "${{ inputs.stack }}-lockfile" + # Azure settings + planRepositoryType: ${{ steps.config.outputs.plan-repository-type }} + blobAccountName: ${{ steps.config.outputs.blob-account-name }} + blobContainerName: ${{ steps.config.outputs.blob-container-name }} + metadataRepositoryType: ${{ steps.config.outputs.metadata-repository-type }} + cosmosContainerName: ${{ steps.config.outputs.cosmos-container-name }} + cosmosDatabaseName: ${{ steps.config.outputs.cosmos-database-name }} + cosmosEndpoint: ${{ steps.config.outputs.cosmos-endpoint }} + - name: Configure AWS Credentials - if: env.ACTIONS_ENABLED == 'true' + if: ${{ steps.config.outputs.aws-region != '' && + steps.config.outputs.aws-region != 'null' && + steps.config.outputs.terraform-apply-role != '' && + steps.config.outputs.terraform-apply-role != 'null' }} uses: aws-actions/configure-aws-credentials@v4.0.2 with: aws-region: ${{ steps.config.outputs.aws-region }} diff --git a/tests/terraform/atmos.yaml b/tests/terraform/atmos.yaml index feb4455..ebd48a4 100644 --- a/tests/terraform/atmos.yaml +++ b/tests/terraform/atmos.yaml @@ -69,6 +69,13 @@ integrations: bucket: __STORAGE_BUCKET__ table: __STORAGE_TABLE__ role: __STORAGE_ROLE__ + plan-repository-type: + blob-account-name: + blob-container-name: + metadata-repository-type: + cosmos-container-name: + cosmos-database-name: + cosmos-endpoint: role: plan: __PLAN_ROLE__ apply: __APPLY_ROLE__ From 04d00d423f3047b6f17aac11dad1246f20fcd4b8 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 19 Sep 2024 14:06:29 +0200 Subject: [PATCH 02/15] Skip AWS auth if Gitops aws configuration empty in atmos settings --- action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/action.yml b/action.yml index dfdc1e2..bd2a559 100644 --- a/action.yml +++ b/action.yml @@ -213,7 +213,6 @@ runs: steps.config.outputs.terraform-state-bucket != '' && steps.config.outputs.terraform-state-bucket != 'null' }} uses: cloudposse/github-action-terraform-plan-storage@v1 - id: retrieve-plan continue-on-error: true with: action: getPlan @@ -258,7 +257,6 @@ runs: steps.config.outputs.cosmos-endpoint != '' && steps.config.outputs.cosmos-endpoint != 'null' }} uses: cloudposse/github-action-terraform-plan-storage@v1 - id: retrieve-plan continue-on-error: true with: action: getPlan From 8cbefd38b9f4c30503131b9179db07772bb0c0d9 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 19 Sep 2024 14:14:57 +0200 Subject: [PATCH 03/15] Skip AWS auth if Gitops aws configuration empty in atmos settings --- action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index bd2a559..724c12a 100644 --- a/action.yml +++ b/action.yml @@ -307,7 +307,8 @@ runs: cosmosEndpoint: ${{ steps.config.outputs.cosmos-endpoint }} - name: Configure AWS Credentials - if: ${{ steps.config.outputs.aws-region != '' && + if: ${{ env.ACTIONS_ENABLED == 'true' && + steps.config.outputs.aws-region != '' && steps.config.outputs.aws-region != 'null' && steps.config.outputs.terraform-apply-role != '' && steps.config.outputs.terraform-apply-role != 'null' }} From 04ee60020c5a2d0cb7342483a36cfa06e7e154b0 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 19 Sep 2024 16:55:01 +0200 Subject: [PATCH 04/15] Skip AWS auth if Gitops aws configuration empty in atmos settings --- tests/terraform/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/terraform/atmos.yaml b/tests/terraform/atmos.yaml index ebd48a4..24c55f2 100644 --- a/tests/terraform/atmos.yaml +++ b/tests/terraform/atmos.yaml @@ -62,7 +62,7 @@ stacks: integrations: github: gitops: - terraform-version: 1.5.2 + terraform-version: 1.5.7 infracost-enabled: __INFRACOST_ENABLED__ artifact-storage: region: __STORAGE_REGION__ From 944b189c405f0f87f25caa559c5bfba0716fab73 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 19 Sep 2024 17:38:05 +0200 Subject: [PATCH 05/15] Added readme --- README.yaml | 38 ++++++++++++++++++++++++++++++++++++-- action.yml | 3 +-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/README.yaml b/README.yaml index e066263..18967b0 100644 --- a/README.yaml +++ b/README.yaml @@ -58,9 +58,20 @@ usage: |- In order to retrieve Terraform Plan Files (not to be confused with Terraform State files, e.g. `tfstate`), we configure an S3 Bucket to store plan files and a DynamoDB table to track plan metadata. Both need to be deployed before running this action. For more on setting up those components, see the [`gitops` component](https://docs.cloudposse.com/components/library/aws/gitops/). This action will then use the [github-action-terraform-plan-storage](https://github.com/cloudposse/github-action-terraform-plan-storage) action to update these resources. + ### Config + > [!IMPORTANT] + > **Please note!** This GitHub Action only works with `atmos >= 1.63.0`. If you are using `atmos < 1.63.0` please use `v1` version of this action. + The action expects the atmos configuration file `atmos.yaml` to be present in the repository. + + The action supports AWS and Azure to store Terraform plan files. + You can read more about plan storage in the [cloudposse/github-action-terraform-plan-storage](https://github.com/cloudposse/github-action-terraform-plan-storage?tab=readme-ov-file#aws-default) documentation. + Depends of cloud provider the following fields should be set in the `atmos.yaml`: + + #### AWS + The config should have the following structure: ```yaml @@ -77,15 +88,38 @@ usage: |- role: arn:aws:iam::xxxxxxxxxxxx:role/cptest-core-ue2-auto-gitops-gha role: plan: arn:aws:iam::yyyyyyyyyyyy:role/cptest-core-gbl-identity-gitops + # Set `apply` empty if you don't want to assume IAM role before terraform apply apply: arn:aws:iam::yyyyyyyyyyyy:role/cptest-core-gbl-identity-gitops matrix: sort-by: .stack_slug group-by: .stack_slug | split("-") | [.[0], .[2]] | join("-") ``` + + #### Azure - > [!IMPORTANT] - > **Please note!** This GitHub Action only works with `atmos >= 1.63.0`. If you are using `atmos < 1.63.0` please use `v1` version of this action. + The config should have the following structure: + ```yaml + integrations: + github: + gitops: + opentofu-version: 1.7.3 + terraform-version: 1.5.2 + infracost-enabled: false + artifact-storage: + plan-repository-type: azureblob + blob-account-name: tfplans + blob-container-name: plans + metadata-repository-type: cosmos + cosmos-container-name: terraform-plan-storage + cosmos-database-name: terraform-plan-storage + cosmos-endpoint: "https://my-cosmo-account.documents.azure.com:443/" + # We remove the `role` section as it is AWS specific + matrix: + sort-by: .stack_slug + group-by: .stack_slug | split("-") | [.[0], .[2]] | join("-") + ``` + ### Support OpenTofu This action supports [OpenTofu](https://opentofu.org/). diff --git a/action.yml b/action.yml index 724c12a..7be0cfe 100644 --- a/action.yml +++ b/action.yml @@ -78,8 +78,7 @@ runs: echo "opentofu-version=$(atmos describe config -f json | jq -r '.integrations.github.gitops["opentofu-version"]')" >> $GITHUB_OUTPUT echo "terraform-version=$(atmos describe config -f json | jq -r '.integrations.github.gitops["terraform-version"]')" >> $GITHUB_OUTPUT echo "enable-infracost=$(atmos describe config -f json | jq -r '.integrations.github.gitops["infracost-enabled"]')" >> $GITHUB_OUTPUT - # AWS IAM role for Terraform plan - echo "terraform-plan-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.plan')" >> $GITHUB_OUTPUT + # AWS IAM role for Terraform apply echo "terraform-apply-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.apply')" >> $GITHUB_OUTPUT # AWS plan storage settings echo "aws-region=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].region')" >> $GITHUB_OUTPUT From 872d21914eaeb0882c468052cb0c53e483fd1038 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 28 Nov 2024 23:07:22 +0100 Subject: [PATCH 06/15] Reconsile with https://github.com/cloudposse/github-action-atmos-terraform-plan/pull/92 --- .github/workflows/integration-tests.yml | 2 +- action.yml | 326 ++++++++++++------------ tests/opentofu/atmos.yaml | 2 + tests/terraform/atmos.yaml | 4 +- 4 files changed, 163 insertions(+), 171 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 09b14aa..c082b53 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -47,7 +47,7 @@ jobs: component: "foobar" stack: "plat-ue2-sandbox" atmos-config-path: ${{ runner.temp }} - atmos-version: 1.86.2 + atmos-version: 1.99.0 - uses: actions/checkout@v4 with: diff --git a/action.yml b/action.yml index 7be0cfe..8b74d56 100644 --- a/action.yml +++ b/action.yml @@ -15,10 +15,14 @@ inputs: description: "Commit SHA to apply. Default: github.sha" required: true default: "${{ github.event.pull_request.head.sha }}" + skip-checkout: + description: "Disable actions/checkout. Useful for when the checkout happens in a previous step and file are modified outside of git through other actions" + required: false + default: 'false' atmos-version: description: The version of atmos to install required: false - default: ">= 1.63.0" + default: ">= 1.99.0" atmos-config-path: description: The path to the atmos.yaml file required: true @@ -57,6 +61,7 @@ runs: node-version: 20 - name: Checkout + if: ${{ inputs.skip-checkout != 'true' }} uses: actions/checkout@v4 - name: Set atmos cli config path vars @@ -71,63 +76,9 @@ runs: token: ${{ inputs.token }} install-wrapper: false - - name: config - shell: bash - id: config - run: |- - echo "opentofu-version=$(atmos describe config -f json | jq -r '.integrations.github.gitops["opentofu-version"]')" >> $GITHUB_OUTPUT - echo "terraform-version=$(atmos describe config -f json | jq -r '.integrations.github.gitops["terraform-version"]')" >> $GITHUB_OUTPUT - echo "enable-infracost=$(atmos describe config -f json | jq -r '.integrations.github.gitops["infracost-enabled"]')" >> $GITHUB_OUTPUT - # AWS IAM role for Terraform apply - echo "terraform-apply-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops.role.apply')" >> $GITHUB_OUTPUT - # AWS plan storage settings - echo "aws-region=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].region')" >> $GITHUB_OUTPUT - echo "terraform-state-role=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].role')" >> $GITHUB_OUTPUT - echo "terraform-state-table=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].table')" >> $GITHUB_OUTPUT - echo "terraform-state-bucket=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"].bucket')" >> $GITHUB_OUTPUT - # Azure plan storage settings - echo "plan-repository-type=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["plan-repository-type"]')" >> $GITHUB_OUTPUT - echo "blob-account-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["blob-account-name"]')" >> $GITHUB_OUTPUT - echo "blob-container-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["blob-container-name"]')" >> $GITHUB_OUTPUT - echo "metadata-repository-type=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["metadata-repository-type"]')" >> $GITHUB_OUTPUT - echo "cosmos-container-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["cosmos-container-name"]')" >> $GITHUB_OUTPUT - echo "cosmos-database-name=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["cosmos-database-name"]')" >> $GITHUB_OUTPUT - echo "cosmos-endpoint=$(atmos describe config -f json | jq -r '.integrations.github.gitops["artifact-storage"]["cosmos-endpoint"]')" >> $GITHUB_OUTPUT - - - - name: Install Terraform - if: ${{ steps.config.outputs.terraform-version != '' && steps.config.outputs.terraform-version != 'null' }} - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: ${{ steps.config.outputs.terraform-version }} - terraform_wrapper: false - - - name: Install Dependencies - uses: cloudposse-github-actions/install-gh-releases@v1 - with: - cache: true - config: |- - opentofu/opentofu: - tag: ${{ startsWith(steps.config.outputs.opentofu-version, 'v') && steps.config.outputs.opentofu-version || format('v{0}', steps.config.outputs.opentofu-version) }} - skip: ${{ steps.config.outputs.opentofu-version == '' || steps.config.outputs.opentofu-version == 'null' }} - suzuki-shunsuke/tfcmt: v4.11.0 - terraform-docs/terraform-docs: v0.18.0 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - if: ${{ steps.config.outputs.aws-region != '' && - steps.config.outputs.aws-region != 'null' && - steps.config.outputs.terraform-apply-role != '' && - steps.config.outputs.terraform-apply-role != 'null' }} - with: - aws-region: ${{ steps.config.outputs.aws-region }} - role-to-assume: ${{ steps.config.outputs.terraform-apply-role }} - role-session-name: "atmos-terraform-apply-gitops" - mask-aws-account-id: "no" - - name: Get atmos settings - uses: cloudposse/github-action-atmos-get-setting@v1 - id: component + id: atmos-settings + uses: cloudposse/github-action-atmos-get-setting@v2 with: settings: | - component: ${{ inputs.component }} @@ -146,6 +97,102 @@ runs: stack: ${{ inputs.stack }} settingsPath: command outputPath: command + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.opentofu-version + outputPath: opentofu-version + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.terraform-version + outputPath: terraform-version + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.infracost-enabled + outputPath: enable-infracost + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.role.plan + outputPath: terraform-plan-role + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.role.apply + outputPath: terraform-apply-role + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.region + outputPath: aws-region + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.role + outputPath: terraform-state-role + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.table + outputPath: terraform-state-table + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.bucket + outputPath: terraform-state-bucket + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.plan-repository-type + outputPath: plan-repository-type + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.blob-account-name + outputPath: blob-account-name + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.blob-container-name + outputPath: blob-container-name + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.metadata-repository-type + outputPath: metadata-repository-type + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.cosmos-container-name + outputPath: cosmos-container-name + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.cosmos-database-name + outputPath: cosmos-database-name + - component: ${{ inputs.component }} + stack: ${{ inputs.stack }} + settingsPath: settings.integrations.github.gitops.artifact-storage.cosmos-endpoint + outputPath: cosmos-endpoint + + - name: Install Terraform + if: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-version != '' && fromJson(steps.atmos-settings.outputs.settings).terraform-version != 'null' }} + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-version }} + terraform_wrapper: false + + - name: Install Dependencies + uses: cloudposse-github-actions/install-gh-releases@v1 + with: + cache: true + config: |- + opentofu/opentofu: + tag: ${{ startsWith(fromJson(steps.atmos-settings.outputs.settings).opentofu-version, 'v') && fromJson(steps.atmos-settings.outputs.settings).opentofu-version || format('v{0}', fromJson(steps.atmos-settings.outputs.settings).opentofu-version) }} + skip: ${{ fromJson(steps.atmos-settings.outputs.settings).opentofu-version == '' || fromJson(steps.atmos-settings.outputs.settings).opentofu-version == 'null' }} + suzuki-shunsuke/tfcmt: v4.14.0 + terraform-docs/terraform-docs: v0.18.0 + + - name: Configure Plan AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + if: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' && + fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && + fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && + fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != '' && + fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != 'null' }} + }} + with: + aws-region: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region }} + role-to-assume: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role }} + role-session-name: "atmos-terraform-apply-gitops" + mask-aws-account-id: "no" - name: Define Job Control State Variables shell: bash @@ -157,7 +204,7 @@ runs: - name: Check If GitHub Actions is Enabled For Component shell: bash run: | - if [[ "${{ fromJson(steps.component.outputs.settings).enabled }}" == "true" ]]; then + if [[ "${{ fromJson(steps.atmos-settings.outputs.settings).enabled }}" == "true" ]]; then echo "ACTIONS_ENABLED=true" >> $GITHUB_ENV else echo "ACTIONS_ENABLED=false" >> $GITHUB_ENV @@ -168,7 +215,7 @@ runs: shell: bash run: |- # Set ATMOS_BASE_PATH allow `cloudposse/utils` provider to read atmos config from the correct path - ATMOS_BASE_PATH="${{ fromJson(steps.component.outputs.settings).base-path }}" + ATMOS_BASE_PATH="${{ fromJson(steps.atmos-settings.outputs.settings).base-path }}" echo "ATMOS_BASE_PATH=$(realpath ${ATMOS_BASE_PATH:-./})" >> $GITHUB_ENV - name: Define Job Variables @@ -177,10 +224,10 @@ runs: shell: bash run: | STACK_NAME=$(echo "${{ inputs.stack }}" | sed 's#/#_#g') - COMPONENT_PATH=$( realpath ${{ fromJson(steps.component.outputs.settings).component-path }}) + COMPONENT_PATH=$( realpath ${{ fromJson(steps.atmos-settings.outputs.settings).component-path }}) COMPONENT_NAME=$(echo "${{ inputs.component }}" | sed 's#/#_#g') COMPONENT_SLUG="$STACK_NAME-$COMPONENT_NAME" - COMPONENT_CACHE_KEY=$(basename "${{ fromJson(steps.component.outputs.settings).component-path }}") + COMPONENT_CACHE_KEY=$(basename "${{ fromJson(steps.atmos-settings.outputs.settings).component-path }}") PLAN_FILE="${COMPONENT_PATH}/$COMPONENT_SLUG-${{ inputs.sha }}.planfile" LOCK_FILE="${COMPONENT_PATH}/.terraform.lock.hcl" @@ -192,129 +239,72 @@ runs: echo "plan_file=$PLAN_FILE" >> $GITHUB_OUTPUT echo "lock_file=$LOCK_FILE" >> $GITHUB_OUTPUT - - name: Configure State AWS Credentials - if: ${{ env.ACTIONS_ENABLED == 'true' && - steps.config.outputs.aws-region != '' && - steps.config.outputs.aws-region != 'null' && - steps.config.outputs.terraform-state-role != '' && - steps.config.outputs.terraform-state-role != 'null' }} + - name: Configure Plan AWS Credentials uses: aws-actions/configure-aws-credentials@v4 + if: ${{ env.ACTIONS_ENABLED == 'true' && + fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' && + fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && + fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && + fromJson(steps.atmos-settings.outputs.settings).terraform-state-role != '' && + fromJson(steps.atmos-settings.outputs.settings).terraform-state-role != 'null' }} + }} with: - aws-region: ${{ steps.config.outputs.aws-region }} - role-to-assume: ${{ steps.config.outputs.terraform-state-role }} + aws-region: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region }} + role-to-assume: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-state-role }} role-session-name: "atmos-terraform-state-gitops" mask-aws-account-id: "no" - - name: Retrieve Plan (AWS) - if: ${{ env.ACTIONS_ENABLED == 'true' && - steps.config.outputs.terraform-state-table != '' && - steps.config.outputs.terraform-state-table != 'null' && - steps.config.outputs.terraform-state-bucket != '' && - steps.config.outputs.terraform-state-bucket != 'null' }} + - name: Retrieve Plan + if: ${{ env.ACTIONS_ENABLED == 'true' }} uses: cloudposse/github-action-terraform-plan-storage@v1 continue-on-error: true with: action: getPlan - planPath: ${{ steps.vars.outputs.plan_file }} - commitSHA: ${{ inputs.sha }} - component: ${{ inputs.component }} - stack: ${{ inputs.stack }} - tableName: ${{ steps.config.outputs.terraform-state-table }} - bucketName: ${{ steps.config.outputs.terraform-state-bucket }} - - - name: Retrieve Lockfile (AWS) - if: ${{ env.ACTIONS_ENABLED == 'true' && - steps.config.outputs.terraform-state-table != '' && - steps.config.outputs.terraform-state-table != 'null' && - steps.config.outputs.terraform-state-bucket != '' && - steps.config.outputs.terraform-state-bucket != 'null' }} - uses: cloudposse/github-action-terraform-plan-storage@v1 - continue-on-error: true - with: - action: getPlan - planPath: ${{ steps.vars.outputs.lock_file }} commitSHA: ${{ inputs.sha }} - component: ${{ inputs.component }} - stack: "${{ inputs.stack }}-lockfile" - tableName: ${{ steps.config.outputs.terraform-state-table }} - bucketName: ${{ steps.config.outputs.terraform-state-bucket }} - - - name: Retrieve Plan (Azure) - if: ${{ env.ACTIONS_ENABLED == 'true' && - steps.config.outputs.plan-repository-type != '' && - steps.config.outputs.plan-repository-type != 'null' && - steps.config.outputs.blob-account-name != '' && - steps.config.outputs.blob-account-name != 'null' && - steps.config.outputs.blob-container-name != '' && - steps.config.outputs.blob-container-name != 'null' && - steps.config.outputs.metadata-repository-type != '' && - steps.config.outputs.metadata-repository-type != 'null' && - steps.config.outputs.cosmos-container-name != '' && - steps.config.outputs.cosmos-container-name != 'null' && - steps.config.outputs.cosmos-database-name != '' && - steps.config.outputs.cosmos-database-name != 'null' && - steps.config.outputs.cosmos-endpoint != '' && - steps.config.outputs.cosmos-endpoint != 'null' }} - uses: cloudposse/github-action-terraform-plan-storage@v1 - continue-on-error: true - with: - action: getPlan planPath: ${{ steps.vars.outputs.plan_file }} - commitSHA: ${{ inputs.sha }} component: ${{ inputs.component }} stack: ${{ inputs.stack }} - # Azure settings - planRepositoryType: ${{ steps.config.outputs.plan-repository-type }} - blobAccountName: ${{ steps.config.outputs.blob-account-name }} - blobContainerName: ${{ steps.config.outputs.blob-container-name }} - metadataRepositoryType: ${{ steps.config.outputs.metadata-repository-type }} - cosmosContainerName: ${{ steps.config.outputs.cosmos-container-name }} - cosmosDatabaseName: ${{ steps.config.outputs.cosmos-database-name }} - cosmosEndpoint: ${{ steps.config.outputs.cosmos-endpoint }} - - - name: Retrieve Lockfile (Azure) - if: ${{ env.ACTIONS_ENABLED == 'true' && - steps.config.outputs.plan-repository-type != '' && - steps.config.outputs.plan-repository-type != 'null' && - steps.config.outputs.blob-account-name != '' && - steps.config.outputs.blob-account-name != 'null' && - steps.config.outputs.blob-container-name != '' && - steps.config.outputs.blob-container-name != 'null' && - steps.config.outputs.metadata-repository-type != '' && - steps.config.outputs.metadata-repository-type != 'null' && - steps.config.outputs.cosmos-container-name != '' && - steps.config.outputs.cosmos-container-name != 'null' && - steps.config.outputs.cosmos-database-name != '' && - steps.config.outputs.cosmos-database-name != 'null' && - steps.config.outputs.cosmos-endpoint != '' && - steps.config.outputs.cosmos-endpoint != 'null' }} + planRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type }} + blobAccountName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-account-name }} + blobContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-container-name }} + metadataRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).metadata-repository-type }} + cosmosContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-container-name }} + cosmosDatabaseName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-database-name }} + cosmosEndpoint: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-endpoint }} + tableName: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-state-table }} + bucketName: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-state-bucket }} + + - name: Retrieve Lockfile + if: ${{ env.ACTIONS_ENABLED == 'true' }} uses: cloudposse/github-action-terraform-plan-storage@v1 continue-on-error: true with: action: getPlan - planPath: ${{ steps.vars.outputs.lock_file }} commitSHA: ${{ inputs.sha }} + planPath: ${{ steps.vars.outputs.lock_file }} component: ${{ inputs.component }} stack: "${{ inputs.stack }}-lockfile" - # Azure settings - planRepositoryType: ${{ steps.config.outputs.plan-repository-type }} - blobAccountName: ${{ steps.config.outputs.blob-account-name }} - blobContainerName: ${{ steps.config.outputs.blob-container-name }} - metadataRepositoryType: ${{ steps.config.outputs.metadata-repository-type }} - cosmosContainerName: ${{ steps.config.outputs.cosmos-container-name }} - cosmosDatabaseName: ${{ steps.config.outputs.cosmos-database-name }} - cosmosEndpoint: ${{ steps.config.outputs.cosmos-endpoint }} - - - name: Configure AWS Credentials - if: ${{ env.ACTIONS_ENABLED == 'true' && - steps.config.outputs.aws-region != '' && - steps.config.outputs.aws-region != 'null' && - steps.config.outputs.terraform-apply-role != '' && - steps.config.outputs.terraform-apply-role != 'null' }} - uses: aws-actions/configure-aws-credentials@v4.0.2 + planRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type }} + blobAccountName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-account-name }} + blobContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-container-name }} + metadataRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).metadata-repository-type }} + cosmosContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-container-name }} + cosmosDatabaseName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-database-name }} + cosmosEndpoint: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-endpoint }} + tableName: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-state-table }} + bucketName: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-state-bucket }} + + - name: Configure Plan AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + if: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' && + fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && + fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && + fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != '' && + fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != 'null' }} + }} with: - aws-region: ${{ steps.config.outputs.aws-region }} - role-to-assume: ${{ steps.config.outputs.terraform-apply-role }} + aws-region: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region }} + role-to-assume: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role }} role-session-name: "atmos-terraform-apply-gitops" mask-aws-account-id: "no" @@ -322,7 +312,7 @@ runs: if: env.ACTIONS_ENABLED == 'true' shell: bash run: | - if [[ "${{ steps.config.outputs.enable-infracost }}" == "true" ]]; then + if [[ "${{ fromJson(steps.atmos-settings.outputs.settings).enable-infracost }}" == "true" ]]; then echo "INFRACOST_ENABLED=true" >> $GITHUB_ENV else echo "INFRACOST_ENABLED=false" >> $GITHUB_ENV @@ -335,11 +325,11 @@ runs: api-key: ${{ inputs.infracost-api-key }} - name: Convert PLANFILE to JSON - if: ${{ steps.config.outputs.enable-infracost == 'true' && steps.atmos-plan.outputs.changes == 'true' }} + if: ${{ fromJson(steps.atmos-settings.outputs.settings).enable-infracost == 'true' && steps.atmos-plan.outputs.changes == 'true' }} shell: bash working-directory: ${{ steps.vars.outputs.component_path }} run: | - ${{ fromJson(steps.component.outputs.settings).command }} show -json "${{ steps.vars.outputs.plan_file }}" > "${{ steps.vars.outputs.plan_file }}.json" + ${{ fromJson(steps.atmos-settings.outputs.settings).command }} show -json "${{ steps.vars.outputs.plan_file }}" > "${{ steps.vars.outputs.plan_file }}.json" - name: Generate Infracost Diff if: env.INFRACOST_ENABLED == 'true' @@ -369,7 +359,7 @@ runs: id: infracost-diff shell: bash run: | - if [[ "${{ steps.config.outputs.enable-infracost }}" == "true" ]]; then + if [[ "${{ fromJson(steps.atmos-settings.outputs.settings).enable-infracost }}" == "true" ]]; then INFRACOST_DIFF_TOTAL_MONTHLY_COST=$(cat /tmp/infracost.json | jq --raw-output .diffTotalMonthlyCost) INFRACOST_DETAILS_DIFF_BREAKDOWN="$(cat /tmp/infracost.txt | base64 --wrap 0)" else @@ -383,7 +373,7 @@ runs: - name: Cache .terraform id: cache uses: actions/cache@v4 - if: ${{ fromJson(steps.component.outputs.settings).enabled }} + if: ${{ fromJson(steps.atmos-settings.outputs.settings).enabled }} with: path: | ${{ steps.vars.outputs.component_path }}/.terraform diff --git a/tests/opentofu/atmos.yaml b/tests/opentofu/atmos.yaml index 39406d9..4686746 100644 --- a/tests/opentofu/atmos.yaml +++ b/tests/opentofu/atmos.yaml @@ -73,6 +73,8 @@ integrations: bucket: __STORAGE_BUCKET__ table: __STORAGE_TABLE__ role: __STORAGE_ROLE__ + plan-repository-type: s3 + metadata-repository-type: dynamo role: plan: __PLAN_ROLE__ apply: __APPLY_ROLE__ diff --git a/tests/terraform/atmos.yaml b/tests/terraform/atmos.yaml index 24c55f2..e4037b0 100644 --- a/tests/terraform/atmos.yaml +++ b/tests/terraform/atmos.yaml @@ -69,10 +69,10 @@ integrations: bucket: __STORAGE_BUCKET__ table: __STORAGE_TABLE__ role: __STORAGE_ROLE__ - plan-repository-type: + plan-repository-type: s3 + metadata-repository-type: dynamo blob-account-name: blob-container-name: - metadata-repository-type: cosmos-container-name: cosmos-database-name: cosmos-endpoint: From 77eb9aaf712175306b1773c23a297d766d4b886f Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 28 Nov 2024 23:12:41 +0100 Subject: [PATCH 07/15] Reconsile with https://github.com/cloudposse/github-action-atmos-terraform-plan/pull/92 --- action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/action.yml b/action.yml index 8b74d56..58aa818 100644 --- a/action.yml +++ b/action.yml @@ -187,7 +187,6 @@ runs: fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != '' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != 'null' }} - }} with: aws-region: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region }} role-to-assume: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role }} @@ -247,7 +246,6 @@ runs: fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && fromJson(steps.atmos-settings.outputs.settings).terraform-state-role != '' && fromJson(steps.atmos-settings.outputs.settings).terraform-state-role != 'null' }} - }} with: aws-region: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region }} role-to-assume: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-state-role }} @@ -301,7 +299,6 @@ runs: fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != '' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != 'null' }} - }} with: aws-region: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region }} role-to-assume: ${{ fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role }} From 1cf287a60be77f8b15c60a5aecbdad58b5277634 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Thu, 28 Nov 2024 23:19:35 +0100 Subject: [PATCH 08/15] Reconsile with https://github.com/cloudposse/github-action-atmos-terraform-plan/pull/92 --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c082b53..ec49b1d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -42,7 +42,7 @@ jobs: sed -i -e 's#__APPLY_ROLE__#${{ secrets.TERRAFORM_APPLY_ROLE }}#g' ${{ runner.temp }}/atmos.yaml - name: Plan Atmos Component - uses: cloudposse/github-action-atmos-terraform-plan@v2 + uses: cloudposse/github-action-atmos-terraform-plan@add-cache-and-azure with: component: "foobar" stack: "plat-ue2-sandbox" @@ -64,5 +64,5 @@ jobs: component: "foobar" stack: "plat-ue2-sandbox" atmos-config-path: ${{ runner.temp }} - atmos-version: 1.81.0 + atmos-version: 1.99.0 debug: true From 6ee604e19b52bf5e1ba9f412b7f8aac41b8da1a4 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Fri, 29 Nov 2024 15:30:54 +0100 Subject: [PATCH 09/15] Define default artifacts storage to s3 --- action.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/action.yml b/action.yml index 58aa818..68f3eaf 100644 --- a/action.yml +++ b/action.yml @@ -182,8 +182,7 @@ runs: - name: Configure Plan AWS Credentials uses: aws-actions/configure-aws-credentials@v4 - if: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' && - fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && + if: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != '' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != 'null' }} @@ -240,8 +239,10 @@ runs: - name: Configure Plan AWS Credentials uses: aws-actions/configure-aws-credentials@v4 - if: ${{ env.ACTIONS_ENABLED == 'true' && - fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' && + if: ${{ ( fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' || + fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == '' || + fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 'null' ) && + env.ACTIONS_ENABLED == 'true' && fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && fromJson(steps.atmos-settings.outputs.settings).terraform-state-role != '' && @@ -262,10 +263,10 @@ runs: planPath: ${{ steps.vars.outputs.plan_file }} component: ${{ inputs.component }} stack: ${{ inputs.stack }} - planRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type }} + planRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type || 's3' }} + metadataRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).metadata-repository-type || 'dynamo' }} blobAccountName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-account-name }} blobContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-container-name }} - metadataRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).metadata-repository-type }} cosmosContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-container-name }} cosmosDatabaseName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-database-name }} cosmosEndpoint: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-endpoint }} @@ -282,10 +283,10 @@ runs: planPath: ${{ steps.vars.outputs.lock_file }} component: ${{ inputs.component }} stack: "${{ inputs.stack }}-lockfile" - planRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type }} + planRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type || 's3' }} + metadataRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).metadata-repository-type || 'dynamo' }} blobAccountName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-account-name }} blobContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).blob-container-name }} - metadataRepositoryType: ${{ fromJson(steps.atmos-settings.outputs.settings).metadata-repository-type }} cosmosContainerName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-container-name }} cosmosDatabaseName: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-database-name }} cosmosEndpoint: ${{ fromJson(steps.atmos-settings.outputs.settings).cosmos-endpoint }} @@ -294,8 +295,7 @@ runs: - name: Configure Plan AWS Credentials uses: aws-actions/configure-aws-credentials@v4 - if: ${{ fromJson(steps.atmos-settings.outputs.settings).plan-repository-type == 's3' && - fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && + if: ${{ fromJson(steps.atmos-settings.outputs.settings).aws-region != '' && fromJson(steps.atmos-settings.outputs.settings).aws-region != 'null' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != '' && fromJson(steps.atmos-settings.outputs.settings).terraform-apply-role != 'null' }} From a7c86abdb5aba74d1cd0d7ca78c8d181c3c97385 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Fri, 29 Nov 2024 20:06:33 +0300 Subject: [PATCH 10/15] clean environment file from .terraform cache to avoid workspace select errors --- action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/action.yml b/action.yml index 68f3eaf..02238a9 100644 --- a/action.yml +++ b/action.yml @@ -383,6 +383,9 @@ runs: working-directory: ${{ steps.vars.outputs.component_path }} run: | set +e + + # Remove the environment file from the cache to avoid conflicts with workspace select + rm -f ./${{ steps.vars.outputs.component_path }}/.terraform/environment TERRAFORM_OUTPUT_FILE="./terraform-${GITHUB_RUN_ID}-output.txt" From 98dd26a709d49d67576e327c1fb71ea10ed510c4 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Fri, 29 Nov 2024 20:54:27 +0300 Subject: [PATCH 11/15] Update README.yaml --- README.yaml | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/README.yaml b/README.yaml index 18967b0..3087aa0 100644 --- a/README.yaml +++ b/README.yaml @@ -62,7 +62,9 @@ usage: |- ### Config > [!IMPORTANT] - > **Please note!** This GitHub Action only works with `atmos >= 1.63.0`. If you are using `atmos < 1.63.0` please use `v1` version of this action. + > **Please note!** This GitHub Action only works with `atmos >= 1.99.0`. + > If you are using `atmos >= 1.63.0, < 1.99.0` please use `v2` version of this action. + > If you are using `atmos < 1.63.0` please use `v1` version of this action. The action expects the atmos configuration file `atmos.yaml` to be present in the repository. @@ -120,6 +122,32 @@ usage: |- group-by: .stack_slug | split("-") | [.[0], .[2]] | join("-") ``` + ### Stack level configuration + + > [!IMPORTANT] + > Wherever it is possible to specify `integration.github.gitops` on stack level + > it is required to define default values in `atmos.yaml` + + It is possible to override integration settings on a stack level by defining `settings.integrations`. + + ```yaml + components: + terraform: + foobar: + settings: + integrations: + github: + gitops: + artifact-storage: + bucket: cptest-plat-ue2-auto-gitops + table: cptest-plat-ue2-auto-gitops-plan-storage + role: arn:aws:iam::xxxxxxxxxxxx:role/cptest-plat-ue2-auto-gitops-gha + role: + # Set `plan` empty if you don't want to assume IAM role before terraform plan + plan: arn:aws:iam::yyyyyyyyyyyy:role/cptest-plat-gbl-identity-gitops + apply: arn:aws:iam::yyyyyyyyyyyy:role/cptest-plat-gbl-identity-gitops + ``` + ### Support OpenTofu This action supports [OpenTofu](https://opentofu.org/). @@ -184,6 +212,17 @@ usage: |- atmos-config-path: ./rootfs/usr/local/etc/atmos/ ``` + ### Migrating from `v2` to `v3` + + The notable changes in `v3` are: + + - `v3` works only with `atmos >= 1.99.0` + - `v3` support azure plan and metadata storage + - `v3` supports stack level integration gitops settings + - `v3` allow to skip internal checkout with `skip-checkout` input + + The only required migration step is updating atmos version to `>= 1.99.0` + ### Migrating from `v1` to `v2` The notable changes in `v2` are: From 1a4f88465de0bbc6eb57ffbb7d299a06901bf8ec Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Fri, 29 Nov 2024 18:55:29 +0100 Subject: [PATCH 12/15] Update readme --- README.md | 78 +++++++++++++++++++++++++++++++++++++++++-- docs/github-action.md | 3 +- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7562d11..101c913 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,22 @@ For Cloud Posse documentation on setting up GitHub OIDC, see our [`github-oidc-p In order to retrieve Terraform Plan Files (not to be confused with Terraform State files, e.g. `tfstate`), we configure an S3 Bucket to store plan files and a DynamoDB table to track plan metadata. Both need to be deployed before running this action. For more on setting up those components, see the [`gitops` component](https://docs.cloudposse.com/components/library/aws/gitops/). This action will then use the [github-action-terraform-plan-storage](https://github.com/cloudposse/github-action-terraform-plan-storage) action to update these resources. + ### Config +> [!IMPORTANT] +> **Please note!** This GitHub Action only works with `atmos >= 1.99.0`. +> If you are using `atmos >= 1.63.0, < 1.99.0` please use `v2` version of this action. +> If you are using `atmos < 1.63.0` please use `v1` version of this action. + The action expects the atmos configuration file `atmos.yaml` to be present in the repository. + +The action supports AWS and Azure to store Terraform plan files. +You can read more about plan storage in the [cloudposse/github-action-terraform-plan-storage](https://github.com/cloudposse/github-action-terraform-plan-storage?tab=readme-ov-file#aws-default) documentation. +Depends of cloud provider the following fields should be set in the `atmos.yaml`: + +#### AWS + The config should have the following structure: ```yaml @@ -85,14 +98,63 @@ integrations: role: arn:aws:iam::xxxxxxxxxxxx:role/cptest-core-ue2-auto-gitops-gha role: plan: arn:aws:iam::yyyyyyyyyyyy:role/cptest-core-gbl-identity-gitops + # Set `apply` empty if you don't want to assume IAM role before terraform apply apply: arn:aws:iam::yyyyyyyyyyyy:role/cptest-core-gbl-identity-gitops matrix: sort-by: .stack_slug group-by: .stack_slug | split("-") | [.[0], .[2]] | join("-") ``` +#### Azure + +The config should have the following structure: + +```yaml +integrations: + github: + gitops: + opentofu-version: 1.7.3 + terraform-version: 1.5.2 + infracost-enabled: false + artifact-storage: + plan-repository-type: azureblob + blob-account-name: tfplans + blob-container-name: plans + metadata-repository-type: cosmos + cosmos-container-name: terraform-plan-storage + cosmos-database-name: terraform-plan-storage + cosmos-endpoint: "https://my-cosmo-account.documents.azure.com:443/" + # We remove the `role` section as it is AWS specific + matrix: + sort-by: .stack_slug + group-by: .stack_slug | split("-") | [.[0], .[2]] | join("-") +``` + +### Stack level configuration + > [!IMPORTANT] -> **Please note!** This GitHub Action only works with `atmos >= 1.63.0`. If you are using `atmos < 1.63.0` please use `v1` version of this action. +> Wherever it is possible to specify `integration.github.gitops` on stack level +> it is required to define default values in `atmos.yaml` + +It is possible to override integration settings on a stack level by defining `settings.integrations`. + +```yaml +components: + terraform: + foobar: + settings: + integrations: + github: + gitops: + artifact-storage: + bucket: cptest-plat-ue2-auto-gitops + table: cptest-plat-ue2-auto-gitops-plan-storage + role: arn:aws:iam::xxxxxxxxxxxx:role/cptest-plat-ue2-auto-gitops-gha + role: + # Set `plan` empty if you don't want to assume IAM role before terraform plan + plan: arn:aws:iam::yyyyyyyyyyyy:role/cptest-plat-gbl-identity-gitops + apply: arn:aws:iam::yyyyyyyyyyyy:role/cptest-plat-gbl-identity-gitops +``` ### Support OpenTofu @@ -158,6 +220,17 @@ We recommend combining this action with the [`affected-stacks`](https://atmos.to atmos-config-path: ./rootfs/usr/local/etc/atmos/ ``` +### Migrating from `v2` to `v3` + +The notable changes in `v3` are: + +- `v3` works only with `atmos >= 1.99.0` +- `v3` support azure plan and metadata storage +- `v3` supports stack level integration gitops settings +- `v3` allow to skip internal checkout with `skip-checkout` input + +The only required migration step is updating atmos version to `>= 1.99.0` + ### Migrating from `v1` to `v2` The notable changes in `v2` are: @@ -322,13 +395,14 @@ Which would produce the same behavior as in `v0`, doing this: | Name | Description | Default | Required | |------|-------------|---------|----------| | atmos-config-path | The path to the atmos.yaml file | N/A | true | -| atmos-version | The version of atmos to install | >= 1.63.0 | false | +| atmos-version | The version of atmos to install | >= 1.99.0 | false | | branding-logo-image | Branding logo image url | https://cloudposse.com/logo-300x69.svg | false | | branding-logo-url | Branding logo url | https://cloudposse.com/ | false | | component | The name of the component to apply. | N/A | true | | debug | Enable action debug mode. Default: 'false' | false | false | | infracost-api-key | Infracost API key | N/A | false | | sha | Commit SHA to apply. Default: github.sha | ${{ github.event.pull\_request.head.sha }} | true | +| skip-checkout | Disable actions/checkout. Useful for when the checkout happens in a previous step and file are modified outside of git through other actions | false | false | | stack | The stack name for the given component. | N/A | true | | token | Used to pull node distributions for Atmos from Cloud Posse's GitHub repository. Since there's a default, this is typically not supplied by the user. When running this action on github.com, the default value is sufficient. When running on GHES, you can pass a personal access token for github.com if you are experiencing rate limiting. | ${{ github.server\_url == 'https://github.com' && github.token \|\| '' }} | false | diff --git a/docs/github-action.md b/docs/github-action.md index c1410b3..09e2968 100644 --- a/docs/github-action.md +++ b/docs/github-action.md @@ -5,13 +5,14 @@ | Name | Description | Default | Required | |------|-------------|---------|----------| | atmos-config-path | The path to the atmos.yaml file | N/A | true | -| atmos-version | The version of atmos to install | >= 1.63.0 | false | +| atmos-version | The version of atmos to install | >= 1.99.0 | false | | branding-logo-image | Branding logo image url | https://cloudposse.com/logo-300x69.svg | false | | branding-logo-url | Branding logo url | https://cloudposse.com/ | false | | component | The name of the component to apply. | N/A | true | | debug | Enable action debug mode. Default: 'false' | false | false | | infracost-api-key | Infracost API key | N/A | false | | sha | Commit SHA to apply. Default: github.sha | ${{ github.event.pull\_request.head.sha }} | true | +| skip-checkout | Disable actions/checkout. Useful for when the checkout happens in a previous step and file are modified outside of git through other actions | false | false | | stack | The stack name for the given component. | N/A | true | | token | Used to pull node distributions for Atmos from Cloud Posse's GitHub repository. Since there's a default, this is typically not supplied by the user. When running this action on github.com, the default value is sufficient. When running on GHES, you can pass a personal access token for github.com if you are experiencing rate limiting. | ${{ github.server\_url == 'https://github.com' && github.token \|\| '' }} | false | From 3e7024b7d21d22a2399c9e378b4b2ce7112dc3f7 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Mon, 2 Dec 2024 17:04:00 +0300 Subject: [PATCH 13/15] Update action.yml --- action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index 02238a9..47ea926 100644 --- a/action.yml +++ b/action.yml @@ -255,7 +255,7 @@ runs: - name: Retrieve Plan if: ${{ env.ACTIONS_ENABLED == 'true' }} - uses: cloudposse/github-action-terraform-plan-storage@v1 + uses: cloudposse/github-action-terraform-plan-storage@v2 continue-on-error: true with: action: getPlan @@ -275,7 +275,7 @@ runs: - name: Retrieve Lockfile if: ${{ env.ACTIONS_ENABLED == 'true' }} - uses: cloudposse/github-action-terraform-plan-storage@v1 + uses: cloudposse/github-action-terraform-plan-storage@v2 continue-on-error: true with: action: getPlan @@ -385,7 +385,7 @@ runs: set +e # Remove the environment file from the cache to avoid conflicts with workspace select - rm -f ./${{ steps.vars.outputs.component_path }}/.terraform/environment + rm -f ./.terraform/environment TERRAFORM_OUTPUT_FILE="./terraform-${GITHUB_RUN_ID}-output.txt" From 5905251c55148702d35578b22e36ad190f722cea Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Mon, 2 Dec 2024 17:09:00 +0300 Subject: [PATCH 14/15] Update action.yml --- action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 47ea926..aa9e373 100644 --- a/action.yml +++ b/action.yml @@ -255,7 +255,7 @@ runs: - name: Retrieve Plan if: ${{ env.ACTIONS_ENABLED == 'true' }} - uses: cloudposse/github-action-terraform-plan-storage@v2 + uses: cloudposse/github-action-terraform-plan-storage@v1 continue-on-error: true with: action: getPlan @@ -275,7 +275,7 @@ runs: - name: Retrieve Lockfile if: ${{ env.ACTIONS_ENABLED == 'true' }} - uses: cloudposse/github-action-terraform-plan-storage@v2 + uses: cloudposse/github-action-terraform-plan-storage@v1 continue-on-error: true with: action: getPlan From 660a01b2d812605a98e058b8b744ce1907d9eec7 Mon Sep 17 00:00:00 2001 From: Igor Rodionov Date: Mon, 2 Dec 2024 21:08:08 +0300 Subject: [PATCH 15/15] Update integration-tests.yml --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ec49b1d..03fe32e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -42,7 +42,7 @@ jobs: sed -i -e 's#__APPLY_ROLE__#${{ secrets.TERRAFORM_APPLY_ROLE }}#g' ${{ runner.temp }}/atmos.yaml - name: Plan Atmos Component - uses: cloudposse/github-action-atmos-terraform-plan@add-cache-and-azure + uses: cloudposse/github-action-atmos-terraform-plan@v4 with: component: "foobar" stack: "plat-ue2-sandbox"