Merge branch 'DBTP-1503-manual-pipeline' into DBTP-1507-setup-codepip…

…eline-notifications

codebase-pipelines/artifactstore.tf

@@ -31,13 +31,10 @@ data "aws_iam_policy_document" "artifact_store_bucket_policy" {
       type        = "*"
       identifiers = ["*"]
     }
-
     actions = [
-      "s3:*",
+      "s3:*"
     ]
-
     effect = "Deny"
-
     condition {
       test     = "Bool"
       variable = "aws:SecureTransport"
@@ -46,7 +43,24 @@ data "aws_iam_policy_document" "artifact_store_bucket_policy" {
         "false",
       ]
     }
+    resources = [
+      aws_s3_bucket.artifact_store.arn,
+      "${aws_s3_bucket.artifact_store.arn}/*",
+    ]
+  }
 
+  statement {
+    effect = "Allow"
+    principals {
+      type = "AWS"
+      identifiers = [
+        for env in local.pipeline_environments :
+        "arn:aws:iam::${env.account}:role/${var.application}-${env.name}-codebase-pipeline-deploy"
+      ]
+    }
+    actions = [
+      "s3:*"
+    ]
     resources = [
       aws_s3_bucket.artifact_store.arn,
       "${aws_s3_bucket.artifact_store.arn}/*",
@@ -70,7 +84,7 @@ resource "aws_kms_key" "artifact_store_kms_key" {
         "Sid" : "Enable IAM User Permissions",
         "Effect" : "Allow",
         "Principal" : {
-          "AWS" : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
+          "AWS" : [for id in local.deploy_account_ids : "arn:aws:iam::${id}:root"]
         },
         "Action" : "kms:*",
         "Resource" : "*"

codebase-pipelines/buildspec-deploy.yml

@@ -22,7 +22,6 @@ phases:
       - |
         if ! aws ecr describe-images --repository-name "${REPOSITORY_NAME}" --image-ids "imageTag=${IMAGE_TAG}" > /dev/null 2>&1; then
           echo "Error: image tag ${IMAGE_TAG} not found in repository ${REPOSITORY_NAME}"
-          
           exit 1
         fi
 

codebase-pipelines/codebuild.tf

@@ -3,7 +3,7 @@ data "aws_codestarconnections_connection" "github_codestar_connection" {
 }
 
 resource "aws_codebuild_project" "codebase_image_build" {
-  name          = "${var.application}-${var.codebase}-codebase-image-build"
+  name          = "${var.application}-${var.codebase}-codebase-pipeline-image-build"
   description   = "Publish images on push to ${var.repository}"
   build_timeout = 30
   service_role  = aws_iam_role.codebase_image_build.arn
@@ -115,12 +115,11 @@ resource "aws_codebuild_webhook" "codebuild_webhook" {
 }
 
 
-resource "aws_codebuild_project" "codebase_deploy_manifests" {
-  for_each       = local.pipeline_map
-  name           = "${var.application}-${var.codebase}-${each.value.name}-codebase-deploy-manifests"
-  description    = "Create image deploy manifests to deploy services"
-  build_timeout  = 5
-  service_role   = aws_iam_role.codebuild_manifests.arn
+resource "aws_codebuild_project" "codebase_deploy" {
+  name           = "${var.application}-${var.codebase}-codebase-pipeline-deploy"
+  description    = "Deploy specified image tag to specified environment"
+  build_timeout  = 30
+  service_role   = aws_iam_role.codebase_deploy.arn
   encryption_key = aws_kms_key.artifact_store_kms_key.arn
 
   artifacts {
@@ -137,59 +136,36 @@ resource "aws_codebuild_project" "codebase_deploy_manifests" {
     image                       = "aws/codebuild/amazonlinux2-x86_64-standard:5.0"
     type                        = "LINUX_CONTAINER"
     image_pull_credentials_type = "CODEBUILD"
+
+    environment_variable {
+      name  = "ENV_CONFIG"
+      value = jsonencode(local.base_env_config)
+    }
   }
 
   logs_config {
     cloudwatch_logs {
-      group_name  = aws_cloudwatch_log_group.codebase_deploy_manifests.name
-      stream_name = aws_cloudwatch_log_stream.codebase_deploy_manifests.name
+      group_name  = aws_cloudwatch_log_group.codebase_deploy.name
+      stream_name = aws_cloudwatch_log_stream.codebase_deploy.name
     }
   }
 
   source {
-    type = "CODEPIPELINE"
-    buildspec = templatefile("${path.module}/buildspec-manifests.yml", {
-      application = var.application,
-      environments = [
-        for env in each.value.environments : upper(env.name)
-      ],
-      services = local.service_export_names
-    })
+    type      = "CODEPIPELINE"
+    buildspec = file("${path.module}/buildspec-deploy.yml")
   }
 
   tags = local.tags
 }
 
-resource "aws_kms_key" "codebuild_kms_key" {
-  description         = "KMS Key for ${var.application} ${var.codebase} CodeBuild encryption"
-  enable_key_rotation = true
-
-  policy = jsonencode({
-    Statement = [
-      {
-        "Sid" : "Enable IAM User Permissions",
-        "Effect" : "Allow",
-        "Principal" : {
-          "AWS" : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
-        },
-        "Action" : "kms:*",
-        "Resource" : "*"
-      }
-    ]
-    Version = "2012-10-17"
-  })
-
-  tags = local.tags
-}
-
-resource "aws_cloudwatch_log_group" "codebase_deploy_manifests" {
+resource "aws_cloudwatch_log_group" "codebase_deploy" {
   # checkov:skip=CKV_AWS_338:Retains logs for 3 months instead of 1 year
   # checkov:skip=CKV_AWS_158:Log groups encrypted using default encryption key instead of KMS CMK
-  name              = "codebuild/${var.application}-${var.codebase}-codebase-deploy-manifests/log-group"
+  name              = "codebuild/${var.application}-${var.codebase}-codebase-deploy/log-group"
   retention_in_days = 90
 }
 
-resource "aws_cloudwatch_log_stream" "codebase_deploy_manifests" {
-  name           = "codebuild/${var.application}-${var.codebase}-codebase-deploy-manifests/log-stream"
-  log_group_name = aws_cloudwatch_log_group.codebase_deploy_manifests.name
+resource "aws_cloudwatch_log_stream" "codebase_deploy" {
+  name           = "codebuild/${var.application}-${var.codebase}-codebase-deploy/log-stream"
+  log_group_name = aws_cloudwatch_log_group.codebase_deploy.name
 }

codebase-pipelines/codepipeline.tf

@@ -26,43 +26,18 @@ resource "aws_codepipeline" "codebase_pipeline" {
     name = "Source"
 
     action {
-      name             = "Source"
+      name             = "GitCheckout"
       category         = "Source"
       owner            = "AWS"
-      provider         = "ECR"
+      provider         = "CodeStarSourceConnection"
       version          = "1"
-      namespace        = "source_ecr"
-      output_artifacts = ["source_output"]
+      output_artifacts = ["deploy_source"]
 
       configuration = {
-        RepositoryName = local.ecr_name
-        ImageTag       = coalesce(each.value.tag, false) ? "tag-latest" : "branch-${each.value.branch}"
-      }
-    }
-  }
-
-  stage {
-    name = "Create-Deploy-Manifests"
-
-    action {
-      name             = "CreateManifests"
-      category         = "Build"
-      owner            = "AWS"
-      provider         = "CodeBuild"
-      input_artifacts  = ["source_output"]
-      output_artifacts = ["manifest_output"]
-      version          = "1"
-      namespace        = "build_manifest"
-
-      configuration = {
-        ProjectName = "${var.application}-${var.codebase}-${each.value.name}-codebase-deploy-manifests"
-        EnvironmentVariables : jsonencode([
-          { name : "APPLICATION", value : var.application },
-          { name : "ENVIRONMENTS", value : jsonencode([for env in each.value.environments : env.name]) },
-          { name : "SERVICES", value : jsonencode(local.services) },
-          { name : "REPOSITORY_URL", value : "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com/${local.ecr_name}" },
-          { name : "IMAGE_TAG", value : "#{variables.IMAGE_TAG}" }
-        ])
+        ConnectionArn    = data.aws_codestarconnections_connection.github_codestar_connection.arn
+        FullRepositoryId = "${var.repository}-deploy"
+        BranchName       = "main"
+        DetectChanges    = false
       }
     }
   }
@@ -87,17 +62,25 @@ resource "aws_codepipeline" "codebase_pipeline" {
       dynamic "action" {
         for_each = local.service_order_list
         content {
-          name            = action.value.name
-          category        = "Deploy"
-          owner           = "AWS"
-          provider        = "ECS"
-          version         = "1"
-          input_artifacts = ["manifest_output"]
-          run_order       = action.value.order + 1
+          name             = action.value.name
+          category         = "Build"
+          owner            = "AWS"
+          provider         = "CodeBuild"
+          input_artifacts  = ["deploy_source"]
+          output_artifacts = []
+          version          = "1"
+          run_order        = action.value.order + 1
+
           configuration = {
-            ClusterName = "#{build_manifest.CLUSTER_NAME_${upper(stage.value.name)}}"
-            ServiceName = "#{build_manifest.SERVICE_NAME_${upper(stage.value.name)}_${upper(replace(action.value.name, "-", "_"))}}"
-            FileName    = "image-definitions-${action.value.name}.json"
+            ProjectName = aws_codebuild_project.codebase_deploy.name
+            EnvironmentVariables : jsonencode([
+              { name : "APPLICATION", value : var.application },
+              { name : "ENVIRONMENT", value : stage.value.name },
+              { name : "SERVICE", value : action.value.name },
+              { name : "REPOSITORY_URL", value : local.repository_url },
+              { name : "REPOSITORY_NAME", value : local.ecr_name },
+              { name : "IMAGE_TAG", value : "#{variables.IMAGE_TAG}" }
+            ])
           }
         }
       }
@@ -106,3 +89,106 @@ resource "aws_codepipeline" "codebase_pipeline" {
 
   tags = local.tags
 }
+
+
+resource "aws_codepipeline" "manual_release_pipeline" {
+  name           = "${var.application}-${var.codebase}-manual-release-pipeline"
+  role_arn       = aws_iam_role.codebase_deploy_pipeline.arn
+  pipeline_type  = "V2"
+  execution_mode = "QUEUED"
+
+  variable {
+    name          = "IMAGE_TAG"
+    default_value = "NONE"
+    description   = "Tagged image in ECR to deploy"
+  }
+
+  variable {
+    name          = "ENVIRONMENT"
+    default_value = "NONE"
+    description   = "Name of the environment to deploy to"
+  }
+
+  artifact_store {
+    location = aws_s3_bucket.artifact_store.bucket
+    type     = "S3"
+
+    encryption_key {
+      id   = aws_kms_key.artifact_store_kms_key.arn
+      type = "KMS"
+    }
+  }
+
+  stage {
+    name = "Source"
+
+    action {
+      name             = "GitCheckout"
+      category         = "Source"
+      owner            = "AWS"
+      provider         = "CodeStarSourceConnection"
+      version          = "1"
+      output_artifacts = ["deploy_source"]
+
+      configuration = {
+        ConnectionArn    = data.aws_codestarconnections_connection.github_codestar_connection.arn
+        FullRepositoryId = "${var.repository}-deploy"
+        BranchName       = "main"
+        DetectChanges    = false
+      }
+    }
+  }
+
+  stage {
+    name = "Deploy"
+
+    dynamic "action" {
+      for_each = local.service_order_list
+      content {
+        name             = action.value.name
+        category         = "Build"
+        owner            = "AWS"
+        provider         = "CodeBuild"
+        input_artifacts  = ["deploy_source"]
+        output_artifacts = []
+        version          = "1"
+        run_order        = action.value.order + 1
+
+        configuration = {
+          ProjectName = aws_codebuild_project.codebase_deploy.name
+          EnvironmentVariables : jsonencode([
+            { name : "APPLICATION", value : var.application },
+            { name : "ENVIRONMENT", value : "#{variables.ENVIRONMENT}" },
+            { name : "SERVICE", value : action.value.name },
+            { name : "REPOSITORY_URL", value : local.repository_url },
+            { name : "REPOSITORY_NAME", value : local.ecr_name },
+            { name : "IMAGE_TAG", value : "#{variables.IMAGE_TAG}" }
+          ])
+        }
+      }
+    }
+  }
+
+  tags = local.tags
+}
+
+# This is a temporary workaround until automatic stage rollback is implemented in terraform-provider-aws
+# https://github.com/hashicorp/terraform-provider-aws/issues/37244
+resource "terraform_data" "update_pipeline" {
+  provisioner "local-exec" {
+    command = "python ${path.module}/custom_pipeline_update/update_pipeline.py"
+    quiet   = true
+    environment = {
+      PIPELINES = jsonencode(local.pipeline_names)
+    }
+  }
+  triggers_replace = [
+    aws_codepipeline.codebase_pipeline,
+    aws_codepipeline.manual_release_pipeline,
+    file("${path.module}/custom_pipeline_update/update_pipeline.py")
+  ]
+  depends_on = [
+    aws_codepipeline.codebase_pipeline,
+    aws_codepipeline.manual_release_pipeline
+  ]
+}