Develop (#29)

* Add option to set docker machine options, based on repo of @@rsrchboy

* Add option to overwrite instance profile. based on repo of  @@rsrchboy

* Replaced hard coded ami list by a filter

* Replace cache user by an instance profile

* Replace cache user by an instance profile

CHANGELOG.md

@@ -5,14 +5,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
+- Changed: Replaced cache user by a instance profile to access the cache from the build
+- Changed: Update gitlab toml cache section, removed deprecated usages of s3
 - Changed: The variable `amazon_optimized_amis` is removed an replaced by a filter to select the AMI. To use the default of the latest AMI set the filter `ami_filter` to `amzn-ami-hvm-2018.03.0.20180622-x86_64-ebs`.
 - Added: Option to set docker machine options via `docker_machine_optionns`.
 - Added: Several output variables.
 
 
 ## [1.8.0] - 2018-12-30
-- Changed: Updated default docker-machine version to 0.16.0
-- Changed: Updated default gitlab runner to 11.6.0
+- Changed: Update default docker-machine version to 0.16.0
+- Changed: Update default gitlab runner to 11.6.0
 - Added: Configuration parameters for post_build_script, pre_clone_script, request_concurrency and output_limit. #22
 - Added: Configurable docker image for runner #27
 - Added: Add pre/post install user-data snippets for runners #26

README.md

@@ -109,11 +109,11 @@ All variables and defaults:
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | allow_iam_service_linked_role_creation | Attach policy to runner instance to create service linked roles. | string | `true` | no |
-| amazon_optimized_amis | AMI map per region-zone for the gitlab-runner instance AMI. | map | `<map>` | no |
+| ami_filter | AMI filter to select the AMI used to host the gitlab runner agent. By default the pattern `amzn-ami-hvm-2018.03*-x86_64-ebs` is used for the name. Currently Amazon Linux 2 `amzn2-ami-hvm-2.0.????????-x86_64-ebs` looks *not* working for this configuration. | list | `<list>` | no |
+| ami_owners | A list of owners used to select the AMI for the instance. | list | `<list>` | no |
 | aws_region | AWS region. | string | - | yes |
 | cache_bucket_prefix | Prefix for s3 cache bucket name. | string | `` | no |
 | cache_expiration_days | Number of days before cache objects expires. | string | `1` | no |
-| cache_user | User name of the user to create to write and read to the s3 cache. | string | `cache_user` | no |
 | docker_machine_instance_type | Instance type used for the instances hosting docker-machine. | string | `m4.large` | no |
 | docker_machine_options | Additional to set options for docker machien. Each element of the list should be key and value. E.g. '["--amazonec2-zone=a"]' | list | `<list>` | no |
 | docker_machine_spot_price_bid | Spot price bid. | string | `0.04` | no |
@@ -122,7 +122,8 @@ All variables and defaults:
 | enable_cloudwatch_logging | Enable or disable the CloudWatch logging. | string | `1` | no |
 | environment | A name that identifies the environment, will used as prefix and for tagging. | string | - | yes |
 | gitlab_runner_version | Version for the gitlab runner. | string | `11.6.0` | no |
-| instance_role_json | Instance role json to override the default. | string | `` | no |
+| instance_role_json | Instance role json for the runner agent ec2 instance to override the default. | string | `` | no |
+| instance_role_runner_json | Instance role json for the docker machine runners to override the default. | string | `` | no |
 | instance_type | Instance type used for the gitlab-runner. | string | `t2.micro` | no |
 | runners_concurrent | Concurrent value for the runners, will be used in the runner config.toml | string | `10` | no |
 | runners_gitlab_url | URL of the gitlab instance to connect to. | string | - | yes |
@@ -158,8 +159,10 @@ All variables and defaults:
 
 | Name | Description |
 |------|-------------|
+| runner_agent role | ARN of the rule used for the ec2 instance for the GitLab runner agent. |
 | runner_as_group_name | Name of the autoscaling group for the gitlab-runner instance |
 | runner_cache_bucket_arn | ARN of the S3 for the build cache. |
+| runner_role | ARN of the rule used for the docker machine runners. |
 
 ## Example
 

bucket.tf

@@ -23,36 +23,3 @@ resource "aws_s3_bucket" "build_cache" {
     }
   }
 }
-
-resource "aws_iam_user" "cache_user" {
-  name = "${var.cache_user}"
-}
-
-resource "aws_iam_access_key" "cache_user" {
-  user = "${aws_iam_user.cache_user.name}"
-}
-
-data "aws_iam_policy_document" "bucket-policy-doc" {
-  statement {
-    actions = [
-      "s3:PutObject",
-      "s3:PutObjectAcl",
-      "s3:GetObject",
-      "s3:GetObjectAcl",
-    ]
-
-    principals = {
-      type        = "AWS"
-      identifiers = ["${aws_iam_user.cache_user.arn}"]
-    }
-
-    resources = [
-      "${aws_s3_bucket.build_cache.arn}/*",
-    ]
-  }
-}
-
-resource "aws_s3_bucket_policy" "bucket-policy" {
-  bucket = "${aws_s3_bucket.build_cache.id}"
-  policy = "${data.aws_iam_policy_document.bucket-policy-doc.json}"
-}

main.tf

@@ -109,6 +109,9 @@ data "template_file" "runners" {
     runners_spot_price_bid      = "${var.docker_machine_spot_price_bid}"
     runners_security_group_name = "${aws_security_group.docker_machine.name}"
     runners_monitoring          = "${var.runners_monitoring}"
+    runners_instance_profile    = "${aws_iam_instance_profile.runners.name}"
+
+    docker_machine_options = "${length(var.docker_machine_options) == 0 ? "" : local.docker_machine_options_string}"
 
     docker_machine_options = "${length(var.docker_machine_options) == 0 ? "" : local.docker_machine_options_string}"
 
@@ -132,8 +135,6 @@ data "template_file" "runners" {
     runners_pre_clone_script          = "${var.runners_pre_clone_script}"
     runners_request_concurrency       = "${var.runners_request_concurrency}"
     runners_output_limit              = "${var.runners_output_limit}"
-    bucket_user_access_key            = "${aws_iam_access_key.cache_user.id}"
-    bucket_user_secret_key            = "${aws_iam_access_key.cache_user.secret}"
     bucket_name                       = "${aws_s3_bucket.build_cache.bucket}"
   }
 }
@@ -237,3 +238,41 @@ resource "aws_iam_role_policy_attachment" "service_linked_role" {
   role       = "${aws_iam_role.instance.name}"
   policy_arn = "${aws_iam_policy.service_linked_role.arn}"
 }
+
+################################################################################
+### docker machine runner role and policies
+################################################################################
+data "template_file" "runners_role_trust_policy" {
+  template = "${length(var.instance_role_runner_json) > 0 ? var.instance_role_runner_json : file("${path.module}/policies/instance-role-trust-policy.json")}"
+}
+
+resource "aws_iam_role" "runners" {
+  name               = "${var.environment}-runners-role"
+  assume_role_policy = "${data.template_file.runners_role_trust_policy.rendered}"
+}
+
+resource "aws_iam_instance_profile" "runners" {
+  name = "${var.environment}-runners-profile"
+  role = "${aws_iam_role.runners.name}"
+}
+
+data "template_file" "cache_policy" {
+  template = "${file("${path.module}/policies/cache.json")}"
+
+  vars {
+    s3_cache_arn = "${aws_s3_bucket.build_cache.arn}"
+  }
+}
+
+resource "aws_iam_policy" "runners" {
+  name        = "${var.environment}-runners-cache-policy"
+  path        = "/"
+  description = "Policy for Runners."
+
+  policy = "${data.template_file.cache_policy.rendered}"
+}
+
+resource "aws_iam_role_policy_attachment" "runners" {
+  role       = "${aws_iam_role.runners.name}"
+  policy_arn = "${aws_iam_policy.runners.arn}"
+}

outputs.tf

@@ -7,3 +7,13 @@ output "runner_cache_bucket_arn" {
   description = "ARN of the S3 for the build cache."
   value       = "${aws_s3_bucket.build_cache.arn}"
 }
+
+output "runner_agent role" {
+  description = "ARN of the rule used for the ec2 instance for the GitLab runner agent."
+  value       = "${aws_iam_role.instance.arn}"
+}
+
+output "runner_role" {
+  description = "ARN of the rule used for the docker machine runners."
+  value       = "${aws_iam_role.runners.arn}"
+}

template/runner-config.tpl

@@ -19,11 +19,8 @@ check_interval = 0
     disable_cache = false
     volumes = ["/cache"]
     shm_size = 0
-  [runners.cache]
-    Type = "s3"
+  [runners.cache.s3]
     ServerAddress = "s3-${aws_region}.amazonaws.com"
-    AccessKey = "${bucket_user_access_key}"
-    SecretKey = "${bucket_user_secret_key}"
     BucketName = "${bucket_name}"
     Insecure = false
   [runners.machine]
@@ -37,12 +34,13 @@ check_interval = 0
       "amazonec2-vpc-id=${runners_vpc_id}",
       "amazonec2-subnet-id=${runners_subnet_id}",
       "amazonec2-private-address-only=${runners_use_private_address}",
-      "amazonec2-request-spot-instance=true", "amazonec2-spot-price=${runners_spot_price_bid}",
+      "amazonec2-request-spot-instance=true",
+      "amazonec2-spot-price=${runners_spot_price_bid}",
       "amazonec2-security-group=${runners_security_group_name}",
       "amazonec2-tags=environment,${environment}",
       "amazonec2-monitoring=${runners_monitoring}",
-      "amazonec2-root-size=${runners_root_size}",
-      "amazonec2-iam-instance-profile=${runners_iam_instance_profile_name}"
+      "amazonec2-iam-instance-profile=${runners_instance_profile}",
+      "amazonec2-root-size=${runners_root_size}"
       ${docker_machine_options}
     ]
     OffPeakTimezone = "${runners_off_peak_timezone}"

variables.tf

@@ -184,12 +184,6 @@ variable "docker_machine_user" {
   default     = "docker-machine"
 }
 
-variable "cache_user" {
-  description = "User name of the user to create to write and read to the s3 cache."
-  type        = "string"
-  default     = "cache_user"
-}
-
 variable "cache_bucket_prefix" {
   description = "Prefix for s3 cache bucket name."
   type        = "string"
@@ -230,7 +224,13 @@ variable "docker_machine_options" {
 }
 
 variable "instance_role_json" {
-  description = "Instance role json to override the default."
+  description = "Instance role json for the runner agent ec2 instance to override the default."
+  type        = "string"
+  default     = ""
+}
+
+variable "instance_role_runner_json" {
+  description = "Instance role json for the docker machine runners to override the default."
   type        = "string"
   default     = ""
 }