diff --git a/CHANGELOG.md b/CHANGELOG.md index d17ef5f13..1238aea03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +## [2.1.0] - 2019-02-28 +- Bugfix: Shared cache is not working #33 +- Bugfix: Missing documentation makes setup fail #31 +- Added: Docker executor to run a single node runner, thanks to @msvechla + ## [2.0.0] - 2019-01-13 - Changed: Replaced cache user by a instance profile to access the cache from the build - Changed: Update gitlab toml cache section, removed deprecated usages of s3 @@ -84,7 +89,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Update default AMI's to The latest Amazon Linux AMI 2017.09.1 - released on 2018-01-17. - Minor updates in the example -[Unreleased]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/2.0.0...HEAD +[Unreleased]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/2.1.0...HEAD +[2.1.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/2.0.0...2.1.0 [2.0.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/1.8.0...2.0.0 [1.8.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/1.7.0...1.8.0 [1.7.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/1.6.0...1.7.0 diff --git a/README.md b/README.md index bac13027c..193c06a6e 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ This repo contains a terraform module and example to run a [GitLab CI multi runn The setup is based on the blog post: [Auto scale GitLab CI runners and save 90% on EC2 costs](https://about.gitlab.com/2017/11/23/autoscale-ci-runners/) The created runner will have by default a shared cache in S3 and logging is streamed to CloudWatch. The cache in S3 will expire in X days, see configuration. The logging can be disabled. +Besides the auto scaling option (docker+machine executor) the docker executor is supported as wel for a single node. + ## Prerequisites ### Terraform @@ -92,20 +94,21 @@ module "gitlab-runner" { ssh_public_key = "${file("${var.ssh_key_file}")}" vpc_id = "${module.vpc.vpc_id}" - subnet_id_gitlab_runner = "${element(module.vpc.private_subnets, 0)}" + subnet_ids_gitlab_runner = "${module.vpc.private_subnets}" subnet_id_runners = "${element(module.vpc.private_subnets, 0)}" - runners_name = "${var.runner_name}" - runners_gitlab_url = "${var.gitlab_url}" - runners_token = "${var.runner_token}" + runners_name = "${var.runner_name}" + runners_gitlab_url = "${var.gitlab_url}" + runners_token = "${var.runner_token}" + + # Optional + runners_off_peak_timezone = "Europe/Amsterdam" + runners_off_peak_periods = "[\"* * 0-9,17-23 * * mon-fri *\", \"* * * * * sat,sun *\"]" } ``` ## Inputs -All variables and defaults: - - | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | allow_iam_service_linked_role_creation | Attach policy to runner instance to create service linked roles. | string | `true` | no | @@ -114,6 +117,8 @@ All variables and defaults: | aws_region | AWS region. | string | - | yes | | cache_bucket_prefix | Prefix for s3 cache bucket name. | string | `` | no | | cache_expiration_days | Number of days before cache objects expires. | string | `1` | no | +| cache_shared | Enables cache sharing between runners, false by default. | string | `false` | no | +| create_runners_iam_instance_profile | | string | `true` | no | | docker_machine_instance_type | Instance type used for the instances hosting docker-machine. | string | `m4.large` | no | | docker_machine_options | Additional to set options for docker machien. Each element of the list should be key and value. E.g. '["--amazonec2-zone=a"]' | list | `` | no | | docker_machine_spot_price_bid | Spot price bid. | string | `0.04` | no | @@ -132,6 +137,7 @@ All variables and defaults: | runners_idle_time | Idle time of the runners, will be used in the runner config.toml | string | `600` | no | | runners_image | Image to run builds, will be used in the runner config.toml | string | `docker:18.03.1-ce` | no | | runners_limit | Limit for the runners, will be used in the runner config.toml | string | `0` | no | +| runners_machine_iam_instance_profile_name | IAM instance profile name to assign to the spot instance which runs the build. | string | `` | no | | runners_monitoring | Enable detailed cloudwatch monitoring for spot instances. | string | `false` | no | | runners_name | Name of the runner, will be used in the runner config.toml | string | - | yes | | runners_off_peak_idle_count | Off peak idle count of the runners, will be used in the runner config.toml. | string | `0` | no | @@ -148,7 +154,7 @@ All variables and defaults: | runners_token | Token for the runner, will be used in the runner config.toml | string | - | yes | | runners_use_private_address | Restrict runners to use only private address | string | `true` | no | | ssh_public_key | Public SSH key used for the gitlab-runner ec2 instance. | string | - | yes | -| subnet_id_gitlab_runner | Subnet used for hosting the gitlab-runner. | string | - | yes | +| subnet_ids_gitlab_runner | Subnets used for hosting the gitlab-runner. | list | - | yes | | subnet_id_runners | Subnet used to hosts the docker-machine runners. | string | - | yes | | tags | Map of tags that will be added to created resources. By default resources will be taggen with name and environemnt. | map | `` | no | | userdata_post_install | User-data script snippet to insert after gitlab-runner install | string | `` | no | @@ -159,10 +165,9 @@ All variables and defaults: | Name | Description | |------|-------------| -| runner_agent role | ARN of the rule used for the ec2 instance for the GitLab runner agent. | +| runner_agent_role | ARN of the rule used for the ec2 instance for the GitLab runner agent. | | runner_as_group_name | Name of the autoscaling group for the gitlab-runner instance | | runner_cache_bucket_arn | ARN of the S3 for the build cache. | -| runner_role | ARN of the rule used for the docker machine runners. | ## Example diff --git a/examples/runner-default/main.tf b/examples/runner-default/main.tf index c93e3b9d7..f1da0bbbe 100644 --- a/examples/runner-default/main.tf +++ b/examples/runner-default/main.tf @@ -25,9 +25,9 @@ module "runner" { ssh_public_key = "${local_file.public_ssh_key.content}" - vpc_id = "${module.vpc.vpc_id}" - subnet_id_gitlab_runner = "${element(module.vpc.private_subnets, 0)}" - subnet_id_runners = "${element(module.vpc.private_subnets, 0)}" + vpc_id = "${module.vpc.vpc_id}" + subnet_ids_gitlab_runner = "${module.vpc.private_subnets}" + subnet_id_runners = "${element(module.vpc.private_subnets, 0)}" runners_name = "${var.runner_name}" runners_gitlab_url = "${var.gitlab_url}" diff --git a/examples/runner-docker/.terraform-version b/examples/runner-docker/.terraform-version new file mode 100644 index 000000000..b80f98e66 --- /dev/null +++ b/examples/runner-docker/.terraform-version @@ -0,0 +1 @@ +0.11.7 diff --git a/examples/runner-docker/README.md b/examples/runner-docker/README.md new file mode 100644 index 000000000..3c1f309b6 --- /dev/null +++ b/examples/runner-docker/README.md @@ -0,0 +1,6 @@ +# Example - Runner - Docker runner + +Example how create a gitlab runner using the docker executor on a single node, running in a private subnet. + +## Prerequisite +The terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv). If you are not using tfenv please check `.terraform-version` for the tested version. diff --git a/examples/runner-docker/key.tf b/examples/runner-docker/key.tf new file mode 100644 index 000000000..84da24f1f --- /dev/null +++ b/examples/runner-docker/key.tf @@ -0,0 +1,25 @@ +resource "tls_private_key" "ssh" { + algorithm = "RSA" +} + +resource "local_file" "public_ssh_key" { + depends_on = ["tls_private_key.ssh"] + + content = "${tls_private_key.ssh.public_key_openssh}" + filename = "${var.public_ssh_key_filename}" +} + +resource "local_file" "private_ssh_key" { + depends_on = ["tls_private_key.ssh"] + + content = "${tls_private_key.ssh.private_key_pem}" + filename = "${var.private_ssh_key_filename}" +} + +resource "null_resource" "file_permission" { + depends_on = ["local_file.private_ssh_key"] + + provisioner "local-exec" { + command = "${format("chmod 600 %s", var.private_ssh_key_filename)}" + } +} diff --git a/examples/runner-docker/main.tf b/examples/runner-docker/main.tf new file mode 100644 index 000000000..a6360d07d --- /dev/null +++ b/examples/runner-docker/main.tf @@ -0,0 +1,34 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "1.37.0" + + name = "vpc-${var.environment}" + cidr = "10.1.0.0/16" + + azs = ["eu-west-1a"] + public_subnets = ["10.1.101.0/24"] + + tags = { + Environment = "${var.environment}" + } +} + +module "runner" { + source = "../../" + + aws_region = "${var.aws_region}" + environment = "${var.environment}" + + ssh_public_key = "${local_file.public_ssh_key.content}" + + runners_use_private_address = false + + vpc_id = "${module.vpc.vpc_id}" + subnet_ids_gitlab_runner = "${module.vpc.public_subnets}" + subnet_id_runners = "${element(module.vpc.public_subnets, 0)}" + + runners_executor = "docker" + runners_name = "${var.runner_name}" + runners_gitlab_url = "${var.gitlab_url}" + runners_token = "${var.runner_token}" +} diff --git a/examples/runner-docker/providers.tf b/examples/runner-docker/providers.tf new file mode 100644 index 000000000..448ccd70f --- /dev/null +++ b/examples/runner-docker/providers.tf @@ -0,0 +1,20 @@ +provider "aws" { + region = "${var.aws_region}" + version = "1.23" +} + +provider "template" { + version = "1.0" +} + +provider "local" { + version = "1.1" +} + +provider "null" { + version = "1.0" +} + +provider "tls" { + version = "1.1" +} diff --git a/examples/runner-docker/terraform.tfvars b/examples/runner-docker/terraform.tfvars new file mode 100644 index 000000000..6b44a8979 --- /dev/null +++ b/examples/runner-docker/terraform.tfvars @@ -0,0 +1,12 @@ +key_name = "gitlab-runner" + +environment = "runner-docker" + +aws_region = "eu-west-1" + +# Add the following variables: +runner_name = "docker.m3" + +gitlab_url = "https://gitlab.com" + +runner_token = "3939146918cced54ecf1dd08e6b87e" diff --git a/examples/runner-docker/variables.tf b/examples/runner-docker/variables.tf new file mode 100644 index 000000000..bd7286242 --- /dev/null +++ b/examples/runner-docker/variables.tf @@ -0,0 +1,34 @@ +variable "aws_region" { + description = "AWS region." + type = "string" + default = "eu-west-1" +} + +variable "environment" { + description = "A name that indentifies the environment, will used as prefix and for taggin." + default = "ci-runners" + type = "string" +} + +variable "public_ssh_key_filename" { + default = "generated/id_rsa.pub" +} + +variable "private_ssh_key_filename" { + default = "generated/id_rsa" +} + +variable "runner_name" { + description = "Name of the runner, will be used in the runner config.toml" + type = "string" +} + +variable "gitlab_url" { + description = "URL of the gitlab instance to connect to." + type = "string" +} + +variable "runner_token" { + description = "Token for the runner, will be used in the runner config.toml" + type = "string" +} diff --git a/examples/runner-public/main.tf b/examples/runner-public/main.tf index c44039965..e874c32a4 100644 --- a/examples/runner-public/main.tf +++ b/examples/runner-public/main.tf @@ -23,18 +23,11 @@ module "runner" { runners_use_private_address = false - vpc_id = "${module.vpc.vpc_id}" - subnet_id_gitlab_runner = "${element(module.vpc.public_subnets, 0)}" - subnet_id_runners = "${element(module.vpc.public_subnets, 0)}" + vpc_id = "${module.vpc.vpc_id}" + subnet_ids_gitlab_runner = "${module.vpc.public_subnets}" + subnet_id_runners = "${element(module.vpc.public_subnets, 0)}" runners_name = "${var.runner_name}" runners_gitlab_url = "${var.gitlab_url}" runners_token = "${var.runner_token}" - - runners_off_peak_timezone = "Europe/Amsterdam" - runners_off_peak_idle_count = 0 - runners_off_peak_idle_time = 60 - - # working 9 to 5 :) - runners_off_peak_periods = "[\"* * 0-9,17-23 * * mon-fri *\", \"* * * * * sat,sun *\"]" } diff --git a/main.tf b/main.tf index 3fba75d43..a1535fe4b 100644 --- a/main.tf +++ b/main.tf @@ -85,6 +85,7 @@ data "template_file" "gitlab_runner" { gitlab_runner_version = "${var.gitlab_runner_version}" docker_machine_version = "${var.docker_machine_version}" runners_config = "${data.template_file.runners.rendered}" + runners_executor = "${var.runners_executor}" pre_install = "${var.userdata_pre_install}" post_install = "${var.userdata_post_install}" } @@ -92,7 +93,8 @@ data "template_file" "gitlab_runner" { locals { // Convert list to a string seperated and prepend by a comma - docker_machine_options_string = "${format(",%s", join(",", formatlist("%q", var.docker_machine_options)))}" + docker_machine_options_string = "${format(",%s", join(",", formatlist("%q", var.docker_machine_options)))}" + runners_off_peak_periods_string = "${var.runners_off_peak_periods == "" ? "" : format("OffPeakPeriods = %s", var.runners_off_peak_periods)}" } data "template_file" "runners" { @@ -109,14 +111,12 @@ data "template_file" "runners" { runners_spot_price_bid = "${var.docker_machine_spot_price_bid}" runners_security_group_name = "${aws_security_group.docker_machine.name}" runners_monitoring = "${var.runners_monitoring}" - runners_instance_profile = "${aws_iam_instance_profile.runners.name}" - - docker_machine_options = "${length(var.docker_machine_options) == 0 ? "" : local.docker_machine_options_string}" docker_machine_options = "${length(var.docker_machine_options) == 0 ? "" : local.docker_machine_options_string}" runners_name = "${var.runners_name}" runners_token = "${var.runners_token}" + runners_executor = "${var.runners_executor}" runners_limit = "${var.runners_limit}" runners_concurrent = "${var.runners_concurrent}" runners_image = "${var.runners_image}" @@ -126,7 +126,7 @@ data "template_file" "runners" { runners_off_peak_timezone = "${var.runners_off_peak_timezone}" runners_off_peak_idle_count = "${var.runners_off_peak_idle_count}" runners_off_peak_idle_time = "${var.runners_off_peak_idle_time}" - runners_off_peak_periods = "${var.runners_off_peak_periods}" + runners_off_peak_periods_string = "${local.runners_off_peak_periods_string}" runners_root_size = "${var.runners_root_size}" runners_iam_instance_profile_name = "${var.runners_iam_instance_profile_name}" runners_use_private_address = "${var.runners_use_private_address}" @@ -136,12 +136,13 @@ data "template_file" "runners" { runners_request_concurrency = "${var.runners_request_concurrency}" runners_output_limit = "${var.runners_output_limit}" bucket_name = "${aws_s3_bucket.build_cache.bucket}" + shared_cache = "${var.cache_shared}" } } resource "aws_autoscaling_group" "gitlab_runner_instance" { name = "${var.environment}-as-group" - vpc_zone_identifier = ["${var.subnet_id_gitlab_runner}"] + vpc_zone_identifier = ["${var.subnet_ids_gitlab_runner}"] # vpc_zone_identifier = ["${var.subnets}"] min_size = "1" @@ -213,6 +214,30 @@ resource "aws_iam_role_policy_attachment" "docker_machine" { policy_arn = "${aws_iam_policy.docker_machine.arn}" } +################################################################################ +### docker machine policy to access the shared cache +################################################################################ +data "template_file" "docker_machine_cache_policy" { + template = "${file("${path.module}/policies/cache.json")}" + + vars { + s3_cache_arn = "${aws_s3_bucket.build_cache.arn}" + } +} + +resource "aws_iam_policy" "docker_machine_cache" { + name = "${var.environment}-docker-machine-cache" + path = "/" + description = "Policy for docker machine instance to access cache" + + policy = "${data.template_file.docker_machine_cache_policy.rendered}" +} + +resource "aws_iam_role_policy_attachment" "docker_machine_cache" { + role = "${aws_iam_role.instance.name}" + policy_arn = "${aws_iam_policy.docker_machine_cache.arn}" +} + ################################################################################ ### Service linked policy, optional ################################################################################ @@ -238,41 +263,3 @@ resource "aws_iam_role_policy_attachment" "service_linked_role" { role = "${aws_iam_role.instance.name}" policy_arn = "${aws_iam_policy.service_linked_role.arn}" } - -################################################################################ -### docker machine runner role and policies -################################################################################ -data "template_file" "runners_role_trust_policy" { - template = "${length(var.instance_role_runner_json) > 0 ? var.instance_role_runner_json : file("${path.module}/policies/instance-role-trust-policy.json")}" -} - -resource "aws_iam_role" "runners" { - name = "${var.environment}-runners-role" - assume_role_policy = "${data.template_file.runners_role_trust_policy.rendered}" -} - -resource "aws_iam_instance_profile" "runners" { - name = "${var.environment}-runners-profile" - role = "${aws_iam_role.runners.name}" -} - -data "template_file" "cache_policy" { - template = "${file("${path.module}/policies/cache.json")}" - - vars { - s3_cache_arn = "${aws_s3_bucket.build_cache.arn}" - } -} - -resource "aws_iam_policy" "runners" { - name = "${var.environment}-runners-cache-policy" - path = "/" - description = "Policy for Runners." - - policy = "${data.template_file.cache_policy.rendered}" -} - -resource "aws_iam_role_policy_attachment" "runners" { - role = "${aws_iam_role.runners.name}" - policy_arn = "${aws_iam_policy.runners.arn}" -} diff --git a/outputs.tf b/outputs.tf index 3f031808d..10d5fef4e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -8,12 +8,7 @@ output "runner_cache_bucket_arn" { value = "${aws_s3_bucket.build_cache.arn}" } -output "runner_agent role" { +output "runner_agent_role" { description = "ARN of the rule used for the ec2 instance for the GitLab runner agent." value = "${aws_iam_role.instance.arn}" } - -output "runner_role" { - description = "ARN of the rule used for the docker machine runners." - value = "${aws_iam_role.runners.arn}" -} diff --git a/template/gitlab-runner.tpl b/template/gitlab-runner.tpl index 28320586c..c83256da9 100644 --- a/template/gitlab-runner.tpl +++ b/template/gitlab-runner.tpl @@ -7,6 +7,13 @@ EOF ${pre_install} +if [[ `echo ${runners_executor}` == "docker" ]] +then + yum install docker -y + usermod -a -G docker ec2-user + service docker start +fi + curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh | bash yum install gitlab-runner-${gitlab_runner_version} -y curl -L https://github.com/docker/machine/releases/download/v${docker_machine_version}/docker-machine-`uname -s`-`uname -m` >/tmp/docker-machine && \ diff --git a/template/runner-config.tpl b/template/runner-config.tpl index 97ab2ea60..fe87a5bed 100644 --- a/template/runner-config.tpl +++ b/template/runner-config.tpl @@ -5,7 +5,7 @@ check_interval = 0 name = "${runners_name}" url = "${gitlab_url}" token = "${runners_token}" - executor = "docker+machine" + executor = "${runners_executor}" pre_build_script = "${runners_pre_build_script}" post_build_script = "${runners_post_build_script}" pre_clone_script = "${runners_pre_clone_script}" @@ -19,10 +19,14 @@ check_interval = 0 disable_cache = false volumes = ["/cache"] shm_size = 0 - [runners.cache.s3] - ServerAddress = "s3-${aws_region}.amazonaws.com" - BucketName = "${bucket_name}" - Insecure = false + [runners.cache] + Type = "s3" + Shared = ${shared_cache} + [runners.cache.s3] + ServerAddress = "s3.amazonaws.com" + BucketName = "${bucket_name}" + BucketLocation = "${aws_region}" + Insecure = false [runners.machine] IdleCount = ${runners_idle_count} IdleTime = ${runners_idle_time} @@ -39,11 +43,10 @@ check_interval = 0 "amazonec2-security-group=${runners_security_group_name}", "amazonec2-tags=environment,${environment}", "amazonec2-monitoring=${runners_monitoring}", - "amazonec2-iam-instance-profile=${runners_instance_profile}", "amazonec2-root-size=${runners_root_size}" ${docker_machine_options} ] OffPeakTimezone = "${runners_off_peak_timezone}" OffPeakIdleCount = ${runners_off_peak_idle_count} OffPeakIdleTime = ${runners_off_peak_idle_time} - OffPeakPeriods = ${runners_off_peak_periods} + ${runners_off_peak_periods_string} diff --git a/variables.tf b/variables.tf index e867a46dc..adc4f73dc 100644 --- a/variables.tf +++ b/variables.tf @@ -18,9 +18,9 @@ variable "subnet_id_runners" { type = "string" } -variable "subnet_id_gitlab_runner" { +variable "subnet_ids_gitlab_runner" { description = "Subnet used for hosting the gitlab-runner." - type = "string" + type = "list" } variable "instance_type" { @@ -54,6 +54,12 @@ variable "runners_name" { type = "string" } +variable "runners_executor" { + description = "The executor to use. Currently supports docker+machine or docker" + type = "string" + default = "docker+machine" +} + variable "runners_gitlab_url" { description = "URL of the gitlab instance to connect to." type = "string" @@ -127,6 +133,10 @@ variable "runners_root_size" { default = 16 } +variable "create_runners_iam_instance_profile" { + default = true +} + variable "runners_iam_instance_profile_name" { description = "IAM instance profile name of the runners, will be used in the runner config.toml" type = "string" @@ -195,6 +205,12 @@ variable "cache_expiration_days" { default = 1 } +variable "cache_shared" { + description = "Enables cache sharing between runners, false by default." + type = "string" + default = "false" +} + variable "gitlab_runner_version" { description = "Version for the gitlab runner." type = "string"