Add warm pool with new ASG lifecycle hooks

.buildkite/steps/launch.sh

@@ -103,10 +103,10 @@ echo "--- Building templates"
 make "mappings-for-${os}-${arch}-image" build/aws-stack.yml "IMAGE_ID=$image_id"
 
 echo "--- Validating templates"
-make validate
+make validate "BUCKET_PREFIX=$stack_name"
 
 echo "--- Creating stack ${stack_name}"
-make create-stack "STACK_NAME=$stack_name" "SERVICE_ROLE=$service_role"
+make create-stack "BUCKET_PREFIX=$stack_name" "STACK_NAME=$stack_name" "SERVICE_ROLE=$service_role"
 
 echo "+++ ⌛️ Waiting for update to complete"
 ./parfait watch-stack "${stack_name}"

Makefile

@@ -143,12 +143,13 @@ ifdef SERVICE_ROLE
 	role_arn="--role-arn=$(SERVICE_ROLE)"
 endif
 
-create-stack: build/aws-stack.yml env-STACK_NAME
+create-stack: build/aws-stack.yml env-STACK_NAME env-BUILDKITE_AWS_STACK_BUCKET env-BUCKET_PREFIX
+	aws s3 cp --content-type 'text/yaml' --acl public-read build/aws-stack.yml "s3://$(BUILDKITE_AWS_STACK_BUCKET)/$(BUCKET_PREFIX)/aws-stack.yml"
 	aws cloudformation create-stack \
 		--output text \
 		--stack-name $(STACK_NAME) \
 		--disable-rollback \
-		--template-body "file://$(PWD)/build/aws-stack.yml" \
+		--template-url "https://$(BUILDKITE_AWS_STACK_BUCKET).s3.amazonaws.com/$(BUCKET_PREFIX)/aws-stack.yml" \
 		--capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \
 		--parameters "$$(cat config.json)" \
 		"$(role_arn)"
@@ -175,10 +176,11 @@ bump-agent-version:
 	git add README.md packer/linux/scripts/install-buildkite-agent.sh packer/windows/scripts/install-buildkite-agent.ps1
 	git commit -m "Bump buildkite-agent to v$(AGENT_VERSION)"
 
-validate: build/aws-stack.yml
+validate: build/aws-stack.yml env-BUILDKITE_AWS_STACK_BUCKET env-BUCKET_PREFIX
+	aws s3 cp --content-type 'text/yaml' --acl public-read build/aws-stack.yml "s3://$(BUILDKITE_AWS_STACK_BUCKET)/$(BUCKET_PREFIX)/aws-stack.yml"
 	aws cloudformation validate-template \
 		--output text \
-		--template-body "file://$(PWD)/build/aws-stack.yml"
+		--template-url "https://$(BUILDKITE_AWS_STACK_BUCKET).s3.amazonaws.com/$(BUCKET_PREFIX)/aws-stack.yml"
 
 generate-toc:
 	docker run -it --rm -v "$(PWD):/app" node:slim bash \

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -215,6 +215,12 @@ fi
 
 systemctl enable "buildkite-agent"
 
+# If warm pool is enabled, don't start the agent until the ASG BootHook
+if [ "${BUILDKITE_WARM_POOL:-false}" == "false" ]
+then
+	systemctl start "buildkite-agent"
+fi
+
 # let the stack know that this host has been initialized successfully
 /opt/aws/bin/cfn-signal \
 	--region "$AWS_REGION" \

packer/windows/conf/bin/bk-install-elastic-stack.ps1

@@ -210,6 +210,11 @@ If ($lastexitcode -ne 0) { Exit $lastexitcode }
 nssm set buildkite-agent AppRestartDelay 5000
 If ($lastexitcode -ne 0) { Exit $lastexitcode }
 
+# If warm pool is enabled, don't start the agent until the ASG BootHook
+If ($null -eq $Env:BUILDKITE_WARM_POOL -or $Env:BUILDKITE_WARM_POOL -eq "false") {
+  Restart-Service buildkite-agent 
+}
+
 # renable debug tracing
 Set-PSDebug -Trace 2