#147: Add unified CI builder for Jenkins and Gitlab CI

cmf/all/.cikit/ci/server-cleaner.yml

@@ -0,0 +1,31 @@
+# This is a playbook that runs if CI server has less than 90% of a drive space
+# available.
+#
+# Available variables:
+# - rc: an exit code of the build process. If it's not "0" then a build failed;
+# - dist: an absolute path to the project's destination;
+# - env: a name of the environment being built;
+# - site-url: a URL of a website being built;
+# - build-id: an ID of CI build;
+# - workspace: an absolute path to the sandbox directory on CI server.
+---
+- hosts: localhost
+  gather_facts: no
+  connection: local
+  become: yes
+
+  vars_files:
+    - ../config.yml
+
+  vars:
+    mysql_query: "mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se"
+
+  tasks:
+    - name: Remove all builds
+      shell: "{{ item }}"
+      args:
+        warn: no
+        executable: bash
+      with_items:
+        - '{{ mysql_query }} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" {{ mysql_query }} "DROP DATABASE @@"'
+        - 'rm -rf {{ webroot }}/*{{ build_slug }}*'

cmf/all/.cikit/config.yml

@@ -4,6 +4,12 @@ tmproot: /tmp
 webroot: /var/www
 # The directory to "cd" into after "vagrant ssh". Defaults to the value of "webroot".
 ssh_home: ~
+# The part of a directory/database name for CI builds.
+build_slug: build
+# Do not try changing these values.
+mysql:
+  user: root
+  pass: root
 APPLICATION_CONFIG: ~
 vm:
   ip: 192.168.56.132

cmf/all/scripts/vars/main.yml

@@ -10,10 +10,6 @@ reinstall_mode: full
 env_vars:
   APP_ENV: "{{ env }}"
 
-mysql:
-  user: root
-  pass: root
-
 databases:
   default:
     name: "{{ cmf }}_{{ project | replace('-', '_') }}_{{ build_id | default(env) }}"

lib/platformsh/setup.yml

@@ -1,3 +1,4 @@
+# This playbook runs by Vagrantfile only.
 ---
 - hosts: localhost
   connection: local

scripts/roles/cikit-ci/tasks/main.yml

@@ -1,5 +1,17 @@
 ---
+- name: Determine CI service
+  set_fact:
+    cikit_ci: "{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}"
+
 - include_role:
     # Install Gitlab CI if either "--gitlab-ci-token" or "--gitlab-ci-url"
     # is passed. Use Jenkins otherwise.
-    name: "cikit-{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}"
+    name: "cikit-{{ cikit_ci }}"
+
+- name: Deploy CI builder
+  template:
+    src: ci-builder.sh.j2
+    dest: /var/ci-builder.sh
+    mode: a+x
+    owner: "{{ jenkins_data.user if cikit_ci == 'jenkins' else 'root' }}"
+    group: "{{ jenkins_data.group if cikit_ci == 'jenkins' else 'root' }}"

scripts/roles/cikit-ci/templates/ci-builder.sh.j2

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+set -e
+export PYTHONUNBUFFERED=1
+
+# ------------------------------------------------------------------------------
+# The required configuration of a process.
+# ------------------------------------------------------------------------------
+
+declare -rA VARIABLES=(
+  [BUILD_NUMBER]="either \"stable\" or any custom value"
+  [BUILD_MODE]="either \"full\" or \"pull\""
+  [BUILD_ENV]="the name of an environment to build or \"default\""
+  [CIKIT_PROJECT_DIR]="the path to directory where repository clones to"
+  [CIKIT_PROJECT_HOSTNAME]="the hostname where the project can be accessed"
+  [RUN_SNIFFERS]="either \"yes\" or whatever"
+  [RUN_TESTS]="either \"yes\" or whatever"
+)
+
+for VARIABLE in "${!VARIABLES[@]}"; do
+  if [ -z ${!VARIABLE+x} ]; then
+    echo "The \"$VARIABLE\" variable is missing! It's value must be ${VARIABLES[$VARIABLE]}."
+    exit 1
+  fi
+done
+
+# ------------------------------------------------------------------------------
+# Read CIKit configuration.
+# ------------------------------------------------------------------------------
+
+declare -A CIKIT_PROJECT_CONFIG=()
+
+for VARIABLE in webroot project build_slug; do
+  VALUE="$(awk '/'"$VARIABLE"':/ {print $2}' < "$CIKIT_PROJECT_DIR/.cikit/config.yml")"
+
+  if [ -z "$VALUE" ]; then
+    echo "The value of \"$VARIABLE\" variable cannot be empty!"
+    exit 2
+  fi
+
+  CIKIT_PROJECT_CONFIG["$VARIABLE"]="$VALUE"
+done
+
+# ------------------------------------------------------------------------------
+# Compute build parameters.
+# ------------------------------------------------------------------------------
+
+BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-"
+
+if [ "$BUILD_NUMBER" == "stable" ]; then
+  IS_COMMIT=false
+  BUILD_ID+="$BUILD_ENV"
+else
+  IS_COMMIT=true
+  BUILD_ID+="${CIKIT_PROJECT_CONFIG['build_slug']}-$BUILD_NUMBER"
+fi
+
+# Replace underscores by dashes in the ID of a build.
+BUILD_ID="${BUILD_ID//_/-}"
+# Form an absolute path to directory where the project is accessible from web.
+DESTINATION="${CIKIT_PROJECT_CONFIG['webroot']}/$BUILD_ID"
+PLAYBOOK_ARGS=(
+  "--site-url=https://$BUILD_ID.$CIKIT_PROJECT_HOSTNAME"
+  "--build-id=$BUILD_ID"
+  "--workspace=$CIKIT_PROJECT_DIR"
+)
+
+# Pass the environment name to Ansible playbooks if it's not a default one.
+if [ "$BUILD_ENV" != "default" ]; then
+  PLAYBOOK_ARGS+=("--env=$BUILD_ENV")
+fi
+
+# ------------------------------------------------------------------------------
+# Define functions.
+# ------------------------------------------------------------------------------
+
+ci_hook() {
+  local HOOK_PLAYBOOK="$CIKIT_PROJECT_DIR/.cikit/ci/$1.yml"
+
+  if [ -f "$HOOK_PLAYBOOK" ]; then
+    cikit "$HOOK_PLAYBOOK" "${PLAYBOOK_ARGS[@]}" --dist="$DESTINATION" --rc="$2"
+  fi
+}
+
+handle_exit() {
+  ci_hook post-deploy $?
+
+  # The drive space is occupied by more than 90%.
+  if [ $(df -H | head -2 | tail -1 | awk '{printf "%d", $5}') -gt 90 ]; then
+    ci_hook server-cleaner 0
+  fi
+
+  # The "$USER" must be either "jenkins" or "gitlab-runner".
+  sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR"
+  sudo chown -R "$USER":"$USER" "$HOME"
+}
+
+trap handle_exit EXIT
+env
+ci_hook pre-deploy 0
+
+# Install a project.
+if ${IS_COMMIT}; then
+  cikit reinstall "${PLAYBOOK_ARGS[@]}" --actions="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")"
+else
+  cikit reinstall "${PLAYBOOK_ARGS[@]}" --reinstall-mode="$BUILD_MODE"
+fi
+
+# Copy codebase to directory accessible from the web.
+sudo rsync -ra --delete --chown=www-data:www-data ./ "$DESTINATION/"
+
+if [ "$RUN_SNIFFERS" == "yes" ]; then
+  cikit sniffers "${PLAYBOOK_ARGS[@]}"
+fi
+
+if [ "$RUN_TESTS" == "yes" ]; then
+  cikit tests "${PLAYBOOK_ARGS[@]}" --run --headless
+fi

scripts/roles/cikit-env/defaults/main.yml

@@ -1,9 +1,7 @@
 ---
 cikit_env:
   # The list of per-file variables to unconditionally add everywhere.
-  global:
-    /etc/environment:
-      CIKIT_PROJECT: "{{ project }}"
+  global: {}
   # The list of per-file variables to add to development environment only (VM).
   local:
     /etc/environment:
@@ -12,6 +10,4 @@ cikit_env:
       CIKIT_PROJECT_URI: "{{ site_url }}"
       SSH_HOME: "{{ ssh_home if ssh_home | default else webroot }}"
   # The list of per-file variables for CI environment (remote CI server).
-  ci:
-    /etc/environment:
-      CIKIT_CI: true
+  ci: {}

scripts/roles/cikit-jenkins/defaults/main.yml

@@ -8,9 +8,6 @@ jenkins_data:
   user: jenkins
   group: jenkins
   jobs:
-    - name: SERVER_CLEANER
-      template: server_cleaner.xml.j2
-
     - name: "{{ jenkins_job }}_BUILDER"
       template: builder.xml.j2
 

scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2

@@ -85,49 +85,12 @@ Pull - do not reinstall an application and just grab latest changes from reposit
 
 set -e
 
-export PROJECT="$(echo "${JOB_NAME//_BUILDER/}" | tr '[:upper:]' '[:lower:]')"
-export BUILD_NAME="${PROJECT}_${BUILD_ENV}"
-export SITE_URL="https://$BUILD_NAME.$(php -r "echo parse_url('${JOB_URL}')['host'];")"
-export DESTINATION="/var/www/$BUILD_NAME"
-export CIKIT_PROJECT_DIR="$WORKSPACE"
-export PYTHONUNBUFFERED=1
 export ANSIBLE_VERBOSITY=2
+export BUILD_NUMBER="stable"
+export CIKIT_PROJECT_DIR="$WORKSPACE"
+export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")"
 
-ARGS=(
-  "--env=$BUILD_ENV"
-  "--site-url=$SITE_URL"
-  "--build-id=$BUILD_NAME"
-  "--workspace=$WORKSPACE"
-)
-
-deploy_hook() {
-  local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml"
-
-  if [ -f "$HOOK_PLAYBOOK" ]; then
-    cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" --rc="$2"
-  fi
-}
-
-handle_shutdown() {
-  deploy_hook post-deploy $?
-  sudo chown -R jenkins:jenkins "$WORKSPACE"
-}
-
-trap handle_shutdown EXIT
-deploy_hook pre-deploy
-cikit reinstall "${ARGS[@]}" --reinstall-mode="$BUILD_MODE"
-
-# Copy codebase to directory, accessible from web.
-sudo rsync --delete -ra ./ "$DESTINATION/"
-sudo chown -R www-data:jenkins $_
-
-if ${RUN_SNIFFERS}; then
-  cikit sniffers "${ARGS[@]}"
-fi
-
-if ${RUN_TESTS}; then
-  cikit tests "${ARGS[@]}" --run --headless
-fi
+/var/ci-builder.sh
 </command>
     </hudson.tasks.Shell>
   </builders>

scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2

@@ -97,70 +97,19 @@
 
 set -e
 
-export PROJECT="$(echo "${JOB_NAME//_BUILDER/}" | tr '[:upper:]' '[:lower:]')"
-export BUILD_NAME="${PROJECT}_build_${BUILD_NUMBER}"
-export SITE_URL="https://$BUILD_NAME.$(php -r "echo parse_url('${JOB_URL}')['host'];")"
-export DESTINATION="/var/www/$BUILD_NAME"
-export CIKIT_PROJECT_DIR="$WORKSPACE"
-export PYTHONUNBUFFERED=1
 export ANSIBLE_VERBOSITY=2
-export BUILD_ACTIONS=$(php -r "echo json_encode(array_map('trim', array_filter(
-  explode(PHP_EOL, '$(git log --format=%B -n1 ${ghprbActualCommit} | awk -vRS="]" -vFS="[" '{print $2}')')
-)));")
-
-ARGS=(
-  "--site-url=$SITE_URL"
-  "--build-id=$BUILD_NAME"
-  "--workspace=$WORKSPACE"
-)
-
-deploy_hook() {
-  local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml"
-
-  if [ -f "$HOOK_PLAYBOOK" ]; then
-    cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" --rc="$2"
-  fi
-}
-
-handle_shutdown() {
-  deploy_hook post-deploy $?
-  sudo chown -R jenkins:jenkins "$WORKSPACE"
-}
-
-trap handle_shutdown EXIT
-deploy_hook pre-deploy
-cikit reinstall "${ARGS[@]}" --actions="$BUILD_ACTIONS"
-
-# Copy codebase to directory, accessible from web.
-sudo rsync --delete -ra ./ "$DESTINATION/"
-sudo chown -R www-data:jenkins $_
+export BUILD_MODE="full"
+export BUILD_ENV="default"
+export CIKIT_PROJECT_DIR="$WORKSPACE"
+export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")"
+export RUN_SNIFFERS="yes"
+export RUN_TESTS="no"
 
-cikit sniffers "${ARGS[@]}"
+/var/ci-builder.sh
 </command>
     </hudson.tasks.Shell>
   </builders>
-  <publishers>
-    <hudson.plugins.parameterizedtrigger.BuildTrigger plugin="[email protected]">
-      <configs>
-        <hudson.plugins.parameterizedtrigger.BuildTriggerConfig>
-          <configs>
-            <hudson.plugins.parameterizedtrigger.BooleanParameters>
-              <configs>
-                <hudson.plugins.parameterizedtrigger.BooleanParameterConfig>
-                  <name>FORCE</name>
-                  <value>false</value>
-                </hudson.plugins.parameterizedtrigger.BooleanParameterConfig>
-              </configs>
-            </hudson.plugins.parameterizedtrigger.BooleanParameters>
-          </configs>
-          <projects>SERVER_CLEANER</projects>
-          <condition>ALWAYS</condition>
-          <triggerWithNoParameters>false</triggerWithNoParameters>
-          <triggerFromChildProjects>false</triggerFromChildProjects>
-        </hudson.plugins.parameterizedtrigger.BuildTriggerConfig>
-      </configs>
-    </hudson.plugins.parameterizedtrigger.BuildTrigger>
-  </publishers>
+  <publishers/>
   <buildWrappers>
     <hudson.plugins.ansicolor.AnsiColorBuildWrapper plugin="[email protected]">
       <colorMapName>xterm</colorMapName>