[ML] SonarQube buildkite integration

.buildkite/pipeline.json.py

@@ -31,37 +31,39 @@
 def main():
  pipeline = {}
  pipeline_steps = step.PipelineStep([])
- pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
- ".buildkite/pipelines/send_slack_notification.sh"))
- pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
- ".buildkite/pipelines/send_email_notification.sh"))
- pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
- ".buildkite/pipelines/format_and_validation.yml.sh"))
+ # pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
+ # ".buildkite/pipelines/send_slack_notification.sh"))
+ # pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
+ # ".buildkite/pipelines/send_email_notification.sh"))
+ # pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
+ # ".buildkite/pipelines/format_and_validation.yml.sh"))
+ pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
+ ".buildkite/pipelines/sonarqube.yml.sh"))
  config = buildConfig.Config()
  config.parse()
- if config.build_windows:
- build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64)
- pipeline_steps.append(build_windows)
- if config.build_macos:
- build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64)
- pipeline_steps.append(build_macos)
- if config.build_linux:
- build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64)
- pipeline_steps.append(build_linux)
+ # if config.build_windows:
+ #  build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64)
+ #  pipeline_steps.append(build_windows)
+ # if config.build_macos:
+ #  build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64)
+ #  pipeline_steps.append(build_macos)
+ # if config.build_linux:
+ #  build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64)
+ #  pipeline_steps.append(build_linux)
 
- if config.build_x86_64:
- pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
- ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
- # We only use linux x86_64 builds for QA tests.
- if config.run_qa_tests:
- pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline",
- ".buildkite/pipelines/run_qa_tests.yml.sh"))
- if config.run_pytorch_tests:
- pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline",
- ".buildkite/pipelines/run_pytorch_tests.yml.sh"))
- if config.build_aarch64:
- pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline",
- ".buildkite/pipelines/run_es_tests_aarch64.yml.sh"))
+ #  if config.build_x86_64:
+ #  pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
+ #  ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
+ #  # We only use linux x86_64 builds for QA tests.
+ #  if config.run_qa_tests:
+ #  pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline",
+ #  ".buildkite/pipelines/run_qa_tests.yml.sh"))
+ #  if config.run_pytorch_tests:
+ #  pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline",
+ #  ".buildkite/pipelines/run_pytorch_tests.yml.sh"))
+ #  if config.build_aarch64:
+ #  pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline",
+ #  ".buildkite/pipelines/run_es_tests_aarch64.yml.sh"))
 
  pipeline["env"] = env
  pipeline["steps"] = pipeline_steps

.buildkite/pipelines/sonarqube.yml.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+cat <<EOL
+steps:
+ - label: "Run SonarQube scanner :sonarqube:"
+ key: "export_compile_commands"
+ # depends_on: "build_test_linux-x86_64-RelWithDebInfo"
+ soft_fail: true
+ agents:
+ cpu: 6
+ memory: "64GB"
+ image: "docker.elastic.co/ml-dev/ml-linux-build:30"
+ env:
+ PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
+ command: 
+ - ".buildkite/scripts/steps/run_sonar-scanner.sh"
+ # artifact_paths:
+ # - "cmake-build-docker/compile_commands.json"
+ notify:
+ - github_commit_status:
+ context: "Run SonarQube scanner"
+EOL

.buildkite/scripts/steps/generate_coverage_report.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+set -eo pipefail
+
+echo "Generating coverage report"
+find ./cmake-build-docker -name "*.gcda" -print0 | xargs -0 -n 1 -P $(nproc) gcov --preserve-paths
+
+# Crate gcov.tar.gz from all the .gcov files in .
+echo "Creating gcov.tar.gz"
+find . -name "*.gcov" -print0 | tar -czf gcov.tar.gz --null -T -

.buildkite/scripts/steps/run_sonar-scanner.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+
+set -eo pipefail
+
+# # Unpack gcov.tar.gz if it exists
+# if [ -f gcov.tar.gz ]; then
+# echo "Unpacking gcov.tar.gz"
+# tar -xzf gcov.tar.gz
+# fi
+
+
+export CWD="$(pwd)"
+
+# Install the sonar-scanner
+echo "Installing sonar-scanner"
+cd /usr/local
+curl -O https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.1.0.4477-linux-x64.zip
+unzip -q sonar-scanner-cli-6.1.0.4477-linux-x64.zip
+mv sonar-scanner-6.1.0.4477-linux-x64/ sonar-scanner
+export PATH=$(pwd)/sonar-scanner/bin:$PATH
+
+# Generate the compile_commands.json file
+echo "Generating compile_commands.json"
+cd "${CWD}"
+
+cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
+
+# Analyze the source code with SonarQube
+# The following script was adapted from the Elastic CI team's script:
+# https://github.com/elastic/sonarqube/blob/main/buildkite-scanner/scan-source-code.sh
+
+trap 'error_handler $?' EXIT
+
+retries=0
+
+error_handler() {
+ local err
+ err=$1
+
+ # Since we need to trap the sonar-scanner script exit code, we don't generate notifications on success.
+ if [[ $err -eq 0 ]]; then
+ exit "$err"
+ fi
+
+ exit "$err"
+}
+
+runScanner(){
+ local max_retries=5
+ local retry_delay=30
+ local exit_code=-1 # Initialize to a negative value to allow while loop to run.
+
+ while [[ $retries -lt $max_retries ]] && [[ $exit_code -ne 0 ]]; do
+ set +e # Temporarily disable exit immediately on error to allow retries
+ if [[ ${BUILDKITE_PULL_REQUEST} =~ ^[0-9]+$ ]];
+ then
+ echo "Spotted PR"
+ sonar-scanner -Dsonar.token="${SONAR_LOGIN}" \
+ -Dsonar.pullrequest.key="${BUILDKITE_PULL_REQUEST}" \
+ -Dsonar.pullrequest.branch="${BUILDKITE_BRANCH}" \
+ -Dsonar.pullrequest.base="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" \
+ -Dsonar.projectVersion="${BUILDKITE_COMMIT}" \
+ -Dsonar.scm.provider=git \
+ -Dsonar.pullrequest.github.repository="elastic/ml-cpp" \
+ -Dsonar.pullrequest.provider=github \
+ -Dsonar.pullrequest.github.endpoint="https://api.github.com"
+ exit_code=$?
+ else
+ sonar-scanner -Dsonar.token="${SONAR_LOGIN}" \
+ -Dsonar.branch.name="${BUILDKITE_BRANCH}" \
+ -Dsonar.projectVersion="${BUILDKITE_COMMIT}" \
+ -Dsonar.scm.provider=git 
+ exit_code=$?
+ fi
+ set -e # Re-enable exit immediately on error
+
+ if [[ $exit_code -ne 0 ]]; then
+ retries=$((retries + 1))
+ echo "Sonar Scanner failed with exit code ${exit_code}. Retrying in ${retry_delay} seconds..."
+
+ sleep $retry_delay
+ retry_delay=$((retry_delay * retries))
+ fi
+ done
+
+ return "$exit_code"
+}
+
+# Check if we are in a git repo
+git rev-parse --is-inside-work-tree >/dev/null
+
+# SonarQube project analyse token was provided
+if [[ -z "${SONAR_LOGIN}" ]]; then
+ echo "No SONAR_LOGIN token was provided, attempting to resolve it via vault..."
+
+ if [[ -z "${VAULT_ADDR}" ]];
+ then
+ echo "VAULT_ADDR is missing."
+ exit 1
+ fi
+ if [[ -z "${VAULT_TOKEN}" ]];
+ then
+ echo "A VAULT_TOKEN is missing for ${VAULT_ADDR}."
+ exit 1
+ fi
+ if [[ -z "${VAULT_SONAR_TOKEN_PATH}" ]];
+ then
+ echo "VAULT_SONAR_TOKEN_PATH is missing."
+ exit 1
+ fi
+
+ if [[ "$VAULT_SONAR_TOKEN_PATH" =~ ^kv/* ]];
+ then
+ SONAR_LOGIN=$(vault kv get --field token "${VAULT_SONAR_TOKEN_PATH}")
+ else
+ SONAR_LOGIN=$(vault read --field token "${VAULT_SONAR_TOKEN_PATH}")
+ fi
+fi
+
+echo "Running sonar-scanner"
+runScanner

generate_coverage.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Find all .gcda files and run gcov on each with --preserve-paths
+find ./cmake-build-relwithdebinfo -name "*.gcda" -exec gcov --preserve-paths {} \;
+
+echo "Coverage data generated."

sonar-project.properties

@@ -0,0 +1,21 @@
+# must be unique in a given SonarQube instance
+sonar.projectKey=elastic_ml-cpp_271ade36-31fc-4c6b-966e-80245560ad14
+
+# Encoding of the source code. Default is default system encoding
+sonar.sourceEncoding=UTF-8
+sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
+
+sonar.host.url=https://sonar.elastic.dev
+sonar.sources= ./bin,./lib,./include
+sonar.tests=./lib/core/unittest,./lib/ver/unittest,./lib/maths/analytics/unittest,./lib/maths/common/unittest,./lib/maths/time_series/unittest,./lib/seccomp/unittest,./lib/api/unittest,./lib/model/unittest
+sonar.language=cpp
+sonar.inclusions = **/*.cc,**/*.h
+sonar.exclusions = **/unittest/**
+sonar.lang.patterns.cpp=**/*.cc,**/*.h
+sonar.lang.patterns.c=**/*.c
+
+# Exclude coverage files from being analyzed as sources
+sonar.coverage.exclusions=**/unittest/**
+
+# # Import coverage report
+# sonar.cfamily.gcov.reportsPath=.