From a88de18acbd740bdf6d2026c35abb726d66c5842 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Thu, 27 Jun 2024 11:23:33 +0200
Subject: [PATCH 01/38] add sonar projects file

---
 sonar-project.properties | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 sonar-project.properties

diff --git a/sonar-project.properties b/sonar-project.properties
new file mode 100644
index 0000000000..ee774c0fe6
--- /dev/null
+++ b/sonar-project.properties
@@ -0,0 +1,28 @@
+# must be unique in a given SonarQube instance
+sonar.projectKey=elastic_ml-cpp
+
+# --- optional properties ---
+
+# defaults to project key
+#sonar.projectName=My project
+# defaults to 'not provided'
+#sonar.projectVersion=1.0
+ 
+# Path is relative to the sonar-project.properties file. Defaults to .
+# sonar.sources=.
+ 
+# Encoding of the source code. Default is default system encoding
+sonar.sourceEncoding=UTF-8
+sonar.cfamily.compile-commands=compile_commands.json
+
+sonar.host.url=https://sonar.elastic.dev
+sonar.sources=./lib, ./bin, ./include
+sonar.host.url=https://sonar.elastic.dev
+sonar.language=c,cpp
+sonar.inclusions = **/*.cc, **/*.h
+
+sonar.cfamily.reportingCppStandardOverride=c++14
+
+# sonar cxx plugin
+sonar.cxx.file.suffixes=.h,.cc
+sonar.cxx.clangtidy.reportPaths=clang-tidy-report.txt

From 0b75bfe900a81d25d4a6cc02233e06d35fb0490d Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:17:28 +0200
Subject: [PATCH 02/38] add buildkite steps

---
 .buildkite/branch.json.py                |  3 +++
 .buildkite/pipelines/build_macos.json.py |  6 +++++-
 .buildkite/pipelines/sonarqube.yml.sh    | 26 ++++++++++++++++++++++++
 sonar-project.properties                 | 15 --------------
 4 files changed, 34 insertions(+), 16 deletions(-)
 create mode 100644 .buildkite/pipelines/sonarqube.yml.sh

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index 0bbda836ae..e118679e79 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -40,6 +40,9 @@ def main():
         build_linux = pipeline_steps.generate_step_template("Linux", "build")
         pipeline_steps.append(build_linux)
 
+    pipeline_steps.append(pipeline_steps.generate_step("Scana and upload SonarQube report", 
+                                                       ".buildkite/pipelines/sonarqube.yml.sh"))
+
     # Build the DRA artifacts and upload to S3 and GCS
     pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
                                                        ".buildkite/pipelines/create_dra.yml.sh"))
diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 522e2a32ae..efed7ff8bd 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -60,8 +60,12 @@ def main(args):
               "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake",
               "RUN_TESTS": "true",
               "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
+              "CMAKE_EXPORT_COMPILE_COMMANDS": "1" if arch == "aarch64" and build_type == "RelWithDebInfo" else "0",
             },
-            "artifact_paths": "*/**/unittest/boost_test_results.junit;*/**/unittest/ml_test_*",
+            "artifact_paths": [
+              "*/**/unittest/boost_test_results.junit;*/**/unittest/ml_test_*",
+              "cmake-build-docker/compile_commands.json" if arch == "aarch64" and build_type == "RelWithDebInfo" else "",
+            ],
             "plugins": {
               "test-collector#v1.2.0": {                                                              
                 "files": "*/*/unittest/boost_test_results.junit",
diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
new file mode 100644
index 0000000000..c5b82fe01a
--- /dev/null
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+cat <<EOL
+steps:
+  - label: "Static code analysis with SonarQube :sonarqube:"
+    key: "sonarqube"
+    depends_on: "build_test_macos-aarch64-RelWithDebInfo"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
+    command: 
+      - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
+      - ".buildkite/scripts/steps/sonar-qube.sh"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    notify:
+      - github_commit_status:
+          context: "Static code analysis with SonarQube"
+EOL
\ No newline at end of file
diff --git a/sonar-project.properties b/sonar-project.properties
index ee774c0fe6..d5f70e7c0d 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -1,28 +1,13 @@
 # must be unique in a given SonarQube instance
 sonar.projectKey=elastic_ml-cpp
 
-# --- optional properties ---
-
-# defaults to project key
-#sonar.projectName=My project
-# defaults to 'not provided'
-#sonar.projectVersion=1.0
- 
-# Path is relative to the sonar-project.properties file. Defaults to .
-# sonar.sources=.
- 
 # Encoding of the source code. Default is default system encoding
 sonar.sourceEncoding=UTF-8
 sonar.cfamily.compile-commands=compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
 sonar.sources=./lib, ./bin, ./include
-sonar.host.url=https://sonar.elastic.dev
 sonar.language=c,cpp
 sonar.inclusions = **/*.cc, **/*.h
 
 sonar.cfamily.reportingCppStandardOverride=c++14
-
-# sonar cxx plugin
-sonar.cxx.file.suffixes=.h,.cc
-sonar.cxx.clangtidy.reportPaths=clang-tidy-report.txt

From 05b6d59e91997a4138593819853345d6a64122ef Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:45:53 +0200
Subject: [PATCH 03/38] generate compile commands in extra step

---
 .buildkite/branch.json.py                |  5 ++---
 .buildkite/pipelines/build_macos.json.py |  6 +-----
 .buildkite/pipelines/sonarqube.yml.sh    | 14 +++++++++++++-
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index e118679e79..83fdbebe9c 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -28,6 +28,8 @@ def main():
                                                        ".buildkite/pipelines/send_email_notification.sh"))
     pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
                                                        ".buildkite/pipelines/format_and_validation.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Scana and upload SonarQube report", 
+                                                       ".buildkite/pipelines/sonarqube.yml.sh"))
     config = buildConfig.Config()
     config.parse()
     if config.build_windows:
@@ -40,9 +42,6 @@ def main():
         build_linux = pipeline_steps.generate_step_template("Linux", "build")
         pipeline_steps.append(build_linux)
 
-    pipeline_steps.append(pipeline_steps.generate_step("Scana and upload SonarQube report", 
-                                                       ".buildkite/pipelines/sonarqube.yml.sh"))
-
     # Build the DRA artifacts and upload to S3 and GCS
     pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
                                                        ".buildkite/pipelines/create_dra.yml.sh"))
diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index efed7ff8bd..522e2a32ae 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -60,12 +60,8 @@ def main(args):
               "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake",
               "RUN_TESTS": "true",
               "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
-              "CMAKE_EXPORT_COMPILE_COMMANDS": "1" if arch == "aarch64" and build_type == "RelWithDebInfo" else "0",
             },
-            "artifact_paths": [
-              "*/**/unittest/boost_test_results.junit;*/**/unittest/ml_test_*",
-              "cmake-build-docker/compile_commands.json" if arch == "aarch64" and build_type == "RelWithDebInfo" else "",
-            ],
+            "artifact_paths": "*/**/unittest/boost_test_results.junit;*/**/unittest/ml_test_*",
             "plugins": {
               "test-collector#v1.2.0": {                                                              
                 "files": "*/*/unittest/boost_test_results.junit",
diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index c5b82fe01a..eaf8e616af 100644
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -10,9 +10,21 @@
 
 cat <<EOL
 steps:
+  - label: "Generate compile_commands.json"
+    key: "export_compile_commands"
+    depends_on: "check_style"
+    agents:
+      image: "docker.elastic.co/ml-dev/ml-linux-build:29"
+    command:
+      - "cmake CMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
+    artifact_paths:
+      - "cmake-build-docker/compile_commands.json"
+    notify:
+      - github_commit_status:
+          context: "Generate compile_commands.json"
   - label: "Static code analysis with SonarQube :sonarqube:"
     key: "sonarqube"
-    depends_on: "build_test_macos-aarch64-RelWithDebInfo"
+    depends_on: "export_compile_commands"
     env:
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 

From 0aad4ae3b9a74be64827412cda87f0e1e1f03df8 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:46:48 +0200
Subject: [PATCH 04/38] typo

---
 .buildkite/branch.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index 83fdbebe9c..66d4d11ee6 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -28,7 +28,7 @@ def main():
                                                        ".buildkite/pipelines/send_email_notification.sh"))
     pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
                                                        ".buildkite/pipelines/format_and_validation.yml.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Scana and upload SonarQube report", 
+    pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
                                                        ".buildkite/pipelines/sonarqube.yml.sh"))
     config = buildConfig.Config()
     config.parse()

From 34ec879d1ac399d805aa152911fa666f740bf54c Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:55:06 +0200
Subject: [PATCH 05/38] only run sonarqube scanning for snapshots

---
 .buildkite/branch.json.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index 66d4d11ee6..dffe2500f5 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -17,6 +17,7 @@
     step,
     config as buildConfig,
 )
+import os
 
 def main():
     pipeline = {}
@@ -28,8 +29,10 @@ def main():
                                                        ".buildkite/pipelines/send_email_notification.sh"))
     pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
                                                        ".buildkite/pipelines/format_and_validation.yml.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
-                                                       ".buildkite/pipelines/sonarqube.yml.sh"))
+    # Only create the SonarQube step for the snapshot builds
+    if  os.environ.get("BUILD_SNAPSHOT", "") == "true":
+        pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
+                                                        ".buildkite/pipelines/sonarqube.yml.sh"))
     config = buildConfig.Config()
     config.parse()
     if config.build_windows:

From e7ca5f8eaba43cd1b88d30adfb80ed601cbd40fa Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 09:50:55 +0200
Subject: [PATCH 06/38] File permissions updated

---
 .buildkite/pipelines/sonarqube.yml.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 .buildkite/pipelines/sonarqube.yml.sh

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
old mode 100644
new mode 100755

From 3b58cab9b7b9365861c866527c64a622b03aa65d Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 10:32:16 +0200
Subject: [PATCH 07/38] update script path for scanning

---
 .buildkite/pipelines/sonarqube.yml.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index eaf8e616af..14d5b516ce 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -29,7 +29,7 @@ steps:
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 
       - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
-      - ".buildkite/scripts/steps/sonar-qube.sh"
+      - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
     notify:

From d995d162c66033fbd6705b5d0ed7fc7e6dfe8a21 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 10:55:36 +0200
Subject: [PATCH 08/38] add env variables

---
 .buildkite/pipelines/sonarqube.yml.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 14d5b516ce..2def3106cd 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -15,6 +15,9 @@ steps:
     depends_on: "check_style"
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:29"
+    env:
+      - PATH: /usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+      - CPP_SRC_HOME: /ml-cpp
     command:
       - "cmake CMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
     artifact_paths:

From b53c8b6b5041c5f6cffe6a165eef8a32dd6955cf Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:03:38 +0200
Subject: [PATCH 09/38] fix env config error

---
 .buildkite/pipelines/sonarqube.yml.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 2def3106cd..d361dc9483 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -12,12 +12,12 @@ cat <<EOL
 steps:
   - label: "Generate compile_commands.json"
     key: "export_compile_commands"
-    depends_on: "check_style"
+    # depends_on: "check_style"
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:29"
     env:
-      - PATH: /usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-      - CPP_SRC_HOME: /ml-cpp
+      PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+      CPP_SRC_HOME: "/ml-cpp"
     command:
       - "cmake CMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
     artifact_paths:

From a5e498cf06d8636d6e36437ac3c948d6a945efa8 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:10:20 +0200
Subject: [PATCH 10/38] comment out other steps

---
 .buildkite/branch.json.py | 48 +++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index dffe2500f5..afa9e59409 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -23,35 +23,35 @@ def main():
     pipeline = {}
 
     pipeline_steps = step.PipelineStep([])
-    pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
-                                                       ".buildkite/pipelines/send_slack_notification.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
-                                                       ".buildkite/pipelines/send_email_notification.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
-                                                       ".buildkite/pipelines/format_and_validation.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
+    #                                                    ".buildkite/pipelines/send_slack_notification.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
+    #                                                    ".buildkite/pipelines/send_email_notification.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
+    #                                                    ".buildkite/pipelines/format_and_validation.yml.sh"))
     # Only create the SonarQube step for the snapshot builds
     if  os.environ.get("BUILD_SNAPSHOT", "") == "true":
         pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
                                                         ".buildkite/pipelines/sonarqube.yml.sh"))
-    config = buildConfig.Config()
-    config.parse()
-    if config.build_windows:
-        build_windows = pipeline_steps.generate_step_template("Windows", "build")
-        pipeline_steps.append(build_windows)
-    if config.build_macos:
-        build_macos = pipeline_steps.generate_step_template("MacOS", "build")
-        pipeline_steps.append(build_macos)
-    if config.build_linux:
-        build_linux = pipeline_steps.generate_step_template("Linux", "build")
-        pipeline_steps.append(build_linux)
+    # config = buildConfig.Config()
+    # config.parse()
+    # if config.build_windows:
+    #     build_windows = pipeline_steps.generate_step_template("Windows", "build")
+    #     pipeline_steps.append(build_windows)
+    # if config.build_macos:
+    #     build_macos = pipeline_steps.generate_step_template("MacOS", "build")
+    #     pipeline_steps.append(build_macos)
+    # if config.build_linux:
+    #     build_linux = pipeline_steps.generate_step_template("Linux", "build")
+    #     pipeline_steps.append(build_linux)
 
-    # Build the DRA artifacts and upload to S3 and GCS
-    pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
-                                                       ".buildkite/pipelines/create_dra.yml.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
-                                                       ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
-                                                       ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
+    # # Build the DRA artifacts and upload to S3 and GCS
+    # pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
+    #                                                    ".buildkite/pipelines/create_dra.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
+    #                                                    ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
+    #                                                    ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
 
     pipeline["steps"] = pipeline_steps
     print(json.dumps(pipeline, indent=2))

From 283ca9e27c5b5593d9f3ce3fa3574bb3ae44a45c Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:13:52 +0200
Subject: [PATCH 11/38] work on compile commands

---
 .buildkite/pipelines/sonarqube.yml.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index d361dc9483..bc56a5e701 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -19,7 +19,8 @@ steps:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
       CPP_SRC_HOME: "/ml-cpp"
     command:
-      - "cmake CMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
+      - "cd $CPP_SRC_HOME"
+      - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
     artifact_paths:
       - "cmake-build-docker/compile_commands.json"
     notify:

From 581a2e3278924ec090a4e533401416e6d8cc7b0b Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:19:53 +0200
Subject: [PATCH 12/38] fixing compile commands

---
 .buildkite/pipelines/sonarqube.yml.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index bc56a5e701..04990f1d05 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -17,9 +17,9 @@ steps:
       image: "docker.elastic.co/ml-dev/ml-linux-build:29"
     env:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-      CPP_SRC_HOME: "/ml-cpp"
+      CPP_SRC_HOME: "/root/ml-cpp"
     command:
-      - "cd $CPP_SRC_HOME"
+      - "cd $(CPP_SRC_HOME)"
       - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
     artifact_paths:
       - "cmake-build-docker/compile_commands.json"

From a817dafad8fe4b5f3a3d354b74844e80f992c72c Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:26:33 +0200
Subject: [PATCH 13/38] set workdir explicitly

---
 .buildkite/pipelines/sonarqube.yml.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 04990f1d05..199dcfb6b8 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -19,10 +19,10 @@ steps:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
       CPP_SRC_HOME: "/root/ml-cpp"
     command:
-      - "cd $(CPP_SRC_HOME)"
+      - "cd /root/ml-cpp"
       - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
     artifact_paths:
-      - "cmake-build-docker/compile_commands.json"
+      - "/root/ml-cpp/cmake-build-docker/compile_commands.json"
     notify:
       - github_commit_status:
           context: "Generate compile_commands.json"
@@ -32,7 +32,8 @@ steps:
     env:
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 
-      - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
+      - "buildkite-agent artifact download cmake-build-docker/compile_commands.json /root/ml-cpp/compile_commands.json"
+      - "cd /root/ml-cpp"
       - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"

From 8f55ace6bb2c1f2849e73ac0084988438c84a286 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:32:57 +0200
Subject: [PATCH 14/38] fixing compile commands step

---
 .buildkite/pipelines/sonarqube.yml.sh | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 199dcfb6b8..42dadd7b4e 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -17,12 +17,10 @@ steps:
       image: "docker.elastic.co/ml-dev/ml-linux-build:29"
     env:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-      CPP_SRC_HOME: "/root/ml-cpp"
     command:
-      - "cd /root/ml-cpp"
       - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
     artifact_paths:
-      - "/root/ml-cpp/cmake-build-docker/compile_commands.json"
+      - "cmake-build-docker/compile_commands.json"
     notify:
       - github_commit_status:
           context: "Generate compile_commands.json"
@@ -32,8 +30,7 @@ steps:
     env:
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 
-      - "buildkite-agent artifact download cmake-build-docker/compile_commands.json /root/ml-cpp/compile_commands.json"
-      - "cd /root/ml-cpp"
+      - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
       - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"

From ecd006e3acfed97d818fb07174f018bee80c2d16 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 11:45:21 +0200
Subject: [PATCH 15/38] correct path to compile_commands.json

---
 sonar-project.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sonar-project.properties b/sonar-project.properties
index d5f70e7c0d..7c2bd1fd0a 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -3,7 +3,7 @@ sonar.projectKey=elastic_ml-cpp
 
 # Encoding of the source code. Default is default system encoding
 sonar.sourceEncoding=UTF-8
-sonar.cfamily.compile-commands=compile_commands.json
+sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
 sonar.sources=./lib, ./bin, ./include

From beac4f8d81efe26398027a183e94582f09d74491 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 12:00:17 +0200
Subject: [PATCH 16/38] make compile_commands with relative paths

---
 .buildkite/pipelines/sonarqube.yml.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 42dadd7b4e..72440d0e5d 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -19,6 +19,7 @@ steps:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
     command:
       - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
+      - "cat cmake-build-docker/compile_commands.json | sed \"s|\\$(pwd)|.|g\" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json"
     artifact_paths:
       - "cmake-build-docker/compile_commands.json"
     notify:

From 00aa58637d24a3db793f2f2a8733a7334a0d6d8a Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 12:31:44 +0200
Subject: [PATCH 17/38] wip

---
 .buildkite/pipelines/sonarqube.yml.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 72440d0e5d..ae4e84e732 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -19,7 +19,7 @@ steps:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
     command:
       - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
-      - "cat cmake-build-docker/compile_commands.json | sed \"s|\\$(pwd)|.|g\" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json"
+      - "cat cmake-build-docker/compile_commands.json | sed 's|$(pwd)|.|g' > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json"
     artifact_paths:
       - "cmake-build-docker/compile_commands.json"
     notify:

From 20041e6ef71519ab133e2284436696bd58eae8cd Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:08:32 +0200
Subject: [PATCH 18/38] introduce export_compile_commands.sh

---
 .buildkite/pipelines/sonarqube.yml.sh            |  4 +---
 .../scripts/steps/export_compile_commands.sh     | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100755 .buildkite/scripts/steps/export_compile_commands.sh

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index ae4e84e732..7589bf41c7 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -17,9 +17,7 @@ steps:
       image: "docker.elastic.co/ml-dev/ml-linux-build:29"
     env:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    command:
-      - "cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker"
-      - "cat cmake-build-docker/compile_commands.json | sed 's|$(pwd)|.|g' > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json"
+    command: ".buildkite/scripts/steps/export_compile_commands.sh"
     artifact_paths:
       - "cmake-build-docker/compile_commands.json"
     notify:
diff --git a/.buildkite/scripts/steps/export_compile_commands.sh b/.buildkite/scripts/steps/export_compile_commands.sh
new file mode 100755
index 0000000000..41ec1744b6
--- /dev/null
+++ b/.buildkite/scripts/steps/export_compile_commands.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+
+set -euo pipefail
+
+cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
+
+cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
\ No newline at end of file

From 108cb52a9a85c790f6d8f3cc242168ec3c268771 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 31 Jul 2024 11:13:08 +0200
Subject: [PATCH 19/38] copy compile_commands to workdir root

---
 .buildkite/pipelines/sonarqube.yml.sh | 1 +
 sonar-project.properties              | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 7589bf41c7..65e192b315 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -30,6 +30,7 @@ steps:
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 
       - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
+      - "cp cmake-build-docker/compile_commands.json ."
       - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
diff --git a/sonar-project.properties b/sonar-project.properties
index 7c2bd1fd0a..d5f70e7c0d 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -3,7 +3,7 @@ sonar.projectKey=elastic_ml-cpp
 
 # Encoding of the source code. Default is default system encoding
 sonar.sourceEncoding=UTF-8
-sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
+sonar.cfamily.compile-commands=compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
 sonar.sources=./lib, ./bin, ./include

From f5938a77b5a67ddbc546725728383589abfefc0b Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 31 Jul 2024 11:33:03 +0200
Subject: [PATCH 20/38] replace compiler string

---
 .buildkite/pipelines/sonarqube.yml.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 65e192b315..9c2f42d23c 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -31,6 +31,7 @@ steps:
     command: 
       - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
       - "cp cmake-build-docker/compile_commands.json ."
+      - "sed -i '' 's|/usr/local/gcc103/bin/g++|g++|g' compile_commands.json"
       - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"

From 8e70cf1ca5d19f7f985a2264ddef3dc1a32f928a Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 31 Jul 2024 11:41:00 +0200
Subject: [PATCH 21/38] replace g++ path

---
 .buildkite/pipelines/sonarqube.yml.sh               | 1 -
 .buildkite/scripts/steps/export_compile_commands.sh | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 9c2f42d23c..65e192b315 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -31,7 +31,6 @@ steps:
     command: 
       - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
       - "cp cmake-build-docker/compile_commands.json ."
-      - "sed -i '' 's|/usr/local/gcc103/bin/g++|g++|g' compile_commands.json"
       - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
diff --git a/.buildkite/scripts/steps/export_compile_commands.sh b/.buildkite/scripts/steps/export_compile_commands.sh
index 41ec1744b6..ce1826d527 100755
--- a/.buildkite/scripts/steps/export_compile_commands.sh
+++ b/.buildkite/scripts/steps/export_compile_commands.sh
@@ -13,4 +13,5 @@ set -euo pipefail
 
 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
 
-cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
\ No newline at end of file
+cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
+sed -i '' 's|/usr/local/gcc103/bin/g++|g++|g' cmake-build-docker/compile_commands.json

From 5635ef75cb196a0d373365eb9e9dd09982863f67 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 31 Jul 2024 11:46:15 +0200
Subject: [PATCH 22/38] fixing script

---
 .buildkite/scripts/steps/export_compile_commands.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/export_compile_commands.sh b/.buildkite/scripts/steps/export_compile_commands.sh
index ce1826d527..9dde3899d0 100755
--- a/.buildkite/scripts/steps/export_compile_commands.sh
+++ b/.buildkite/scripts/steps/export_compile_commands.sh
@@ -14,4 +14,4 @@ set -euo pipefail
 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
 
 cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
-sed -i '' 's|/usr/local/gcc103/bin/g++|g++|g' cmake-build-docker/compile_commands.json
+sed -i "s|/usr/local/gcc103/bin/g++|g++|g" cmake-build-docker/compile_commands.json

From 62e9ae44a415be00782cdfa855351702980dbfe9 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Wed, 31 Jul 2024 11:58:55 +0200
Subject: [PATCH 23/38] replace absolute path in compile_commands

---
 .buildkite/pipelines/sonarqube.yml.sh              |  2 +-
 .../scripts/steps/update_compile_commands.sh       | 14 ++++++++++++++
 sonar-project.properties                           |  2 +-
 3 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100755 .buildkite/scripts/steps/update_compile_commands.sh

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 65e192b315..2ef286bc21 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -30,7 +30,7 @@ steps:
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 
       - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
-      - "cp cmake-build-docker/compile_commands.json ."
+      - ".buildkite/scripts/steps/update_compile_commands.sh"
       - "/scan-source-code.sh"
     agents:
       image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
diff --git a/.buildkite/scripts/steps/update_compile_commands.sh b/.buildkite/scripts/steps/update_compile_commands.sh
new file mode 100755
index 0000000000..aa8e324be3
--- /dev/null
+++ b/.buildkite/scripts/steps/update_compile_commands.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+
+set -euo pipefail
+
+sed -i "s|./|$(pwd)/|g" cmake-build-docker/compile_commands.json
\ No newline at end of file
diff --git a/sonar-project.properties b/sonar-project.properties
index d5f70e7c0d..7c2bd1fd0a 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -3,7 +3,7 @@ sonar.projectKey=elastic_ml-cpp
 
 # Encoding of the source code. Default is default system encoding
 sonar.sourceEncoding=UTF-8
-sonar.cfamily.compile-commands=compile_commands.json
+sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
 sonar.sources=./lib, ./bin, ./include

From db962d951711cc60bac2bda3bf74666e4076b8d6 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 2 Sep 2024 12:16:51 +0200
Subject: [PATCH 24/38] merge compile commands and sonar-scanner

---
 .buildkite/pipelines/sonarqube.yml.sh         |  39 ++---
 .../scripts/steps/export_compile_commands.sh  |  17 --
 .buildkite/scripts/steps/run_sonar-scanner.sh | 148 ++++++++++++++++++
 .../scripts/steps/update_compile_commands.sh  |  14 --
 sonar-project.properties                      |  11 +-
 5 files changed, 174 insertions(+), 55 deletions(-)
 delete mode 100755 .buildkite/scripts/steps/export_compile_commands.sh
 create mode 100755 .buildkite/scripts/steps/run_sonar-scanner.sh
 delete mode 100755 .buildkite/scripts/steps/update_compile_commands.sh

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 2ef286bc21..e157683c8b 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -14,27 +14,28 @@ steps:
     key: "export_compile_commands"
     # depends_on: "check_style"
     agents:
-      image: "docker.elastic.co/ml-dev/ml-linux-build:29"
+      image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
-      PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    command: ".buildkite/scripts/steps/export_compile_commands.sh"
-    artifact_paths:
-      - "cmake-build-docker/compile_commands.json"
+      - PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+      - VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
+    command: ".buildkite/scripts/steps/run_sonar-scanner.sh"
+    # artifact_paths:
+    #   - "cmake-build-docker/compile_commands.json"
     notify:
       - github_commit_status:
           context: "Generate compile_commands.json"
-  - label: "Static code analysis with SonarQube :sonarqube:"
-    key: "sonarqube"
-    depends_on: "export_compile_commands"
-    env:
-      VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
-    command: 
-      - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
-      - ".buildkite/scripts/steps/update_compile_commands.sh"
-      - "/scan-source-code.sh"
-    agents:
-      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-    notify:
-      - github_commit_status:
-          context: "Static code analysis with SonarQube"
+  # - label: "Static code analysis with SonarQube :sonarqube:"
+  #   key: "sonarqube"
+  #   depends_on: "export_compile_commands"
+  #   env:
+  #     VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
+  #   command: 
+  #     - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
+  #     - ".buildkite/scripts/steps/update_compile_commands.sh"
+  #     - "/scan-source-code.sh"
+  #   agents:
+  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+  #   notify:
+  #     - github_commit_status:
+  #         context: "Static code analysis with SonarQube"
 EOL
\ No newline at end of file
diff --git a/.buildkite/scripts/steps/export_compile_commands.sh b/.buildkite/scripts/steps/export_compile_commands.sh
deleted file mode 100755
index 9dde3899d0..0000000000
--- a/.buildkite/scripts/steps/export_compile_commands.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0 and the following additional limitation. Functionality enabled by the
-# files subject to the Elastic License 2.0 may only be used in production when
-# invoked by an Elasticsearch process with a license key installed that permits
-# use of machine learning features. You may not use this file except in
-# compliance with the Elastic License 2.0 and the foregoing additional
-# limitation.
-#
-
-set -euo pipefail
-
-cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
-
-cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
-sed -i "s|/usr/local/gcc103/bin/g++|g++|g" cmake-build-docker/compile_commands.json
diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
new file mode 100755
index 0000000000..37f124e6e7
--- /dev/null
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+
+set -euo pipefail
+
+export CWD="$(pwd)"
+
+# Install the sonar-scanner
+cd /usr/local
+curl -O https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.1.0.4477-linux-x64.zip
+unzip sonar-scanner-cli-6.1.0.4477-linux-x64.zip
+mv sonar-scanner-6.1.0.4477-linux-x64/ sonar-scanner
+export PATH=$(pwd):$PATH
+
+# Generate the compile_commands.json file
+
+cd "${CWD}"
+
+cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
+
+# Analyze the source code with SonarQube
+
+trap 'error_handler $?' EXIT
+
+retries=0
+
+slack_notification() {
+  local err
+  err=$1
+
+  if [[ "${BUILDKITE}" == "true" ]]; then
+    echo "Sending Slack notification that Sonar Scanner step failed with exit code ${err}. Please retry the other step and not this one."
+    cat <<-HEREDOC | buildkite-agent pipeline upload
+    steps:
+      - label: ":slack: Slack Notification - Sonar Scanner Failed"
+        command: "echo 'Sonar Scanner failed with exit code ${err}'"
+        soft_fail: false
+        agents:
+          image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent"
+        notify:
+          - slack:
+              channels:
+                - "#sonar"
+              message: "Sonar Scanner failed with exit code ${err} for attempt ${retries} on ${BUILDKITE_PIPELINE_SLUG}"
+HEREDOC
+  else
+    echo "Sonar Scanner failed with exit code ${err}"
+  fi
+}
+
+error_handler() {
+  local err
+  err=$1
+
+  # Since we need to trap the sonar-scanner script exit code, we don't generate notifications on success.
+  if [[ $err -eq 0 ]]; then
+    exit "$err"
+  fi
+  
+  slack_notification "$err"
+
+  exit "$err"
+}
+
+runScanner(){
+  local max_retries=5
+  local retry_delay=30
+  local exit_code=-1 # Initialize to a negative value to allow while loop to run.
+
+  while [[ $retries -lt $max_retries ]] && [[ $exit_code -ne 0 ]]; do
+    set +e  # Temporarily disable exit immediately on error to allow retries
+    if [[ ${BUILDKITE_PULL_REQUEST} =~ ^[0-9]+$ ]];
+    then
+      echo "Spotted PR"
+      sonar-scanner -Dsonar.token="${SONAR_LOGIN}" \
+                    -Dsonar.pullrequest.key="${BUILDKITE_PULL_REQUEST}" \
+                    -Dsonar.pullrequest.branch="${BUILDKITE_BRANCH}" \
+                    -Dsonar.pullrequest.base="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" \
+                    -Dsonar.projectVersion="${BUILDKITE_COMMIT}" \
+                    -Dsonar.scm.provider=git 
+      exit_code=$?
+    else
+      sonar-scanner -Dsonar.token="${SONAR_LOGIN}"  \
+                    -Dsonar.branch.name="${BUILDKITE_BRANCH}" \
+                    -Dsonar.projectVersion="${BUILDKITE_COMMIT}" \
+                    -Dsonar.scm.provider=git 
+      exit_code=$?
+    fi
+    set -e # Re-enable exit immediately on error
+ 
+    if [[ $exit_code -ne 0 ]]; then
+      retries=$((retries + 1))
+      echo "Sonar Scanner failed with exit code ${exit_code}. Retrying in ${retry_delay} seconds..."
+      # Only send slack notification on first and last retry to avoid spamming the channel.
+      if [[ $retries -eq 1 ]] || [[ $retries -eq $max_retries ]]; then
+        slack_notification "${exit_code}"
+      fi
+      sleep $retry_delay
+      retry_delay=$((retry_delay * retries))
+    fi
+  done
+
+  return "$exit_code"
+}
+
+# Check if we are in a git repo
+git rev-parse --is-inside-work-tree >/dev/null
+
+# SonarQube project analyse token was provided
+if [[ -z "${SONAR_LOGIN}" ]]; then
+  echo "No SONAR_LOGIN token was provided, attempting to resolve it via vault..."
+
+  if [[ -z "${VAULT_ADDR}" ]];
+  then
+      echo "VAULT_ADDR is missing."
+      exit 1
+  fi
+  if [[ -z "${VAULT_TOKEN}" ]];
+  then
+      echo "A VAULT_TOKEN is missing for ${VAULT_ADDR}."
+      exit 1
+  fi
+  if [[ -z "${VAULT_SONAR_TOKEN_PATH}" ]];
+  then
+      echo "VAULT_SONAR_TOKEN_PATH is missing."
+      exit 1
+  fi
+
+  if [[ "$VAULT_SONAR_TOKEN_PATH" =~ ^kv/* ]];
+  then
+    SONAR_LOGIN=$(vault kv get --field token "${VAULT_SONAR_TOKEN_PATH}")
+  else
+    SONAR_LOGIN=$(vault read --field token "${VAULT_SONAR_TOKEN_PATH}")
+  fi
+fi
+
+runScanner
+
+# cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
+# sed -i "s|/usr/local/gcc103/bin/g++|g++|g" cmake-build-docker/compile_commands.json
diff --git a/.buildkite/scripts/steps/update_compile_commands.sh b/.buildkite/scripts/steps/update_compile_commands.sh
deleted file mode 100755
index aa8e324be3..0000000000
--- a/.buildkite/scripts/steps/update_compile_commands.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0 and the following additional limitation. Functionality enabled by the
-# files subject to the Elastic License 2.0 may only be used in production when
-# invoked by an Elasticsearch process with a license key installed that permits
-# use of machine learning features. You may not use this file except in
-# compliance with the Elastic License 2.0 and the foregoing additional
-# limitation.
-#
-
-set -euo pipefail
-
-sed -i "s|./|$(pwd)/|g" cmake-build-docker/compile_commands.json
\ No newline at end of file
diff --git a/sonar-project.properties b/sonar-project.properties
index 7c2bd1fd0a..f8ca0ddd48 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -1,13 +1,14 @@
 # must be unique in a given SonarQube instance
-sonar.projectKey=elastic_ml-cpp
+sonar.projectKey=elastic_ml-cpp_271ade36-31fc-4c6b-966e-80245560ad14
 
 # Encoding of the source code. Default is default system encoding
 sonar.sourceEncoding=UTF-8
 sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
-sonar.sources=./lib, ./bin, ./include
-sonar.language=c,cpp
-sonar.inclusions = **/*.cc, **/*.h
+sonar.sources= ./bin
+sonar.language=cpp
+sonar.inclusions = **/*.cc,**/*.h
+sonar.lang.patterns.cpp=**/*.cc,**/*.h
+sonar.lang.patterns.c=**/*.c
 
-sonar.cfamily.reportingCppStandardOverride=c++14

From 2e21541f74ec88333aa13764f1440c24300e1aca Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:03:40 +0200
Subject: [PATCH 25/38] fix env formatting

---
 .buildkite/pipelines/sonarqube.yml.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index e157683c8b..307dfd00c9 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -16,8 +16,8 @@ steps:
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
-      - PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-      - VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
+      PATH="/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+      VAULT_SONAR_TOKEN_PATH="secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: ".buildkite/scripts/steps/run_sonar-scanner.sh"
     # artifact_paths:
     #   - "cmake-build-docker/compile_commands.json"

From ea47e99bc78d48ce7e4d9d1982568df98aab2121 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:13:11 +0200
Subject: [PATCH 26/38] fix environment

---
 .buildkite/pipelines/sonarqube.yml.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 307dfd00c9..f85fa47fa5 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -16,8 +16,8 @@ steps:
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
-      PATH="/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-      VAULT_SONAR_TOKEN_PATH="secret/ci/elastic-ml-cpp/sonar-analyze-token"
+      PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+      VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: ".buildkite/scripts/steps/run_sonar-scanner.sh"
     # artifact_paths:
     #   - "cmake-build-docker/compile_commands.json"

From d91b542dfd531f73e69594d3c994b977dd26800e Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:25:22 +0200
Subject: [PATCH 27/38] fix unbound variable error

---
 .buildkite/scripts/steps/run_sonar-scanner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
index 37f124e6e7..546c9335bd 100755
--- a/.buildkite/scripts/steps/run_sonar-scanner.sh
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -9,7 +9,7 @@
 # limitation.
 #
 
-set -euo pipefail
+set -eo pipefail
 
 export CWD="$(pwd)"
 

From 7cf23cb59fcc6d3f4e7c8c49770ba261d463738b Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:39:52 +0200
Subject: [PATCH 28/38] fix path issue

---
 .buildkite/scripts/steps/run_sonar-scanner.sh | 36 ++-----------------
 1 file changed, 2 insertions(+), 34 deletions(-)

diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
index 546c9335bd..57f30d49c3 100755
--- a/.buildkite/scripts/steps/run_sonar-scanner.sh
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -18,7 +18,7 @@ cd /usr/local
 curl -O https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.1.0.4477-linux-x64.zip
 unzip sonar-scanner-cli-6.1.0.4477-linux-x64.zip
 mv sonar-scanner-6.1.0.4477-linux-x64/ sonar-scanner
-export PATH=$(pwd):$PATH
+export PATH=$(pwd)/sonar-scanner/bin:$PATH
 
 # Generate the compile_commands.json file
 
@@ -32,30 +32,6 @@ trap 'error_handler $?' EXIT
 
 retries=0
 
-slack_notification() {
-  local err
-  err=$1
-
-  if [[ "${BUILDKITE}" == "true" ]]; then
-    echo "Sending Slack notification that Sonar Scanner step failed with exit code ${err}. Please retry the other step and not this one."
-    cat <<-HEREDOC | buildkite-agent pipeline upload
-    steps:
-      - label: ":slack: Slack Notification - Sonar Scanner Failed"
-        command: "echo 'Sonar Scanner failed with exit code ${err}'"
-        soft_fail: false
-        agents:
-          image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent"
-        notify:
-          - slack:
-              channels:
-                - "#sonar"
-              message: "Sonar Scanner failed with exit code ${err} for attempt ${retries} on ${BUILDKITE_PIPELINE_SLUG}"
-HEREDOC
-  else
-    echo "Sonar Scanner failed with exit code ${err}"
-  fi
-}
-
 error_handler() {
   local err
   err=$1
@@ -64,8 +40,6 @@ error_handler() {
   if [[ $err -eq 0 ]]; then
     exit "$err"
   fi
-  
-  slack_notification "$err"
 
   exit "$err"
 }
@@ -99,10 +73,7 @@ runScanner(){
     if [[ $exit_code -ne 0 ]]; then
       retries=$((retries + 1))
       echo "Sonar Scanner failed with exit code ${exit_code}. Retrying in ${retry_delay} seconds..."
-      # Only send slack notification on first and last retry to avoid spamming the channel.
-      if [[ $retries -eq 1 ]] || [[ $retries -eq $max_retries ]]; then
-        slack_notification "${exit_code}"
-      fi
+
       sleep $retry_delay
       retry_delay=$((retry_delay * retries))
     fi
@@ -143,6 +114,3 @@ if [[ -z "${SONAR_LOGIN}" ]]; then
 fi
 
 runScanner
-
-# cat cmake-build-docker/compile_commands.json | sed "s|$(pwd)|.|g" > compile_commands.json.tmp && mv compile_commands.json.tmp cmake-build-docker/compile_commands.json
-# sed -i "s|/usr/local/gcc103/bin/g++|g++|g" cmake-build-docker/compile_commands.json

From 9f6b39f4fa3a56cfab1b6c354096cc6328e98a1b Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:48:21 +0200
Subject: [PATCH 29/38] clean up

---
 .buildkite/branch.json.py                     | 48 +++++++++----------
 .buildkite/scripts/steps/run_sonar-scanner.sh |  2 +
 sonar-project.properties                      |  2 +-
 3 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index c1f96b3b11..8c0458a425 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -23,35 +23,35 @@ def main():
     pipeline = {}
 
     pipeline_steps = step.PipelineStep([])
-    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
-    #                                                    ".buildkite/pipelines/send_slack_notification.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
-    #                                                    ".buildkite/pipelines/send_email_notification.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
-    #                                                    ".buildkite/pipelines/format_and_validation.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
+                                                       ".buildkite/pipelines/send_slack_notification.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
+                                                       ".buildkite/pipelines/send_email_notification.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
+                                                       ".buildkite/pipelines/format_and_validation.yml.sh"))
     # Only create the SonarQube step for the snapshot builds
     if  os.environ.get("BUILD_SNAPSHOT", "") == "true":
         pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
                                                         ".buildkite/pipelines/sonarqube.yml.sh"))
-    # config = buildConfig.Config()
-    # config.parse()
-    # if config.build_windows:
-    #     build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64)
-    #     pipeline_steps.append(build_windows)
-    # if config.build_macos:
-    #     build_macos = pipeline_steps.generate_step_template("MacOS", "build", config.build_aarch64, config.build_x86_64)
-    #     pipeline_steps.append(build_macos)
-    # if config.build_linux:
-    #     build_linux = pipeline_steps.generate_step_template("Linux", "build", config.build_aarch64, config.build_x86_64)
-    #     pipeline_steps.append(build_linux)
+    config = buildConfig.Config()
+    config.parse()
+    if config.build_windows:
+        build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64)
+        pipeline_steps.append(build_windows)
+    if config.build_macos:
+        build_macos = pipeline_steps.generate_step_template("MacOS", "build", config.build_aarch64, config.build_x86_64)
+        pipeline_steps.append(build_macos)
+    if config.build_linux:
+        build_linux = pipeline_steps.generate_step_template("Linux", "build", config.build_aarch64, config.build_x86_64)
+        pipeline_steps.append(build_linux)
 
-    # # Build the DRA artifacts and upload to S3 and GCS
-    # pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
-    #                                                    ".buildkite/pipelines/create_dra.yml.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
-    #                                                    ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
-    #                                                    ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
+    # Build the DRA artifacts and upload to S3 and GCS
+    pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
+                                                       ".buildkite/pipelines/create_dra.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
+                                                       ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
+                                                       ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
 
     pipeline["steps"] = pipeline_steps
     print(json.dumps(pipeline, indent=2))
diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
index 57f30d49c3..032c3c9c0e 100755
--- a/.buildkite/scripts/steps/run_sonar-scanner.sh
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -27,6 +27,8 @@ cd "${CWD}"
 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
 
 # Analyze the source code with SonarQube
+# The following script was adapted from the Elastic CI team's script:
+# https://github.com/elastic/sonarqube/blob/main/buildkite-scanner/scan-source-code.sh
 
 trap 'error_handler $?' EXIT
 
diff --git a/sonar-project.properties b/sonar-project.properties
index f8ca0ddd48..99077f1bbd 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -6,7 +6,7 @@ sonar.sourceEncoding=UTF-8
 sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
-sonar.sources= ./bin
+sonar.sources= ./bin ./lib ./include
 sonar.language=cpp
 sonar.inclusions = **/*.cc,**/*.h
 sonar.lang.patterns.cpp=**/*.cc,**/*.h

From 146171a5b7166343d00a1a4577d0e6f9eb9d32ad Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:45:35 +0000
Subject: [PATCH 30/38] code coverage

---
 .buildkite/branch.json.py                     | 40 +++++++++----------
 .buildkite/pipelines/build_linux.json.py      |  8 +++-
 .buildkite/pipelines/sonarqube.yml.sh         | 24 +++--------
 .../scripts/steps/generate_coverage_report.sh | 17 ++++++++
 .buildkite/scripts/steps/run_sonar-scanner.sh | 13 +++++-
 generate_coverage.sh                          |  6 +++
 sonar-project.properties                      |  9 ++++-
 7 files changed, 75 insertions(+), 42 deletions(-)
 create mode 100755 .buildkite/scripts/steps/generate_coverage_report.sh
 create mode 100755 generate_coverage.sh

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index 8c0458a425..c8ab346d6e 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -23,35 +23,35 @@ def main():
     pipeline = {}
 
     pipeline_steps = step.PipelineStep([])
-    pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
-                                                       ".buildkite/pipelines/send_slack_notification.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
-                                                       ".buildkite/pipelines/send_email_notification.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
-                                                       ".buildkite/pipelines/format_and_validation.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
+    #                                                    ".buildkite/pipelines/send_slack_notification.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
+    #                                                    ".buildkite/pipelines/send_email_notification.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
+    #                                                    ".buildkite/pipelines/format_and_validation.yml.sh"))
     # Only create the SonarQube step for the snapshot builds
     if  os.environ.get("BUILD_SNAPSHOT", "") == "true":
         pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
                                                         ".buildkite/pipelines/sonarqube.yml.sh"))
     config = buildConfig.Config()
     config.parse()
-    if config.build_windows:
-        build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64)
-        pipeline_steps.append(build_windows)
-    if config.build_macos:
-        build_macos = pipeline_steps.generate_step_template("MacOS", "build", config.build_aarch64, config.build_x86_64)
-        pipeline_steps.append(build_macos)
+    # if config.build_windows:
+    #     build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64)
+    #     pipeline_steps.append(build_windows)
+    # if config.build_macos:
+    #     build_macos = pipeline_steps.generate_step_template("MacOS", "build", config.build_aarch64, config.build_x86_64)
+    #     pipeline_steps.append(build_macos)
     if config.build_linux:
-        build_linux = pipeline_steps.generate_step_template("Linux", "build", config.build_aarch64, config.build_x86_64)
+        build_linux = pipeline_steps.generate_step_template("Linux", "build", "", config.build_x86_64)
         pipeline_steps.append(build_linux)
 
-    # Build the DRA artifacts and upload to S3 and GCS
-    pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
-                                                       ".buildkite/pipelines/create_dra.yml.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
-                                                       ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
-                                                       ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
+    # # Build the DRA artifacts and upload to S3 and GCS
+    # pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
+    #                                                    ".buildkite/pipelines/create_dra.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
+    #                                                    ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
+    #                                                    ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
 
     pipeline["steps"] = pipeline_steps
     print(json.dumps(pipeline, indent=2))
diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py
index 8416a9a042..9344387151 100755
--- a/.buildkite/pipelines/build_linux.json.py
+++ b/.buildkite/pipelines/build_linux.json.py
@@ -63,7 +63,7 @@ def main(args):
                   f'if [[ "{args.action}" == "debug" ]]; then export ML_DEBUG=1; fi',
                   ".buildkite/scripts/steps/build_and_test.sh"
                 ],
-                "depends_on": "check_style",
+                # "depends_on": "check_style",
                 "key": f"build_test_linux-{arch}-{build_type}",
                 "env": {
                   "ML_DEBUG": "0",
@@ -87,6 +87,12 @@ def main(args):
                   },
                 ],
             })
+        # For linux x86_64 additionally generate the test coverage report
+        if args.build_x86_64 and arch == "x86_64":
+            last_step = pipeline_steps[-1]
+            last_step["commands"].append(".buildkite/scripts/steps/generate_coverage_report.sh")
+            last_step["artifact_paths"] = "*/**/unittest/boost_test_results.junit,gcov.tar.gz"
+            last_step["env"]["CODE_COVERAGE"] = "ON"
 
     # Never cross-compile for linux-aarch64 in the nightly debug build.
     if os.environ.get("BUILDKITE_PIPELINE_SLUG", "ml-cpp-pr-builds") != "ml-cpp-debug-build" and \
diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index f85fa47fa5..263ce939ce 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -10,32 +10,20 @@
 
 cat <<EOL
 steps:
-  - label: "Generate compile_commands.json"
+  - label: "Run SonarQube scanner :sonarqube:"
     key: "export_compile_commands"
-    # depends_on: "check_style"
+    depends_on: "build_test_linux-x86_64-RelWithDebInfo"
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
-    command: ".buildkite/scripts/steps/run_sonar-scanner.sh"
+    command: 
+      - "buildkite-agent artifact download gcov.tar.gz ."
+      - ".buildkite/scripts/steps/run_sonar-scanner.sh"
     # artifact_paths:
     #   - "cmake-build-docker/compile_commands.json"
     notify:
       - github_commit_status:
-          context: "Generate compile_commands.json"
-  # - label: "Static code analysis with SonarQube :sonarqube:"
-  #   key: "sonarqube"
-  #   depends_on: "export_compile_commands"
-  #   env:
-  #     VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
-  #   command: 
-  #     - "buildkite-agent artifact download cmake-build-docker/compile_commands.json ."
-  #     - ".buildkite/scripts/steps/update_compile_commands.sh"
-  #     - "/scan-source-code.sh"
-  #   agents:
-  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-  #   notify:
-  #     - github_commit_status:
-  #         context: "Static code analysis with SonarQube"
+          context: "Run SonarQube scanner"
 EOL
\ No newline at end of file
diff --git a/.buildkite/scripts/steps/generate_coverage_report.sh b/.buildkite/scripts/steps/generate_coverage_report.sh
new file mode 100755
index 0000000000..e2adac8868
--- /dev/null
+++ b/.buildkite/scripts/steps/generate_coverage_report.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+set -eo pipefail
+
+find ./cmake-build-docker -name "*.gcda" -print0 | xargs -0 -n 1 -P $(nproc) gcov --preserve-paths
+
+# Crate gcov.tar.gz from all the .gcov files in .
+find . -name "*.gcov" -print0 | tar -czf gcov.tar.gz --null -T -
+rm *.gcov
diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
index 032c3c9c0e..66acfb05e5 100755
--- a/.buildkite/scripts/steps/run_sonar-scanner.sh
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -11,17 +11,25 @@
 
 set -eo pipefail
 
+# Unpack gcov.tar.gz if it exists
+if [ -f gcov.tar.gz ]; then
+  echo "Unpacking gcov.tar.gz"
+  tar -xzf gcov.tar.gz
+fi
+
+
 export CWD="$(pwd)"
 
 # Install the sonar-scanner
+echo "Installing sonar-scanner"
 cd /usr/local
 curl -O https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.1.0.4477-linux-x64.zip
-unzip sonar-scanner-cli-6.1.0.4477-linux-x64.zip
+unzip -q sonar-scanner-cli-6.1.0.4477-linux-x64.zip
 mv sonar-scanner-6.1.0.4477-linux-x64/ sonar-scanner
 export PATH=$(pwd)/sonar-scanner/bin:$PATH
 
 # Generate the compile_commands.json file
-
+echo "Generating compile_commands.json"
 cd "${CWD}"
 
 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -B cmake-build-docker
@@ -115,4 +123,5 @@ if [[ -z "${SONAR_LOGIN}" ]]; then
   fi
 fi
 
+echo "Running sonar-scanner"
 runScanner
diff --git a/generate_coverage.sh b/generate_coverage.sh
new file mode 100755
index 0000000000..27bbfbf07b
--- /dev/null
+++ b/generate_coverage.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Find all .gcda files and run gcov on each with --preserve-paths
+find ./cmake-build-relwithdebinfo -name "*.gcda" -exec gcov --preserve-paths {} \;
+
+echo "Coverage data generated."
diff --git a/sonar-project.properties b/sonar-project.properties
index 99077f1bbd..6f1e833352 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -6,9 +6,16 @@ sonar.sourceEncoding=UTF-8
 sonar.cfamily.compile-commands=cmake-build-docker/compile_commands.json
 
 sonar.host.url=https://sonar.elastic.dev
-sonar.sources= ./bin ./lib ./include
+sonar.sources= ./bin,./lib,./include
+sonar.tests=./lib/core/unittest,./lib/ver/unittest,./lib/maths/analytics/unittest,./lib/maths/common/unittest,./lib/maths/time_series/unittest,./lib/seccomp/unittest,./lib/api/unittest,./lib/model/unittest
 sonar.language=cpp
 sonar.inclusions = **/*.cc,**/*.h
+sonar.exclusions = **/unittest/**
 sonar.lang.patterns.cpp=**/*.cc,**/*.h
 sonar.lang.patterns.c=**/*.c
 
+# Exclude coverage files from being analyzed as sources
+sonar.coverage.exclusions=**/unittest/**
+
+# Import coverage report
+sonar.cfamily.gcov.reportsPath=.

From d2b007df12411c0bb07876953e5b2b06d6455f91 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:48:47 +0000
Subject: [PATCH 31/38] cmake changes

---
 cmake/compiler/gcc.cmake | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/cmake/compiler/gcc.cmake b/cmake/compiler/gcc.cmake
index 804bfa0b88..4cae248649 100644
--- a/cmake/compiler/gcc.cmake
+++ b/cmake/compiler/gcc.cmake
@@ -38,6 +38,26 @@ endif()
 
 SET(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -ru <TARGET> <OBJECTS>")
 
+# Set CODE_COVERAGE if it is defined in the environment
+if(DEFINED ENV{CODE_COVERAGE})
+  set(CODE_COVERAGE $ENV{CODE_COVERAGE})
+endif()
+
+option(CODE_COVERAGE "Enable code coverage" OFF)
+
+if(CODE_COVERAGE)
+  message(STATUS "Code coverage enabled.")
+  set(COVERAGE_CXX_FLAGS "-fprofile-arcs -ftest-coverage")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_CXX_FLAGS}")
+#  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COVERAGE_CXX_FLAGS}")
+#  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COVERAGE_CXX_FLAGS}")
+#  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${COVERAGE_CXX_FLAGS}")
+#  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_CXX_FLAGS}")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcov")
+else()
+  message(STATUS "Code coverage disabled.")
+endif()
+
 list(APPEND ML_C_FLAGS
   ${CROSS_FLAGS}
   ${ARCHCFLAGS}

From 2e4d652fa11c8a86640006f34537d095d818bfdf Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 3 Sep 2024 11:03:52 +0000
Subject: [PATCH 32/38] fix coverage files

---
 .buildkite/pipelines/build_linux.json.py             | 2 +-
 .buildkite/scripts/steps/generate_coverage_report.sh | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py
index 9344387151..d35b68e729 100755
--- a/.buildkite/pipelines/build_linux.json.py
+++ b/.buildkite/pipelines/build_linux.json.py
@@ -91,7 +91,7 @@ def main(args):
         if args.build_x86_64 and arch == "x86_64":
             last_step = pipeline_steps[-1]
             last_step["commands"].append(".buildkite/scripts/steps/generate_coverage_report.sh")
-            last_step["artifact_paths"] = "*/**/unittest/boost_test_results.junit,gcov.tar.gz"
+            last_step["artifact_paths"].append("gcov.tar.gz")
             last_step["env"]["CODE_COVERAGE"] = "ON"
 
     # Never cross-compile for linux-aarch64 in the nightly debug build.
diff --git a/.buildkite/scripts/steps/generate_coverage_report.sh b/.buildkite/scripts/steps/generate_coverage_report.sh
index e2adac8868..5e170ab219 100755
--- a/.buildkite/scripts/steps/generate_coverage_report.sh
+++ b/.buildkite/scripts/steps/generate_coverage_report.sh
@@ -10,8 +10,9 @@
 
 set -eo pipefail
 
+echo "Generating coverage report"
 find ./cmake-build-docker -name "*.gcda" -print0 | xargs -0 -n 1 -P $(nproc) gcov --preserve-paths
 
 # Crate gcov.tar.gz from all the .gcov files in .
+echo "Creating gcov.tar.gz"
 find . -name "*.gcov" -print0 | tar -czf gcov.tar.gz --null -T -
-rm *.gcov

From 145597ba8e242d2805532822a629236ba57d1bd3 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Tue, 3 Sep 2024 11:06:49 +0000
Subject: [PATCH 33/38] fix artifact pipeline

---
 .buildkite/pipelines/build_linux.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py
index d35b68e729..cf5779fb49 100755
--- a/.buildkite/pipelines/build_linux.json.py
+++ b/.buildkite/pipelines/build_linux.json.py
@@ -91,7 +91,7 @@ def main(args):
         if args.build_x86_64 and arch == "x86_64":
             last_step = pipeline_steps[-1]
             last_step["commands"].append(".buildkite/scripts/steps/generate_coverage_report.sh")
-            last_step["artifact_paths"].append("gcov.tar.gz")
+            last_step["artifact_paths"] = [last_step["artifact_paths"], "gcov.tar.gz"]
             last_step["env"]["CODE_COVERAGE"] = "ON"
 
     # Never cross-compile for linux-aarch64 in the nightly debug build.

From 7d53f293cbfc2ad06c9dd58b1a9aafdc5f2d1860 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Fri, 6 Sep 2024 14:30:28 +0200
Subject: [PATCH 34/38] enable on PR

---
 .buildkite/branch.json.py                     | 45 +++++++-------
 .buildkite/pipeline.json.py                   | 58 ++++++++++---------
 .buildkite/pipelines/build_linux.json.py      |  8 +--
 .buildkite/pipelines/sonarqube.yml.sh         |  1 +
 .buildkite/scripts/steps/run_sonar-scanner.sh |  5 +-
 cmake/compiler/gcc.cmake                      | 20 -------
 6 files changed, 56 insertions(+), 81 deletions(-)

diff --git a/.buildkite/branch.json.py b/.buildkite/branch.json.py
index c8ab346d6e..3c279e3a93 100755
--- a/.buildkite/branch.json.py
+++ b/.buildkite/branch.json.py
@@ -17,41 +17,36 @@
     step,
     config as buildConfig,
 )
-import os
 
 def main():
     pipeline = {}
 
     pipeline_steps = step.PipelineStep([])
-    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
-    #                                                    ".buildkite/pipelines/send_slack_notification.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
-    #                                                    ".buildkite/pipelines/send_email_notification.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
-    #                                                    ".buildkite/pipelines/format_and_validation.yml.sh"))
-    # Only create the SonarQube step for the snapshot builds
-    if  os.environ.get("BUILD_SNAPSHOT", "") == "true":
-        pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
-                                                        ".buildkite/pipelines/sonarqube.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
+                                                       ".buildkite/pipelines/send_slack_notification.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
+                                                       ".buildkite/pipelines/send_email_notification.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
+                                                       ".buildkite/pipelines/format_and_validation.yml.sh"))
     config = buildConfig.Config()
     config.parse()
-    # if config.build_windows:
-    #     build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64)
-    #     pipeline_steps.append(build_windows)
-    # if config.build_macos:
-    #     build_macos = pipeline_steps.generate_step_template("MacOS", "build", config.build_aarch64, config.build_x86_64)
-    #     pipeline_steps.append(build_macos)
+    if config.build_windows:
+        build_windows = pipeline_steps.generate_step_template("Windows", "build", "", config.build_x86_64)
+        pipeline_steps.append(build_windows)
+    if config.build_macos:
+        build_macos = pipeline_steps.generate_step_template("MacOS", "build", config.build_aarch64, config.build_x86_64)
+        pipeline_steps.append(build_macos)
     if config.build_linux:
-        build_linux = pipeline_steps.generate_step_template("Linux", "build", "", config.build_x86_64)
+        build_linux = pipeline_steps.generate_step_template("Linux", "build", config.build_aarch64, config.build_x86_64)
         pipeline_steps.append(build_linux)
 
-    # # Build the DRA artifacts and upload to S3 and GCS
-    # pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
-    #                                                    ".buildkite/pipelines/create_dra.yml.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
-    #                                                    ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
-    # pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
-    #                                                    ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
+    # Build the DRA artifacts and upload to S3 and GCS
+    pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
+                                                       ".buildkite/pipelines/create_dra.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to S3",
+                                                       ".buildkite/pipelines/upload_dra_to_s3.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Upload daily releasable artifacts to GCS",
+                                                       ".buildkite/pipelines/upload_dra_to_gcs.yml.sh"))
 
     pipeline["steps"] = pipeline_steps
     print(json.dumps(pipeline, indent=2))
diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py
index 1840d5dc9c..eca3517a62 100755
--- a/.buildkite/pipeline.json.py
+++ b/.buildkite/pipeline.json.py
@@ -31,37 +31,39 @@
 def main():
     pipeline = {}
     pipeline_steps = step.PipelineStep([])
-    pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
-                                                       ".buildkite/pipelines/send_slack_notification.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
-                                                       ".buildkite/pipelines/send_email_notification.sh"))
-    pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
-                                                       ".buildkite/pipelines/format_and_validation.yml.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :slack: notification for the pipeline",
+    #                                                    ".buildkite/pipelines/send_slack_notification.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Queue a :email: notification for the pipeline",
+    #                                                    ".buildkite/pipelines/send_email_notification.sh"))
+    # pipeline_steps.append(pipeline_steps.generate_step("Upload clang-format validation",
+    #                                                    ".buildkite/pipelines/format_and_validation.yml.sh"))
+    pipeline_steps.append(pipeline_steps.generate_step("Scan and upload SonarQube report", 
+                                                        ".buildkite/pipelines/sonarqube.yml.sh"))
     config = buildConfig.Config()
     config.parse()
-    if config.build_windows:
-        build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64)
-        pipeline_steps.append(build_windows)
-    if config.build_macos:
-        build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64)
-        pipeline_steps.append(build_macos)
-    if config.build_linux:
-        build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64)
-        pipeline_steps.append(build_linux)
+    # if config.build_windows:
+    #     build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64)
+    #     pipeline_steps.append(build_windows)
+    # if config.build_macos:
+    #     build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64)
+    #     pipeline_steps.append(build_macos)
+    # if config.build_linux:
+    #     build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64)
+    #     pipeline_steps.append(build_linux)
 
-        if config.build_x86_64:
-            pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
-                                                               ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
-            # We only use linux x86_64 builds for QA tests.
-            if config.run_qa_tests:
-                pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline",
-                                                                   ".buildkite/pipelines/run_qa_tests.yml.sh"))
-            if config.run_pytorch_tests:
-                pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline",
-                                                                   ".buildkite/pipelines/run_pytorch_tests.yml.sh"))
-        if config.build_aarch64:
-            pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline",
-                                                               ".buildkite/pipelines/run_es_tests_aarch64.yml.sh"))
+    #     if config.build_x86_64:
+    #         pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
+    #                                                            ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
+    #         # We only use linux x86_64 builds for QA tests.
+    #         if config.run_qa_tests:
+    #             pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline",
+    #                                                                ".buildkite/pipelines/run_qa_tests.yml.sh"))
+    #         if config.run_pytorch_tests:
+    #             pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline",
+    #                                                                ".buildkite/pipelines/run_pytorch_tests.yml.sh"))
+    #     if config.build_aarch64:
+    #         pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline",
+    #                                                            ".buildkite/pipelines/run_es_tests_aarch64.yml.sh"))
 
     pipeline["env"] = env
     pipeline["steps"] = pipeline_steps
diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py
index cf5779fb49..8416a9a042 100755
--- a/.buildkite/pipelines/build_linux.json.py
+++ b/.buildkite/pipelines/build_linux.json.py
@@ -63,7 +63,7 @@ def main(args):
                   f'if [[ "{args.action}" == "debug" ]]; then export ML_DEBUG=1; fi',
                   ".buildkite/scripts/steps/build_and_test.sh"
                 ],
-                # "depends_on": "check_style",
+                "depends_on": "check_style",
                 "key": f"build_test_linux-{arch}-{build_type}",
                 "env": {
                   "ML_DEBUG": "0",
@@ -87,12 +87,6 @@ def main(args):
                   },
                 ],
             })
-        # For linux x86_64 additionally generate the test coverage report
-        if args.build_x86_64 and arch == "x86_64":
-            last_step = pipeline_steps[-1]
-            last_step["commands"].append(".buildkite/scripts/steps/generate_coverage_report.sh")
-            last_step["artifact_paths"] = [last_step["artifact_paths"], "gcov.tar.gz"]
-            last_step["env"]["CODE_COVERAGE"] = "ON"
 
     # Never cross-compile for linux-aarch64 in the nightly debug build.
     if os.environ.get("BUILDKITE_PIPELINE_SLUG", "ml-cpp-pr-builds") != "ml-cpp-debug-build" and \
diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 263ce939ce..03509ffa93 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -13,6 +13,7 @@ steps:
   - label: "Run SonarQube scanner :sonarqube:"
     key: "export_compile_commands"
     depends_on: "build_test_linux-x86_64-RelWithDebInfo"
+    soft_fail: true
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
index 66acfb05e5..f1279481ba 100755
--- a/.buildkite/scripts/steps/run_sonar-scanner.sh
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -69,7 +69,10 @@ runScanner(){
                     -Dsonar.pullrequest.branch="${BUILDKITE_BRANCH}" \
                     -Dsonar.pullrequest.base="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" \
                     -Dsonar.projectVersion="${BUILDKITE_COMMIT}" \
-                    -Dsonar.scm.provider=git 
+                    -Dsonar.scm.provider=git \
+                    -Dsonar.pullrequest.github.repository="elastic/ml-cpp" \
+                    -Dsonar.pullrequest.provider=github \
+                    -Dsonar.pullrequest.github.endpoint="https://api.github.com"
       exit_code=$?
     else
       sonar-scanner -Dsonar.token="${SONAR_LOGIN}"  \
diff --git a/cmake/compiler/gcc.cmake b/cmake/compiler/gcc.cmake
index 4cae248649..804bfa0b88 100644
--- a/cmake/compiler/gcc.cmake
+++ b/cmake/compiler/gcc.cmake
@@ -38,26 +38,6 @@ endif()
 
 SET(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -ru <TARGET> <OBJECTS>")
 
-# Set CODE_COVERAGE if it is defined in the environment
-if(DEFINED ENV{CODE_COVERAGE})
-  set(CODE_COVERAGE $ENV{CODE_COVERAGE})
-endif()
-
-option(CODE_COVERAGE "Enable code coverage" OFF)
-
-if(CODE_COVERAGE)
-  message(STATUS "Code coverage enabled.")
-  set(COVERAGE_CXX_FLAGS "-fprofile-arcs -ftest-coverage")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_CXX_FLAGS}")
-#  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COVERAGE_CXX_FLAGS}")
-#  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COVERAGE_CXX_FLAGS}")
-#  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${COVERAGE_CXX_FLAGS}")
-#  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_CXX_FLAGS}")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcov")
-else()
-  message(STATUS "Code coverage disabled.")
-endif()
-
 list(APPEND ML_C_FLAGS
   ${CROSS_FLAGS}
   ${ARCHCFLAGS}

From be53a779e1a343e1f81320b97d05e3aa692a91b4 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Fri, 6 Sep 2024 14:34:41 +0200
Subject: [PATCH 35/38] remove deps

---
 .buildkite/pipelines/sonarqube.yml.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 03509ffa93..919441f1d6 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -12,7 +12,7 @@ cat <<EOL
 steps:
   - label: "Run SonarQube scanner :sonarqube:"
     key: "export_compile_commands"
-    depends_on: "build_test_linux-x86_64-RelWithDebInfo"
+    # depends_on: "build_test_linux-x86_64-RelWithDebInfo"
     soft_fail: true
     agents:
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"

From cbed2ba7dca81b2f040444999f7b3229395d6b34 Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Fri, 6 Sep 2024 14:35:57 +0200
Subject: [PATCH 36/38] remove code coverage

---
 .buildkite/pipelines/sonarqube.yml.sh         |  1 -
 .buildkite/scripts/steps/run_sonar-scanner.sh | 10 +++++-----
 sonar-project.properties                      |  4 ++--
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 919441f1d6..fc2077c254 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -20,7 +20,6 @@ steps:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
       VAULT_SONAR_TOKEN_PATH: "secret/ci/elastic-ml-cpp/sonar-analyze-token"
     command: 
-      - "buildkite-agent artifact download gcov.tar.gz ."
       - ".buildkite/scripts/steps/run_sonar-scanner.sh"
     # artifact_paths:
     #   - "cmake-build-docker/compile_commands.json"
diff --git a/.buildkite/scripts/steps/run_sonar-scanner.sh b/.buildkite/scripts/steps/run_sonar-scanner.sh
index f1279481ba..63484b2c1b 100755
--- a/.buildkite/scripts/steps/run_sonar-scanner.sh
+++ b/.buildkite/scripts/steps/run_sonar-scanner.sh
@@ -11,11 +11,11 @@
 
 set -eo pipefail
 
-# Unpack gcov.tar.gz if it exists
-if [ -f gcov.tar.gz ]; then
-  echo "Unpacking gcov.tar.gz"
-  tar -xzf gcov.tar.gz
-fi
+# # Unpack gcov.tar.gz if it exists
+# if [ -f gcov.tar.gz ]; then
+#   echo "Unpacking gcov.tar.gz"
+#   tar -xzf gcov.tar.gz
+# fi
 
 
 export CWD="$(pwd)"
diff --git a/sonar-project.properties b/sonar-project.properties
index 6f1e833352..6431a39f92 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -17,5 +17,5 @@ sonar.lang.patterns.c=**/*.c
 # Exclude coverage files from being analyzed as sources
 sonar.coverage.exclusions=**/unittest/**
 
-# Import coverage report
-sonar.cfamily.gcov.reportsPath=.
+# # Import coverage report
+# sonar.cfamily.gcov.reportsPath=.

From cbbbbef83857be9f436cf7aec0d27ed8c263ca9c Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Fri, 6 Sep 2024 14:50:15 +0200
Subject: [PATCH 37/38] increate resources

---
 .buildkite/pipelines/sonarqube.yml.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index fc2077c254..976c6ed971 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -15,6 +15,8 @@ steps:
     # depends_on: "build_test_linux-x86_64-RelWithDebInfo"
     soft_fail: true
     agents:
+      cpu: 6
+      memory: "16GB"
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

From de35531013b7c82c68dd73f1ee6acc604803695d Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Fri, 6 Sep 2024 15:00:38 +0200
Subject: [PATCH 38/38] increase memory

---
 .buildkite/pipelines/sonarqube.yml.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/sonarqube.yml.sh b/.buildkite/pipelines/sonarqube.yml.sh
index 976c6ed971..3dd781eae4 100755
--- a/.buildkite/pipelines/sonarqube.yml.sh
+++ b/.buildkite/pipelines/sonarqube.yml.sh
@@ -16,7 +16,7 @@ steps:
     soft_fail: true
     agents:
       cpu: 6
-      memory: "16GB"
+      memory: "64GB"
       image: "docker.elastic.co/ml-dev/ml-linux-build:30"
     env:
       PATH: "/usr/local/gcc103/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"