ci: dump debug data about builds in CI

.gitignore

@@ -55,6 +55,7 @@ var/
 *.spec
 
 # Installer logs
+debug_ext_metadata.txt
 pip-log.txt
 pip-delete-this-directory.txt
 

.gitlab/tests.yml

@@ -60,14 +60,17 @@ build_base_venvs:
       - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
   variables:
     CMAKE_BUILD_PARALLEL_LEVEL: 12
+    CARGO_BUILD_JOBS: 12
     PIP_VERBOSE: 1
     DD_PROFILING_NATIVE_TESTS: 1
+    _DD_DEBUG_EXT: 1
   script:
     - pip install riot==0.20.1
     - riot -P -v generate --python=$PYTHON_VERSION
   artifacts:
     name: venv_$PYTHON_VERSION
     paths:
+      - debug_ext_metadata.txt
       - .riot/venv_*
       - ddtrace/**/*.so*
       - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe*

hatch.toml

@@ -186,7 +186,7 @@ DD_CIVISIBILITY_CODE_COVERAGE_ENABLED = "1"
 DD_CIVISIBILITY_ITR_ENABLED = "1"
 DD_PATCH_MODULES = "unittest:false"
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
-
+CARGO_BUILD_JOBS = "12"
 
 ## ASM Django
 
@@ -202,6 +202,7 @@ dependencies = [
 
 [envs.appsec_threats_django.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
 
 [envs.appsec_threats_django.scripts]
 test = [
@@ -250,6 +251,7 @@ dependencies = [
 
 [envs.appsec_threats_flask.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
 
 [envs.appsec_threats_flask.scripts]
 test = [
@@ -315,6 +317,7 @@ dependencies = [
 
 [envs.appsec_threats_fastapi.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
 
 [envs.appsec_threats_fastapi.scripts]
 test = [
@@ -357,6 +360,7 @@ dependencies = [
 
 [envs.appsec_aggregated_leak_testing.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
 DD_IAST_ENABLED = "true"
 
 [envs.appsec_aggregated_leak_testing.scripts]
@@ -388,6 +392,7 @@ dependencies = [
 DD_PROFILING_ENABLED = "true"
 DD_PROFILING_PYTORCH_ENABLED = "true"
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
 
 [envs.profiling_pytorch.scripts]
 test = [
@@ -413,6 +418,7 @@ dependencies = [
 DD_IAST_ENABLED = "false"
 DD_REMOTE_CONFIGURATION_ENABLED = "false"
 CMAKE_BUILD_PARALLEL_LEVEL="6"
+CARGO_BUILD_JOBS = "6"
 
 [envs.ddtrace_unit_tests.scripts]
 test = [
@@ -521,6 +527,7 @@ dependencies = [
 [evs.selenium.env-vars]
 DD_AGENT_TRACER_URL = "9126"
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
 
 [envs.selenium.scripts]
 test = [

riotfile.py

@@ -103,6 +103,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
         "DD_INJECT_FORCE": "1",
         "DD_PATCH_MODULES": "unittest:false",
         "CMAKE_BUILD_PARALLEL_LEVEL": "12",
+        "CARGO_BUILD_JOBS": "12",
         "DD_PYTEST_USE_NEW_PLUGIN_BETA": "true",
     },
     venvs=[

scripts/docs/build.sh

@@ -3,7 +3,7 @@ set -eux
 
 # DEV: unless it's built with editable, following sphinx-build fails
 if [ -z ${CIRCLECI+x} ]; then
-  CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) pip install -v -e .
+  CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) CARGO_BUILD_JOBS=$(nprocs) pip install -v -e .
 fi
 
 if [[ "$(uname)" == "Darwin" ]]; then

setup.py

@@ -1,3 +1,4 @@
+import atexit
 import hashlib
 import os
 import platform
@@ -7,10 +8,12 @@
 import sys
 import sysconfig
 import tarfile
+import time
 
 import cmake
 from setuptools_rust import Binding
 from setuptools_rust import RustExtension
+from setuptools_rust import build_rust
 
 
 from setuptools import Extension, find_packages, setup  # isort: skip
@@ -413,6 +416,57 @@ def build_extension_cmake(self, ext):
         subprocess.run([cmake_command, "--install", ".", *install_args], cwd=cmake_build_dir, check=True)
 
 
+class DebugMetadata:
+    start_ns = 0
+    enabled = "_DD_DEBUG_EXT" in os.environ
+    metadata_file = os.getenv("_DD_DEBUG_EXT_FILE", "debug_ext_metadata.txt")
+    build_times = {}
+
+    @classmethod
+    def dump_metadata(cls):
+        if not cls.enabled or not cls.build_times:
+            return
+
+        total_ns = time.time_ns() - cls.start_ns
+        total_s = total_ns / 1e9
+
+        build_total_ns = sum(cls.build_times.values())
+        build_total_s = build_total_ns / 1e9
+        build_percent = (build_total_ns / total_ns) * 100.0
+
+        with open(cls.metadata_file, "w") as f:
+            f.write(f"Total time: {total_s:0.2f}s\n")
+            f.write("Environment:\n")
+            f.write(f"\tCARGO_BUILD_JOBS: {os.getenv('CARGO_BUILD_JOBS', 'unset')}\n")
+            f.write(f"\tCMAKE_BUILD_PARALLEL_LEVEL: {os.getenv('CMAKE_BUILD_PARALLEL_LEVEL', 'unset')}\n")
+            f.write(f"\tDD_COMPILE_DEBUG: {DEBUG_COMPILE}\n")
+            f.write(f"\tDD_USE_SCCACHE: {SCCACHE_COMPILE}\n")
+            f.write("Extension build times:\n")
+            f.write(f"\tTotal: {build_total_s:0.2f}s ({build_percent:0.2f}%)\n")
+            for ext, elapsed_ns in sorted(cls.build_times.items(), key=lambda x: x[1], reverse=True):
+                elapsed_s = elapsed_ns / 1e9
+                ext_percent = (elapsed_ns / total_ns) * 100.0
+                f.write(f"\t{ext.name}: {elapsed_s:0.2f}s ({ext_percent:0.2f}%)\n")
+
+
+def debug_build_extension(fn):
+    def wrapper(self, ext, *args, **kwargs):
+        start = time.time_ns()
+        try:
+            return fn(self, ext, *args, **kwargs)
+        finally:
+            DebugMetadata.build_times[ext] = time.time_ns() - start
+
+    return wrapper
+
+
+if DebugMetadata.enabled:
+    DebugMetadata.start_ns = time.time_ns()
+    CMakeBuild.build_extension = debug_build_extension(CMakeBuild.build_extension)
+    build_rust.build_extension = debug_build_extension(build_rust.build_extension)
+    atexit.register(DebugMetadata.dump_metadata)
+
+
 class CMakeExtension(Extension):
     def __init__(
         self,
@@ -588,6 +642,7 @@ def get_exts_for(name):
     cmdclass={
         "build_ext": CMakeBuild,
         "build_py": LibraryDownloader,
+        "build_rust": build_rust,
         "clean": CleanLibraries,
     },
     setup_requires=["setuptools_scm[toml]>=4", "cython", "cmake>=3.24.2,<3.28", "setuptools-rust"],