chore(tracing): add first trace telemetry (#7844)

To determine the effectiveness of various different installation types,
the type, time and identifier of an install is potentially propagated to
the library. This data will enable us to see what mechanism users are
opting for and how successful they are.

RFC:
https://docs.google.com/document/d/14vsrCbnAKnXmJAkacX9I6jKPGKmxsq0PKUb3dfiZpWE/edit?pli=1

DataDog/system-tests#1823 validates that the
data is being used.

https://datadoghq.atlassian.net/browse/AIT-8954

## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [x] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [x] This PR doesn't touch any of that.

---------

Co-authored-by: Brett Langdon <[email protected]>

ddtrace/internal/telemetry/writer.py

@@ -127,6 +127,12 @@ def __init__(self, endpoint):
             "DD-Client-Library-Language": "python",
             "DD-Client-Library-Version": _pep440_to_semver(),
         }
+        if config._install_id:
+            self._headers["DD-Agent-Install-Id"] = config._install_id
+        if config._install_type:
+            self._headers["DD-Agent-Install-Type"] = config._install_type
+        if config._install_time:
+            self._headers["DD-Agent-Install-Time"] = config._install_time
 
     @property
     def url(self):

ddtrace/settings/config.py

@@ -509,6 +509,9 @@ def __init__(self):
         self._ddtrace_bootstrapped = False
         self._subscriptions = []  # type: List[Tuple[List[str], Callable[[Config, List[str]], None]]]
         self._span_aggregator_rlock = asbool(os.getenv("DD_TRACE_SPAN_AGGREGATOR_RLOCK", True))
+        self._install_id = os.getenv("DD_INSTRUMENTATION_INSTALL_ID", "")
+        self._install_time = os.getenv("DD_INSTRUMENTATION_INSTALL_TIME", "")
+        self._install_type = os.getenv("DD_INSTRUMENTATION_INSTALL_TYPE", "")
 
         self.trace_methods = os.getenv("DD_TRACE_METHODS")
 

ddtrace/tracer.py

@@ -279,6 +279,7 @@ def __init__(
         self._shutdown_lock = RLock()
 
         self._new_process = False
+        self._first_trace = True
         config._subscribe(["_trace_sample_rate"], self._on_global_config_update)
 
     def _atexit(self) -> None:
@@ -714,6 +715,15 @@ def _start_span(
             if config.report_hostname:
                 span.set_tag_str(HOSTNAME_KEY, hostname.get_hostname())
 
+        if self._first_trace:
+            if config._install_id:
+                span._meta["_dd.install.id"] = config._install_id
+            if config._install_time:
+                span._meta["_dd.install.time"] = config._install_time
+            if config._install_type:
+                span._meta["_dd.install.type"] = config._install_type
+            self._first_trace = False
+
         if not span._parent:
             span.set_tag_str("runtime-id", get_runtime_id())
             span._metrics[PID] = self._pid

docker-compose.yml

@@ -143,7 +143,7 @@ services:
             - DD_POOL_TRACE_CHECK_FAILURES=true
             - DD_DISABLE_ERROR_RESPONSES=true
             - ENABLED_CHECKS=trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service
-            - SNAPSHOT_IGNORED_ATTRS=span_id,trace_id,parent_id,duration,start,metrics.system.pid,metrics.system.process_id,metrics.process_id,meta.runtime-id,meta._dd.p.tid,meta.pathway.hash,metrics._dd.tracer_kr
+            - SNAPSHOT_IGNORED_ATTRS=span_id,trace_id,parent_id,duration,start,metrics.system.pid,metrics.system.process_id,metrics.process_id,meta.runtime-id,meta._dd.p.tid,meta.pathway.hash,metrics._dd.tracer_kr,meta._dd.install.id,meta._dd.install.time,meta._dd.install.type
     vertica:
         image: sumitchawla/vertica
         environment:

tests/telemetry/test_writer.py

@@ -2,6 +2,7 @@
 import time
 from typing import Any  # noqa:F401
 from typing import Dict  # noqa:F401
+import uuid
 
 import httpretty
 import mock
@@ -39,9 +40,44 @@ def test_add_event(telemetry_writer, test_agent_session, mock_time):
     assert requests[0]["headers"]["DD-Telemetry-Request-Type"] == payload_type
     assert requests[0]["headers"]["DD-Telemetry-API-Version"] == "v2"
     assert requests[0]["headers"]["DD-Telemetry-Debug-Enabled"] == "False"
+    assert "DD-Agent-Install-Id" not in requests[0]["headers"]
+    assert "DD-Agent-Install-Time" not in requests[0]["headers"]
+    assert "DD-Agent-Install-Type" not in requests[0]["headers"]
+
     assert requests[0]["body"] == _get_request_body(payload, payload_type)
 
 
+def test_add_event_first_trace(test_agent_session):
+    """asserts that when first trace config values are set appropriate headers are added"""
+    install_id = str(uuid.uuid4())
+    install_time = str(int(time.time()))
+    install_type = "k8s_single_step"
+
+    with override_global_config(dict(_install_id=install_id, _install_type=install_type, _install_time=install_time)):
+        telemetry_writer = TelemetryWriter(is_periodic=False)
+
+        payload = {"test": "123"}
+        payload_type = "test-event"
+        # add event to the queue
+        telemetry_writer.add_event(payload, payload_type)
+        # send request to the agent
+        telemetry_writer.periodic()
+
+    requests = [
+        i for i in test_agent_session.get_requests() if i["body"].get("request_type") != "app-dependencies-loaded"
+    ]
+    assert len(requests) == 1
+    assert requests[0]["headers"]["Content-Type"] == "application/json"
+    assert requests[0]["headers"]["DD-Client-Library-Language"] == "python"
+    assert requests[0]["headers"]["DD-Client-Library-Version"] == _pep440_to_semver()
+    assert requests[0]["headers"]["DD-Telemetry-Request-Type"] == payload_type
+    assert requests[0]["headers"]["DD-Telemetry-API-Version"] == "v2"
+    assert requests[0]["headers"]["DD-Telemetry-Debug-Enabled"] == "False"
+    assert requests[0]["headers"]["DD-Agent-Install-Id"] == install_id
+    assert requests[0]["headers"]["DD-Agent-Install-Time"] == install_time
+    assert requests[0]["headers"]["DD-Agent-Install-Type"] == install_type
+
+
 def test_add_event_disabled_writer(telemetry_writer, test_agent_session):
     """asserts that add_event() does not create a telemetry request when telemetry writer is disabled"""
     telemetry_writer.disable()

tests/tracer/test_tracer.py

@@ -10,7 +10,9 @@
 from os import getpid
 import sys
 import threading
+import time
 from unittest.case import SkipTest
+import uuid
 import weakref
 
 import mock
@@ -1932,6 +1934,43 @@ def test_finish_span_with_ancestors(tracer):
     assert span3.finished
 
 
+def test_first_trace_install_telemetry(tracer):
+    """
+    When ``config._install_id``, ``config._install_time``, and ``config._install_type`` config values are set.
+        The first span created by the tracer gets tagged with ``_dd.install.id``, ``_dd.install.time``,
+        ``_dd.install.type`` tag values are set.
+
+        All other spans created do not have the ``_dd.install.id``, ``_dd.install.time``, ``_dd.install.type`` tags set.
+    """
+    install_id = str(uuid.uuid4())
+    install_time = str(int(time.time()))
+    install_type = "k8s_single_step"
+
+    def assert_has_tags(span):
+        assert span.get_tag("_dd.install.id") == install_id
+        assert span.get_tag("_dd.install.time") == install_time
+        assert span.get_tag("_dd.install.type") == install_type
+
+    def assert_not_has_tags(span):
+        assert span.get_tag("_dd.install.id") is None
+        assert span.get_tag("_dd.install.time") is None
+        assert span.get_tag("_dd.install.type") is None
+
+    with override_global_config(dict(_install_id=install_id, _install_type=install_type, _install_time=install_time)):
+        # The first span created by this tracer is tagged, every other span created is not
+        with tracer.trace("span1") as span1:
+            assert_has_tags(span1)
+
+            with tracer.trace("child1") as child1:
+                assert_not_has_tags(child1)
+
+        with tracer.trace("span2") as span2:
+            assert_not_has_tags(span2)
+
+            with tracer.trace("child2") as child2:
+                assert_not_has_tags(child2)
+
+
 def test_ctx_api():
     from ddtrace.internal import core
 

tests/utils.py

@@ -137,6 +137,9 @@ def override_global_config(values):
         "propagation_http_baggage_enabled",
         "_telemetry_enabled",
         "_telemetry_dependency_collection",
+        "_install_id",
+        "_install_type",
+        "_install_time",
     ]
 
     asm_config_keys = [