feat(ci_visibility): make pytest v2 plugin the default version

.gitlab/tests.yml

@@ -6,7 +6,6 @@ stages:
 variables:
   RIOT_RUN_CMD: riot -P -v run --exitfirst --pass-env -s
   REPO_LANG: python # "python" is used everywhere rather than "py"
-  DD_PYTEST_USE_NEW_PLUGIN_BETA: "true"
   PYTEST_ADDOPTS: "-s"
   # CI_DEBUG_SERVICES: "true"
 

ddtrace/contrib/internal/pytest/_plugin_v1.py

@@ -452,9 +452,8 @@ def pytest_load_initial_conftests(early_config, parser, args):
 
 def pytest_configure(config):
     deprecate(
-        "this version of the pytest ddtrace plugin is slated for deprecation",
-        message="set DD_PYTEST_USE_NEW_PLUGIN_BETA=true in your environment to preview the next version of the plugin.",
-        removal_version="3.0.0",
+        "this version of the pytest ddtrace plugin is deprecated",
+        message="remove _DD_PYTEST_USE_LEGACY_PLUGIN from your environment to use the currently supported version.",
         category=DDTraceDeprecationWarning,
     )
     unpatch_unittest()

ddtrace/contrib/internal/pytest/_plugin_v2.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 import re
 import typing as t
@@ -182,7 +183,6 @@ def pytest_load_initial_conftests(early_config, parser, args):
 
     try:
         take_over_logger_stream_handler()
-        log.warning("This version of the ddtrace pytest plugin is currently in beta.")
         # Freezegun is proactively patched to avoid it interfering with internal timing
         patch(freezegun=True)
         dd_config.test_visibility.itr_skipping_level = ITR_SKIPPING_LEVEL.SUITE
@@ -199,14 +199,14 @@ def pytest_load_initial_conftests(early_config, parser, args):
 
 
 def pytest_configure(config: pytest_Config) -> None:
-    # The only way we end up in pytest_configure is if the environment variable is being used, and logging the warning
-    # now ensures it shows up in output regardless of the use of the -s flag
-    deprecate(
-        "the DD_PYTEST_USE_NEW_PLUGIN_BETA environment variable is deprecated",
-        message="this preview version of the pytest ddtrace plugin will become the only version.",
-        removal_version="3.0.0",
-        category=DDTraceDeprecationWarning,
-    )
+    if os.getenv("DD_PYTEST_USE_NEW_PLUGIN_BETA"):
+        # Logging the warning at this point ensures it shows up in output regardless of the use of the -s flag.
+        deprecate(
+            "the DD_PYTEST_USE_NEW_PLUGIN_BETA environment variable is deprecated",
+            message="the new pytest plugin is now the default version. No additional configurations are required.",
+            removal_version="3.0.0",
+            category=DDTraceDeprecationWarning,
+        )
 
     try:
         if is_enabled(config):

ddtrace/contrib/internal/pytest/_utils.py

@@ -30,7 +30,7 @@
 
 _NODEID_REGEX = re.compile("^(((?P<module>.*)/)?(?P<suite>[^/]*?))::(?P<name>.*?)$")
 
-_USE_PLUGIN_V2 = asbool(os.environ.get("DD_PYTEST_USE_NEW_PLUGIN_BETA", "false"))
+_USE_PLUGIN_V2 = not asbool(os.environ.get("_DD_PYTEST_USE_LEGACY_PLUGIN", "false"))
 
 
 class _PYTEST_STATUS:

docs/configuration.rst

@@ -616,22 +616,6 @@ Test Visibility
      version_added:
         v2.16.0:
 
-   DD_PYTEST_USE_NEW_PLUGIN_BETA:
-     type: Boolean
-     default: False
-
-     description: |
-        Configures the ``CIVisibility`` service to use a beta release of the new version of the pytest plugin,
-        supporting `Auto Test Retries <https://docs.datadoghq.com/tests/flaky_test_management/auto_test_retries>`_,
-        `Early Flake Detection <https://docs.datadoghq.com/tests/flaky_test_management/early_flake_detection>`_, and
-        improved coverage collection for `Test Impact Analysis
-        <https://docs.datadoghq.com/tests/test_impact_analysis>`_. This version of the plugin will become the default in
-        the future. See the `release notes for v2.18.0 <https://github.com/DataDog/dd-trace-py/releases/tag/v2.18.0>`_
-        for more information.
-
-     version_added:
-        v2.18.0:
-
    DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS:
      type: Integer
      default: 500

hatch.toml

@@ -558,7 +558,6 @@ dependencies = [
 ]
 
 [envs.pytest_plugin_v2.env-vars]
-DD_PYTEST_USE_NEW_PLUGIN_BETA = "true"
 DD_AGENT_PORT = "9126"
 
 [envs.pytest_plugin_v2.scripts]
@@ -599,9 +598,6 @@ dependencies = [
     "hypothesis",
 ]
 
-[envs.freezegun.env-vars]
-DD_PYTEST_USE_NEW_PLUGIN_BETA = "true"
-
 [envs.freezegun.scripts]
 test = [
     "pytest tests/contrib/freezegun {args:}",
@@ -637,6 +633,6 @@ tested_pytest_plugin_version = ["v1", "v2"]
 
 [envs.selenium.overrides]
 matrix.tested_pytest_plugin_version.env-vars = [
-    { key = "_TESTED_PYTEST_PLUGIN_VERSION", value = "false", if = ["v1"]},
-    { key = "_TESTED_PYTEST_PLUGIN_VERSION", value = "true", if = ["v2"]}
+    { key = "_TESTED_PYTEST_LEGACY_PLUGIN", value = "true", if = ["v1"]},
+    { key = "_TESTED_PYTEST_LEGACY_PLUGIN", value = "false", if = ["v2"]}
 ]

releasenotes/notes/ci_visibility-feat-pytest-v2-default-db160906c0ba26dc.yaml

@@ -0,0 +1,19 @@
+---
+upgrade:
+  - |
+    CI Visibility: Official release of the new version of the pytest plugin, introducing the following features:
+      - `Auto Test Retries <https://docs.datadoghq.com/tests/flaky_test_management/auto_test_retries>`_
+      - `Early Flake Detection <https://docs.datadoghq.com/tests/flaky_test_management/early_flake_detection>`_
+      - Improved coverage collection for `Test Impact Analysis <https://docs.datadoghq.com/tests/test_impact_analysis>`_
+        (formerly Intelligent Test Runner), using an internal collection method instead of `coverage.py
+        <https://github.com/nedbat/coveragepy>`_, with improved dependency discovery.
+
+    **NOTE:** this new version of the plugin introduces breaking changes:
+      - ``module``, ``suite``, and ``test`` names are now parsed from the ``item.nodeid`` attribute 
+      - test names now include the class for class-based tests
+      - Test skipping by Test Impact Analysis (formerly Intelligent Test Runner) is now done at the suite level, instead
+        of at the test level
+
+    A beta version of the plugin had been available since v2.18.0, and could be enabled via the
+    ``DD_PYTEST_USE_NEW_PLUGIN_BETA`` environment variable. The new version is now the default, and the environment
+    variable is not used anymore.

riotfile.py

@@ -103,7 +103,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
         "DD_INJECT_FORCE": "1",
         "DD_PATCH_MODULES": "unittest:false",
         "CMAKE_BUILD_PARALLEL_LEVEL": "12",
-        "DD_PYTEST_USE_NEW_PLUGIN_BETA": "true",
     },
     venvs=[
         Venv(
@@ -1604,7 +1603,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             },
             env={
                 "DD_AGENT_PORT": "9126",
-                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
             },
             venvs=[
                 Venv(
@@ -1634,12 +1632,12 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                             venvs=[
                                 Venv(
                                     env={
-                                        "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0",
+                                        "_DD_PYTEST_USE_LEGACY_PLUGIN": "true",
                                     },
                                 ),
                                 Venv(
                                     env={
-                                        "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
+                                        "_DD_PYTEST_USE_LEGACY_PLUGIN": "false",
                                     },
                                 ),
                             ],
@@ -1662,12 +1660,12 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                     venvs=[
                         Venv(
                             env={
-                                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0",
+                                "DD_PYTEST_LEGACY_PLUGIN": "true",
                             },
                         ),
                         Venv(
                             env={
-                                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
+                                "_DD_PYTEST_USE_LEGACY_PLUGIN": "false",
                             },
                         ),
                     ],
@@ -1734,12 +1732,12 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                     venvs=[
                         Venv(
                             env={
-                                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0",
+                                "_DD_PYTEST_USE_LEGACY_PLUGIN": "true",
                             },
                         ),
                         Venv(
                             env={
-                                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
+                                "_DD_PYTEST_USE_LEGACY_PLUGIN": "false",
                             },
                         ),
                     ],
@@ -1755,12 +1753,12 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                     venvs=[
                         Venv(
                             env={
-                                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0",
+                                "_DD_PYTEST_USE_LEGACY_PLUGIN": "true",
                             },
                         ),
                         Venv(
                             env={
-                                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
+                                "_DD_PYTEST_USE_LEGACY_PLUGIN": "false",
                             },
                         ),
                     ],
@@ -1783,7 +1781,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                         ]
                     },
                     env={
-                        "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0",
+                        "_DD_PYTEST_USE_LEGACY_PLUGIN": "true",
                     },
                 ),
                 Venv(
@@ -1793,7 +1791,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                         ]
                     },
                     env={
-                        "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
+                        "_DD_PYTEST_USE_LEGACY_PLUGIN": "false",
                     },
                 ),
             ],
@@ -3072,8 +3070,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             env={
                 "DD_PROFILING_ENABLE_ASSERTS": "1",
                 "DD_PROFILING_EXPORT_LIBDD_ENABLED": "1",
-                # Enable pytest v2 plugin to handle pytest-cpp items in the test suite
-                "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1",
                 "CPUCOUNT": "12",
             },
             pkgs={

tests/contrib/flask/test_appsec_flask_pytest_iast_no_snapshot.py

@@ -9,8 +9,8 @@
 
 @pytest.mark.parametrize("iast_enabled", ["true", "false"])
 @pytest.mark.parametrize("iast_request_sampling", ["100.0", "0.0"])
-@pytest.mark.parametrize("pytest_use_new_plugin", ["true", "false"])
-def test_flask_pytest_iast(iast_enabled, iast_request_sampling, pytest_use_new_plugin):
+@pytest.mark.parametrize("pytest_use_legacy_plugin", ["false", "true"])
+def test_flask_pytest_iast(iast_enabled, iast_request_sampling, pytest_use_legacy_plugin):
     from tests.utils import _build_env
 
     env = _build_env()
@@ -21,7 +21,7 @@ def test_flask_pytest_iast(iast_enabled, iast_request_sampling, pytest_use_new_p
             "DD_TRACE_SQLITE_ENABLED": "0",
             "DD_IAST_ENABLED": iast_enabled,
             "DD_TRACE_DEBUG": "true",
-            "DD_PYTEST_USE_NEW_PLUGIN_BETA": pytest_use_new_plugin,
+            "_DD_PYTEST_USE_LEGACY_PLUGIN": pytest_use_legacy_plugin,
             "DD_IAST_REQUEST_SAMPLING": iast_request_sampling,
             # "DD_API_KEY": "invalidapikey",
             # "DD_CIVISIBILITY_AGENTLESS_ENABLED": "1",

tests/contrib/pytest/test_pytest_snapshot.py

@@ -85,6 +85,7 @@ def test_add_two_number_list():
                         DD_PATCH_MODULES="sqlite3:false",
                         CI_PROJECT_DIR=str(self.testdir.tmpdir),
                         DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
+                        _DD_PYTEST_USE_LEGACY_PLUGIN="true",
                     )
                 ),
             )
@@ -130,6 +131,7 @@ def test_add_two_number_list():
                         DD_PATCH_MODULES="sqlite3:false",
                         CI_PROJECT_DIR=str(self.testdir.tmpdir),
                         DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
+                        _DD_PYTEST_USE_LEGACY_PLUGIN="true",
                     )
                 ),
             )
@@ -164,6 +166,7 @@ def test_call_urllib():
                         DD_CIVISIBILITY_ITR_ENABLED="false",
                         CI_PROJECT_DIR=str(self.testdir.tmpdir),
                         DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
+                        _DD_PYTEST_USE_LEGACY_PLUGIN="true",
                         DD_PATCH_MODULES="httpx:true",
                     )
                 ),

tests/contrib/pytest/test_pytest_snapshot_v2.py

@@ -85,7 +85,6 @@ def test_add_two_number_list():
                         DD_PATCH_MODULES="sqlite3:false",
                         CI_PROJECT_DIR=str(self.testdir.tmpdir),
                         DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
-                        DD_PYTEST_USE_NEW_PLUGIN_BETA="true",
                     )
                 ),
             )
@@ -131,7 +130,6 @@ def test_add_two_number_list():
                         DD_PATCH_MODULES="sqlite3:false",
                         CI_PROJECT_DIR=str(self.testdir.tmpdir),
                         DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
-                        DD_PYTEST_USE_NEW_PLUGIN_BETA="true",
                     )
                 ),
             )
@@ -167,7 +165,6 @@ def test_call_urllib():
                         CI_PROJECT_DIR=str(self.testdir.tmpdir),
                         DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
                         DD_PATCH_MODULES="httpx:true",
-                        DD_PYTEST_USE_NEW_PLUGIN_BETA="true",
                     )
                 ),
             )

tests/contrib/selenium/test_selenium_chrome.py

@@ -116,7 +116,7 @@ def test_selenium_local_pass():
                 DD_PATCH_MODULES="sqlite3:false",
                 CI_PROJECT_DIR=str(testdir.tmpdir),
                 DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
-                DD_PYTEST_USE_NEW_PLUGIN_BETA=os.environ.get("_TESTED_PYTEST_PLUGIN_VERSION"),
+                _DD_PYTEST_USE_LEGACY_PLUGIN=os.environ.get("_TESTED_PYTEST_LEGACY_PLUGIN"),
             )
         ),
     )
@@ -167,7 +167,7 @@ def test_selenium_local_pass():
                 DD_PATCH_MODULES="sqlite3:false",
                 CI_PROJECT_DIR=str(testdir.tmpdir),
                 DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
-                DD_PYTEST_USE_NEW_PLUGIN_BETA=os.environ.get("_TESTED_PYTEST_PLUGIN_VERSION"),
+                _DD_PYTEST_USE_LEGACY_PLUGIN=os.environ.get("_TESTED_PYTEST_LEGACY_PLUGIN"),
             )
         ),
     )
@@ -221,7 +221,7 @@ def test_selenium_local_unpatch():
                 DD_PATCH_MODULES="sqlite3:false",
                 CI_PROJECT_DIR=str(testdir.tmpdir),
                 DD_CIVISIBILITY_AGENTLESS_ENABLED="false",
-                DD_PYTEST_USE_NEW_PLUGIN_BETA=os.environ.get("_TESTED_PYTEST_PLUGIN_VERSION"),
+                _DD_PYTEST_USE_LEGACY_PLUGIN=os.environ.get("_TESTED_PYTEST_LEGACY_PLUGIN"),
             )
         ),
     )