[BUG]: ddtrace client doesn't work with Airflow 3 #16443
Description
Tracer Version(s)
4.2.0
Python Version(s)
3.12
Pip Version(s)
23.3.1
Bug Report
When using ddtrace to instrument Apache Airflow api servers ddtrace-run airflow api-server, it will make all FastAPI calls fail with 500 errors. The error was TypeError: cannot pickle '_contextvars.ContextVar' object. Attached the error stacktrace below.
Airflow 3 is using https://github.com/zmievsa/cadwyn as part of their FastAPI stack and it looks the error happens when it tries to serialize/deserialize for dynamic routing generation.
This issue makes all Airflow DAGs execution fail and makes ddtrace unusable for all Airflow 3 users. I found a temporary workaround by pinning ddtrace to version 2.5.0 and prior but that is just a temporary bandaid.
Reproduction Code
ddtrace-run airflow api-server
Error Logs
ERROR: Exception in ASGI application
Exception Group Traceback (most recent call last):
File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 79, in collapse_excgroups
yield
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 183, in __call__
async with anyio.create_task_group() as task_group:
File "/usr/local/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 783, in __aexit__
raise BaseExceptionGroup(
ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
'+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
| result = await app( # type: ignore[func-returns-value]
| File "/usr/local/lib/python3.12/site-packages/fastapi/applications.py", line 1082, in __call__
| await super().__call__(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/applications.py", line 113, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/ddtrace/contrib/internal/asgi/middleware.py", line 459, in __call__
| return await self.app(scope, wrapped_recv, wrapped_send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
| await self.app(scope, receive, _send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/gzip.py", line 29, in __call__
| await responder(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/gzip.py", line 130, in __call__
| await super().__call__(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/gzip.py", line 46, in __call__
| await self.app(scope, receive, self.send_with_compression)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/cors.py", line 85, in __call__
| await self.app(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 182, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.12/contextlib.py", line 158, in __exit__
| self.gen.throw(value)
| File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 85, in collapse_excgroups
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 184, in __call__
| response = await self.dispatch_func(request, call_next)
| File "/usr/local/lib/python3.12/site-packages/airflow/api_fastapi/auth/middlewares/refresh_token.py", line 51, in dispatch
| response = await call_next(request)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 159, in call_next
| raise app_exc
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 144, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
| await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 716, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
| await route.handle(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 462, in handle
| await self.app(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/cadwyn/applications.py", line 258, in __call__
| self._cadwyn_initialize()
| File "/usr/local/lib/python3.12/site-packages/cadwyn/applications.py", line 263, in _cadwyn_initialize
| generated_routers = generate_versioned_routers(
| File "/usr/local/lib/python3.12/site-packages/cadwyn/route_generation.py", line 82, in generate_versioned_routers
| return _EndpointTransformer(router, versions, webhooks).transform()
| File "/usr/local/lib/python3.12/site-packages/cadwyn/route_generation.py", line 146, in transform
| router = copy_router(router)
| File "/usr/local/lib/python3.12/site-packages/cadwyn/route_generation.py", line 101, in copy_router
| router.routes = [copy_route(r) for r in router.routes]
| File "/usr/local/lib/python3.12/site-packages/cadwyn/route_generation.py", line 112, in copy_route
| new_route = deepcopy(route)
| File "/usr/local/lib/python3.12/copy.py", line 162, in deepcopy
| y = _reconstruct(x, memo, *rv)
| [Multiple deepcopy frames for nested dictionaries...]
| File "/usr/local/lib/python3.12/copy.py", line 151, in deepcopy
| rv = reductor(4)
| TypeError: cannot pickle '_contextvars.ContextVar' object
'+------------------------------------
Libraries in Use
ddtrace
Operating System
22.04.1-Ubuntu