From e3692405fac496e14965429833609181ec99eb55 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Thu, 12 Dec 2024 16:01:40 +1000 Subject: [PATCH 01/17] POC error metric --- apollo-router/src/plugins/telemetry/mod.rs | 2 +- apollo-router/src/services/router/service.rs | 45 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 8b403b8346..136592d1fb 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -174,7 +174,7 @@ pub(crate) mod utils; // Tracing consts pub(crate) const CLIENT_NAME: &str = "apollo_telemetry::client_name"; -const CLIENT_VERSION: &str = "apollo_telemetry::client_version"; +pub(crate) const CLIENT_VERSION: &str = "apollo_telemetry::client_version"; const SUBGRAPH_FTV1: &str = "apollo_telemetry::subgraph_ftv1"; pub(crate) const STUDIO_EXCLUDE: &str = "apollo_telemetry::studio::exclude"; pub(crate) const SUPERGRAPH_SCHEMA_ID_CONTEXT_KEY: &str = "apollo::supergraph_schema_id"; diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index 818ee188b3..edd68d4c7a 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -43,6 +43,11 @@ use crate::cache::DeduplicatingCache; use crate::configuration::Batching; use crate::configuration::BatchingMode; use crate::context::CONTAINS_GRAPHQL_ERROR; +use crate::context::OPERATION_KIND; +use crate::context::OPERATION_NAME; +use crate::plugins::telemetry::CLIENT_NAME; +use crate::plugins::telemetry::CLIENT_VERSION; +use crate::query_planner::APOLLO_OPERATION_ID; use crate::graphql; use crate::http_ext; #[cfg(test)] @@ -304,6 +309,21 @@ impl RouterService { Self::count_errors(&response.errors); } + if let Some(path) = response.path.clone() { + println!("response path: {}", path); + } + + let operation_id: String = context.get::<_, String>(APOLLO_OPERATION_ID).unwrap_or_default().unwrap_or_default(); + let operation_kind = context.get::<_, String>(OPERATION_KIND).unwrap_or_default().unwrap_or_default(); + let operation_name = context.get::<_, String>(OPERATION_NAME).unwrap_or_default().unwrap_or_default(); + let client_name = context.get::<_, String>(CLIENT_NAME).unwrap_or_default().unwrap_or_default(); + let client_version = context.get::<_, String>(CLIENT_VERSION).unwrap_or_default().unwrap_or_default(); + println!("operation_id: {}", operation_id); + println!("operation_kind: {}", operation_kind); + println!("operation_name: {}", operation_name); + println!("client_name: {}", client_name); + println!("client_version: {}", client_version); + parts .headers .insert(CONTENT_TYPE, APPLICATION_JSON_HEADER_VALUE.clone()); @@ -778,6 +798,31 @@ impl RouterService { let code = error.extensions.get("code").and_then(|c| c.as_str()); let entry = map.entry(code).or_insert(0u64); *entry += 1; + + let temp_code = code.unwrap_or_default().to_string(); + let temp_service = error.extensions.get("service").and_then(|s| s.as_str()) + .unwrap_or_default().to_string(); + let temp_path = match &error.path { + None => "".into(), + Some(path) => path.to_string() + }; + println!("temp_code: {temp_code}"); + println!("temp_path: {temp_path}"); + println!("temp_service: {temp_service}"); + + u64_counter!( + "temp.nick.apollo.router.operations.error", + "Temp metric", + 1, + "apollo.operation.id" = "todo", + "apollo.operation.name" = "todo", + "apollo.client.name" = "todo", + "apollo.client.version" = "todo", + "graphql.error.extensions.code" = temp_code, + "graphql.error.path" = temp_path, + "apollo.router.error.source" = temp_service, + "apollo.router.error.coordinate" = "todo" + ); } for (code, count) in map { From 98eef0af8051147d7ecbec2548197217467d7c39 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 6 Jan 2025 15:27:12 +1000 Subject: [PATCH 02/17] Cleaned up POC --- apollo-router/src/services/router/service.rs | 99 +++++++++++--------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index edd68d4c7a..bd90e19c5f 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -45,16 +45,16 @@ use crate::configuration::BatchingMode; use crate::context::CONTAINS_GRAPHQL_ERROR; use crate::context::OPERATION_KIND; use crate::context::OPERATION_NAME; -use crate::plugins::telemetry::CLIENT_NAME; -use crate::plugins::telemetry::CLIENT_VERSION; -use crate::query_planner::APOLLO_OPERATION_ID; use crate::graphql; use crate::http_ext; #[cfg(test)] use crate::plugin::test::MockSupergraphService; +use crate::plugins::telemetry::CLIENT_NAME; +use crate::plugins::telemetry::CLIENT_VERSION; use crate::protocols::multipart::Multipart; use crate::protocols::multipart::ProtocolMode; use crate::query_planner::InMemoryCachePlanner; +use crate::query_planner::APOLLO_OPERATION_ID; use crate::router_factory::RouterFactory; use crate::services::layers::apq::APQLayer; use crate::services::layers::content_negotiation; @@ -305,14 +305,16 @@ impl RouterService { && !response.subscribed.unwrap_or(false) && (accepts_json || accepts_wildcard) { + println!("pre-count_errors 1"); if !response.errors.is_empty() { - Self::count_errors(&response.errors); + Self::count_errors(&response.errors, &context); } if let Some(path) = response.path.clone() { println!("response path: {}", path); } + /* let operation_id: String = context.get::<_, String>(APOLLO_OPERATION_ID).unwrap_or_default().unwrap_or_default(); let operation_kind = context.get::<_, String>(OPERATION_KIND).unwrap_or_default().unwrap_or_default(); let operation_name = context.get::<_, String>(OPERATION_NAME).unwrap_or_default().unwrap_or_default(); @@ -323,6 +325,7 @@ impl RouterService { println!("operation_name: {}", operation_name); println!("client_name: {}", client_name); println!("client_version: {}", client_version); + */ parts .headers @@ -350,8 +353,9 @@ impl RouterService { ); } + println!("pre-count_errors 2"); if !response.errors.is_empty() { - Self::count_errors(&response.errors); + Self::count_errors(&response.errors, &context); } // Useful when you're using a proxy like nginx which enable proxy_buffering by default (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) @@ -360,22 +364,30 @@ impl RouterService { ACCEL_BUFFERING_HEADER_VALUE.clone(), ); let multipart_stream = match response.subscribed { - Some(true) => StreamBody::new(Multipart::new( - body.inspect(|response| { - if !response.errors.is_empty() { - Self::count_errors(&response.errors); - } - }), - ProtocolMode::Subscription, - )), - _ => StreamBody::new(Multipart::new( - once(ready(response)).chain(body.inspect(|response| { - if !response.errors.is_empty() { - Self::count_errors(&response.errors); - } - })), - ProtocolMode::Defer, - )), + Some(true) => { + let context_clone = context.clone(); + StreamBody::new(Multipart::new( + body.inspect(move |response: &graphql::Response| { + println!("pre-count_errors 3"); + if !response.errors.is_empty() { + Self::count_errors(&response.errors, &context_clone); + } + }), + ProtocolMode::Subscription, + )) + } + _ => { + let context_clone = context.clone(); + StreamBody::new(Multipart::new( + once(ready(response)).chain(body.inspect(move |response| { + println!("pre-count_errors 4"); + if !response.errors.is_empty() { + Self::count_errors(&response.errors, &context_clone); + } + })), + ProtocolMode::Defer, + )) + } }; let response = (parts, multipart_stream).into_response().map(|body| { // Axum makes this `body` have type: @@ -792,36 +804,37 @@ impl RouterService { Ok(graphql_requests) } - fn count_errors(errors: &[graphql::Error]) { + fn count_errors(errors: &[graphql::Error], context: &Context) { + let unwrap_context_string = |context_key: &str| -> String { + context + .get::<_, String>(context_key) + .unwrap_or_default() + .unwrap_or_default() + }; + + let operation_id = unwrap_context_string(APOLLO_OPERATION_ID); + let operation_name = unwrap_context_string(OPERATION_NAME); + let operation_kind = unwrap_context_string(OPERATION_KIND); + let client_name = unwrap_context_string(CLIENT_NAME); + let client_version = unwrap_context_string(CLIENT_VERSION); + let mut map = HashMap::new(); for error in errors { let code = error.extensions.get("code").and_then(|c| c.as_str()); let entry = map.entry(code).or_insert(0u64); *entry += 1; - let temp_code = code.unwrap_or_default().to_string(); - let temp_service = error.extensions.get("service").and_then(|s| s.as_str()) - .unwrap_or_default().to_string(); - let temp_path = match &error.path { - None => "".into(), - Some(path) => path.to_string() - }; - println!("temp_code: {temp_code}"); - println!("temp_path: {temp_path}"); - println!("temp_service: {temp_service}"); - + let code_str = code.unwrap_or_default().to_string(); u64_counter!( - "temp.nick.apollo.router.operations.error", - "Temp metric", + "apollo.router.operations.error", + "Number of errors returned by operation", 1, - "apollo.operation.id" = "todo", - "apollo.operation.name" = "todo", - "apollo.client.name" = "todo", - "apollo.client.version" = "todo", - "graphql.error.extensions.code" = temp_code, - "graphql.error.path" = temp_path, - "apollo.router.error.source" = temp_service, - "apollo.router.error.coordinate" = "todo" + "apollo.operation.id" = operation_id.clone(), + "graphql.operation.name" = operation_name.clone(), + "graphql.operation.type" = operation_kind.clone(), + "apollo.client.name" = client_name.clone(), + "apollo.client.version" = client_version.clone(), + "graphql.error.extensions.code" = code_str ); } From 4e4d7281eb58e6f4ba0f5bc5e863b83355e444a8 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 6 Jan 2025 15:29:01 +1000 Subject: [PATCH 03/17] Remove temp code --- apollo-router/src/services/router/service.rs | 21 -------------------- 1 file changed, 21 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index bd90e19c5f..6f5eeab36d 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -305,28 +305,10 @@ impl RouterService { && !response.subscribed.unwrap_or(false) && (accepts_json || accepts_wildcard) { - println!("pre-count_errors 1"); if !response.errors.is_empty() { Self::count_errors(&response.errors, &context); } - if let Some(path) = response.path.clone() { - println!("response path: {}", path); - } - - /* - let operation_id: String = context.get::<_, String>(APOLLO_OPERATION_ID).unwrap_or_default().unwrap_or_default(); - let operation_kind = context.get::<_, String>(OPERATION_KIND).unwrap_or_default().unwrap_or_default(); - let operation_name = context.get::<_, String>(OPERATION_NAME).unwrap_or_default().unwrap_or_default(); - let client_name = context.get::<_, String>(CLIENT_NAME).unwrap_or_default().unwrap_or_default(); - let client_version = context.get::<_, String>(CLIENT_VERSION).unwrap_or_default().unwrap_or_default(); - println!("operation_id: {}", operation_id); - println!("operation_kind: {}", operation_kind); - println!("operation_name: {}", operation_name); - println!("client_name: {}", client_name); - println!("client_version: {}", client_version); - */ - parts .headers .insert(CONTENT_TYPE, APPLICATION_JSON_HEADER_VALUE.clone()); @@ -353,7 +335,6 @@ impl RouterService { ); } - println!("pre-count_errors 2"); if !response.errors.is_empty() { Self::count_errors(&response.errors, &context); } @@ -368,7 +349,6 @@ impl RouterService { let context_clone = context.clone(); StreamBody::new(Multipart::new( body.inspect(move |response: &graphql::Response| { - println!("pre-count_errors 3"); if !response.errors.is_empty() { Self::count_errors(&response.errors, &context_clone); } @@ -380,7 +360,6 @@ impl RouterService { let context_clone = context.clone(); StreamBody::new(Multipart::new( once(ready(response)).chain(body.inspect(move |response| { - println!("pre-count_errors 4"); if !response.errors.is_empty() { Self::count_errors(&response.errors, &context_clone); } From b6e97beedb732e3251662fc0b59564f34d8cf901 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Tue, 7 Jan 2025 16:06:00 +1000 Subject: [PATCH 04/17] Merge and accept header fix --- apollo-router/src/services/router/service.rs | 23 +++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index c7cafd4217..607229a0c8 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -48,10 +48,6 @@ use crate::graphql; use crate::http_ext; #[cfg(test)] use crate::plugin::test::MockSupergraphService; -<<<<<<< HEAD -use crate::plugins::telemetry::CLIENT_NAME; -use crate::plugins::telemetry::CLIENT_VERSION; -======= use crate::plugins::telemetry::config_new::attributes::HTTP_REQUEST_BODY; use crate::plugins::telemetry::config_new::attributes::HTTP_REQUEST_HEADERS; use crate::plugins::telemetry::config_new::attributes::HTTP_REQUEST_URI; @@ -60,7 +56,8 @@ use crate::plugins::telemetry::config_new::events::log_event; use crate::plugins::telemetry::config_new::events::DisplayRouterRequest; use crate::plugins::telemetry::config_new::events::DisplayRouterResponse; use crate::plugins::telemetry::config_new::events::RouterResponseBodyExtensionType; ->>>>>>> next +use crate::plugins::telemetry::CLIENT_NAME; +use crate::plugins::telemetry::CLIENT_VERSION; use crate::protocols::multipart::Multipart; use crate::protocols::multipart::ProtocolMode; use crate::query_planner::InMemoryCachePlanner; @@ -381,12 +378,8 @@ impl RouterService { Ok(RouterResponse { response, context }) } else { - u64_counter!( - "apollo.router.graphql_error", - "Number of GraphQL error responses returned by the router", - 1, - code = "INVALID_ACCEPT_HEADER" - ); + Self::count_error_codes(vec![Some("INVALID_ACCEPT_HEADER")], &context); + // Useful for selector in spans/instruments/events context.insert_json_value( CONTAINS_GRAPHQL_ERROR, @@ -821,6 +814,11 @@ impl RouterService { } fn count_errors(errors: &[graphql::Error], context: &Context) { + let codes = errors.iter().map(|e| e.extensions.get("code").and_then(|c| c.as_str())).collect(); + Self::count_error_codes(codes, context); + } + + fn count_error_codes(codes: Vec>, context: &Context) { let unwrap_context_string = |context_key: &str| -> String { context .get::<_, String>(context_key) @@ -835,8 +833,7 @@ impl RouterService { let client_version = unwrap_context_string(CLIENT_VERSION); let mut map = HashMap::new(); - for error in errors { - let code = error.extensions.get("code").and_then(|c| c.as_str()); + for code in &codes { let entry = map.entry(code).or_insert(0u64); *entry += 1; From f98b6089902c5f26a2630c61e86d07aed9ae9f1a Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Wed, 8 Jan 2025 15:40:35 +1000 Subject: [PATCH 05/17] lint fix --- apollo-router/src/services/router/service.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index 607229a0c8..cc968f8f2e 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -814,7 +814,10 @@ impl RouterService { } fn count_errors(errors: &[graphql::Error], context: &Context) { - let codes = errors.iter().map(|e| e.extensions.get("code").and_then(|c| c.as_str())).collect(); + let codes = errors + .iter() + .map(|e| e.extensions.get("code").and_then(|c| c.as_str())) + .collect(); Self::count_error_codes(codes, context); } From 90eaebd7cd50be178c1a33bb86e0172256991873 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 13 Jan 2025 15:55:06 +1000 Subject: [PATCH 06/17] Configuration --- apollo-router/src/plugins/telemetry/apollo.rs | 14 +++++ apollo-router/src/services/router/service.rs | 56 +++++++++++++------ 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/apollo-router/src/plugins/telemetry/apollo.rs b/apollo-router/src/plugins/telemetry/apollo.rs index 97ac2fbdac..568cd9e83e 100644 --- a/apollo-router/src/plugins/telemetry/apollo.rs +++ b/apollo-router/src/plugins/telemetry/apollo.rs @@ -121,6 +121,9 @@ pub(crate) struct Config { pub(crate) struct ErrorsConfiguration { /// Handling of errors coming from subgraph pub(crate) subgraph: SubgraphErrorConfig, + + /// Configuration for storing and sending error metrics via OTLP + pub(crate) experimental_otlp_error_metrics: OtlpErrorMetricsMode, } #[derive(Debug, Clone, Deserialize, JsonSchema, Default)] @@ -160,6 +163,17 @@ impl SubgraphErrorConfig { } } +/// Open Telemetry error metrics mode +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, Copy)] +#[serde(deny_unknown_fields, rename_all = "lowercase")] +pub(crate) enum OtlpErrorMetricsMode { + /// Do not store OTLP error metrics + #[default] + Disabled, + /// Send OTLP error metrics to Apollo Studio + Enabled, +} + const fn default_field_level_instrumentation_sampler() -> SamplerOption { SamplerOption::TraceIdRatioBased(0.01) } diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index cc968f8f2e..f7b96ae40d 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -48,6 +48,8 @@ use crate::graphql; use crate::http_ext; #[cfg(test)] use crate::plugin::test::MockSupergraphService; +use crate::plugins::telemetry::apollo::OtlpErrorMetricsMode; +use crate::plugins::telemetry::config::Conf; use crate::plugins::telemetry::config_new::attributes::HTTP_REQUEST_BODY; use crate::plugins::telemetry::config_new::attributes::HTTP_REQUEST_HEADERS; use crate::plugins::telemetry::config_new::attributes::HTTP_REQUEST_URI; @@ -107,6 +109,7 @@ pub(crate) struct RouterService { persisted_query_layer: Arc, query_analysis_layer: QueryAnalysisLayer, batching: Batching, + oltp_error_metrics_mode: OtlpErrorMetricsMode, } impl RouterService { @@ -116,6 +119,7 @@ impl RouterService { persisted_query_layer: Arc, query_analysis_layer: QueryAnalysisLayer, batching: Batching, + oltp_error_metrics_mode: OtlpErrorMetricsMode, ) -> Self { RouterService { supergraph_creator, @@ -123,6 +127,7 @@ impl RouterService { persisted_query_layer, query_analysis_layer, batching, + oltp_error_metrics_mode, } } } @@ -314,7 +319,7 @@ impl RouterService { && (accepts_json || accepts_wildcard) { if !response.errors.is_empty() { - Self::count_errors(&response.errors, &context); + self.count_errors(&response.errors, &context); } parts @@ -351,7 +356,7 @@ impl RouterService { } if !response.errors.is_empty() { - Self::count_errors(&response.errors, &context); + self.count_errors(&response.errors, &context); } // Useful when you're using a proxy like nginx which enable proxy_buffering by default (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) @@ -378,7 +383,7 @@ impl RouterService { Ok(RouterResponse { response, context }) } else { - Self::count_error_codes(vec![Some("INVALID_ACCEPT_HEADER")], &context); + self.count_error_codes(vec![Some("INVALID_ACCEPT_HEADER")], &context); // Useful for selector in spans/instruments/events context.insert_json_value( @@ -813,15 +818,15 @@ impl RouterService { Ok(graphql_requests) } - fn count_errors(errors: &[graphql::Error], context: &Context) { + fn count_errors(&self, errors: &[graphql::Error], context: &Context) { let codes = errors .iter() .map(|e| e.extensions.get("code").and_then(|c| c.as_str())) .collect(); - Self::count_error_codes(codes, context); + self.count_error_codes(codes, context); } - fn count_error_codes(codes: Vec>, context: &Context) { + fn count_error_codes(&self, codes: Vec>, context: &Context) { let unwrap_context_string = |context_key: &str| -> String { context .get::<_, String>(context_key) @@ -840,18 +845,20 @@ impl RouterService { let entry = map.entry(code).or_insert(0u64); *entry += 1; - let code_str = code.unwrap_or_default().to_string(); - u64_counter!( - "apollo.router.operations.error", - "Number of errors returned by operation", - 1, - "apollo.operation.id" = operation_id.clone(), - "graphql.operation.name" = operation_name.clone(), - "graphql.operation.type" = operation_kind.clone(), - "apollo.client.name" = client_name.clone(), - "apollo.client.version" = client_version.clone(), - "graphql.error.extensions.code" = code_str - ); + if matches!(self.oltp_error_metrics_mode, OtlpErrorMetricsMode::Enabled) { + let code_str = code.unwrap_or_default().to_string(); + u64_counter!( + "apollo.router.operations.error", + "Number of errors returned by operation", + 1, + "apollo.operation.id" = operation_id.clone(), + "graphql.operation.name" = operation_name.clone(), + "graphql.operation.type" = operation_kind.clone(), + "apollo.client.name" = client_name.clone(), + "apollo.client.version" = client_version.clone(), + "graphql.error.extensions.code" = code_str + ); + } } for (code, count) in map { @@ -899,6 +906,7 @@ pub(crate) struct RouterCreator { pub(crate) persisted_query_layer: Arc, query_analysis_layer: QueryAnalysisLayer, batching: Batching, + oltp_error_metrics_mode: OtlpErrorMetricsMode, } impl ServiceFactory for RouterCreator { @@ -950,6 +958,16 @@ impl RouterCreator { // For now just call activate to make the gauges work on the happy path. apq_layer.activate(); + let oltp_error_metrics_mode = match configuration.apollo_plugins.plugins.get("telemetry") { + Some(telemetry_config) => { + match serde_json::from_value::(telemetry_config.clone()) { + Ok(conf) => conf.apollo.errors.experimental_otlp_error_metrics, + _ => OtlpErrorMetricsMode::default(), + } + } + _ => OtlpErrorMetricsMode::default(), + }; + Ok(Self { supergraph_creator, static_page, @@ -957,6 +975,7 @@ impl RouterCreator { query_analysis_layer, persisted_query_layer, batching: configuration.batching.clone(), + oltp_error_metrics_mode, }) } @@ -974,6 +993,7 @@ impl RouterCreator { self.persisted_query_layer.clone(), self.query_analysis_layer.clone(), self.batching.clone(), + self.oltp_error_metrics_mode.clone(), )); ServiceBuilder::new() From 498bbeb459f3769d1a7101e28d1b66473e2c7bba Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 13 Jan 2025 17:13:20 +1000 Subject: [PATCH 07/17] lint fix --- apollo-router/src/services/router/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index f7b96ae40d..a2cbe681cb 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -993,7 +993,7 @@ impl RouterCreator { self.persisted_query_layer.clone(), self.query_analysis_layer.clone(), self.batching.clone(), - self.oltp_error_metrics_mode.clone(), + self.oltp_error_metrics_mode, )); ServiceBuilder::new() From d2e261064a9ea79d5a001eee146b1dab77b69fc7 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 13 Jan 2025 18:42:43 +1000 Subject: [PATCH 08/17] snapshot update --- ...nfiguration__tests__schema_generation.snap | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 1ad8e2ef17..e540528453 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -2651,6 +2651,10 @@ expression: "&schema" "ErrorsConfiguration": { "additionalProperties": false, "properties": { + "experimental_otlp_error_metrics": { + "$ref": "#/definitions/OtlpErrorMetricsMode", + "description": "#/definitions/OtlpErrorMetricsMode" + }, "subgraph": { "$ref": "#/definitions/SubgraphErrorConfig", "description": "#/definitions/SubgraphErrorConfig" @@ -4357,6 +4361,25 @@ expression: "&schema" } ] }, + "OtlpErrorMetricsMode": { + "description": "Open Telemetry error metrics mode", + "oneOf": [ + { + "description": "Do not store OTLP error metrics", + "enum": [ + "disabled" + ], + "type": "string" + }, + { + "description": "Send OTLP error metrics to Apollo Studio", + "enum": [ + "enabled" + ], + "type": "string" + } + ] + }, "PersistedQueries": { "additionalProperties": false, "description": "Persisted Queries (PQ) configuration", From a46fee75b562ce5ee2022099f4a4ea9fbd765927 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Thu, 16 Jan 2025 06:25:27 +1000 Subject: [PATCH 09/17] Unit test --- apollo-router/src/services/router/tests.rs | 107 +++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/apollo-router/src/services/router/tests.rs b/apollo-router/src/services/router/tests.rs index 0adf3cacb1..052a3c0819 100644 --- a/apollo-router/src/services/router/tests.rs +++ b/apollo-router/src/services/router/tests.rs @@ -9,13 +9,21 @@ use http::HeaderValue; use http::Method; use http::Uri; use mime::APPLICATION_JSON; +use opentelemetry_api::KeyValue; use serde_json_bytes::json; use tower::ServiceExt; use tower_service::Service; +use crate::context::OPERATION_KIND; +use crate::context::OPERATION_NAME; use crate::graphql; +use crate::metrics::FutureMetricsExt; +use crate::plugins::telemetry::CLIENT_NAME; +use crate::plugins::telemetry::CLIENT_VERSION; +use crate::query_planner::APOLLO_OPERATION_ID; use crate::services::router; use crate::services::router::service::from_supergraph_mock_callback; +use crate::services::router::service::from_supergraph_mock_callback_and_configuration; use crate::services::router::service::process_vary_header; use crate::services::subgraph; use crate::services::supergraph; @@ -23,6 +31,7 @@ use crate::services::SupergraphRequest; use crate::services::SupergraphResponse; use crate::services::MULTIPART_DEFER_CONTENT_TYPE; use crate::test_harness::make_fake_batch; +use crate::Configuration; use crate::Context; // Test Vary processing @@ -590,3 +599,101 @@ async fn escaped_quotes_in_string_literal() { // The string literal made it through unchanged: assert!(subgraph_query.contains(r#"reviewsForAuthor(authorID: "\"1\"")"#)); } + +#[tokio::test] +async fn it_stores_operation_error_when_config_is_enabled() { + async { + let query = "query operationName { __typename }"; + let operation_name = "operationName"; + let operation_type = "query"; + let operation_id = "opId"; + let client_name = "client"; + let client_version = "version"; + + let mut config = Configuration::default(); + config.apollo_plugins.plugins.insert( + "telemetry".to_string(), + serde_json::json!({ + "apollo": { + "errors": { + "experimental_otlp_error_metrics": "enabled" + } + } + }), + ); + + let mut router_service = from_supergraph_mock_callback_and_configuration( + move |req| { + let example_response = graphql::Response::builder() + .data(json!({"data": null})) + .errors(vec![ + graphql::Error::builder() + .message("some error") + .extension_code("SOME_ERROR_CODE") + .build(), + graphql::Error::builder() + .message("some other error") + .extension_code("SOME_OTHER_ERROR_CODE") + .build(), + ]) + .build(); + + Ok(SupergraphResponse::new_from_graphql_response( + example_response, + req.context, + )) + }, + Arc::new(config), + ) + .await; + + let context = Context::new(); + context.insert_json_value(APOLLO_OPERATION_ID, operation_id.into()); + context.insert_json_value(OPERATION_NAME, operation_name.into()); + context.insert_json_value(OPERATION_KIND, query.into()); + context.insert_json_value(CLIENT_NAME, client_name.into()); + context.insert_json_value(CLIENT_VERSION, client_version.into()); + + let post_request = supergraph::Request::builder() + .query(query) + .operation_name(operation_name) + .header(CONTENT_TYPE, APPLICATION_JSON.essence_str()) + .uri(Uri::from_static("/")) + .method(Method::POST) + .context(context) + .build() + .unwrap(); + + router_service + .call(post_request.try_into().unwrap()) + .await + .unwrap(); + + assert_counter!( + "apollo.router.operations.error", + 1, + &[ + KeyValue::new("apollo.operation.id", operation_id), + KeyValue::new("graphql.operation.name", operation_name), + KeyValue::new("graphql.operation.type", operation_type), + KeyValue::new("apollo.client.name", client_name), + KeyValue::new("apollo.client.version", client_version), + KeyValue::new("graphql.error.extensions.code", "SOME_ERROR_CODE"), + ] + ); + assert_counter!( + "apollo.router.operations.error", + 1, + &[ + KeyValue::new("apollo.operation.id", operation_id), + KeyValue::new("graphql.operation.name", operation_name), + KeyValue::new("graphql.operation.type", operation_type), + KeyValue::new("apollo.client.name", client_name), + KeyValue::new("apollo.client.version", client_version), + KeyValue::new("graphql.error.extensions.code", "SOME_OTHER_ERROR_CODE"), + ] + ); + } + .with_metrics() + .await; +} From da4e95cd90ec0480e79668ee0d5d66508e57ff47 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Thu, 16 Jan 2025 06:27:52 +1000 Subject: [PATCH 10/17] update dev docs --- dev-docs/metrics.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-docs/metrics.md b/dev-docs/metrics.md index 37b8431d71..84d0b01e07 100644 --- a/dev-docs/metrics.md +++ b/dev-docs/metrics.md @@ -131,6 +131,8 @@ When using the macro in a test you will need a different pattern depending on if Make sure to use `.with_metrics()` method on the async block to ensure that the metrics are stored in a task local. *Tests will silently fail to record metrics if this is not done.* ```rust + use crate::metrics::FutureMetricsExt; + #[tokio::test(flavor = "multi_thread")] async fn test_async_multi() { // Multi-threaded runtime needs to use a tokio task local to avoid tests interfering with each other From e7b18e9c272ba920c6d70269445e5cf36138d076 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Thu, 16 Jan 2025 06:37:47 +1000 Subject: [PATCH 11/17] Added changeset --- .changesets/exp_njm_operation_error_metrics.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changesets/exp_njm_operation_error_metrics.md diff --git a/.changesets/exp_njm_operation_error_metrics.md b/.changesets/exp_njm_operation_error_metrics.md new file mode 100644 index 0000000000..1eaf7f2dad --- /dev/null +++ b/.changesets/exp_njm_operation_error_metrics.md @@ -0,0 +1,13 @@ +### Experimental per-operation error metrics ([PR #6443](https://github.com/apollographql/router/pull/6443)) + +Adds a new experimental OpenTelemetry metric that includes error counts at a per-operation and per-client level. These metrics contain the following attributes: +* Operation name +* Operation type (query/mutation/subscription) +* Apollo operation ID +* Client name +* Client version +* Error code + +This metric is currently only sent to GraphOS and is not available in 3rd-party OTel destinations. + +By [@bonnici](https://github.com/bonnici) in https://github.com/apollographql/router/pull/6443 From 05616488d9c347333d66c7b6aa9a5c964633946d Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Fri, 17 Jan 2025 15:21:20 +1000 Subject: [PATCH 12/17] Refactor to make it easier to get path and service --- apollo-router/src/services/router/service.rs | 35 ++++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index a2cbe681cb..a179a3c17f 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -46,6 +46,8 @@ use crate::context::OPERATION_KIND; use crate::context::OPERATION_NAME; use crate::graphql; use crate::http_ext; +use crate::json_ext::Object; +use crate::json_ext::Value; #[cfg(test)] use crate::plugin::test::MockSupergraphService; use crate::plugins::telemetry::apollo::OtlpErrorMetricsMode; @@ -383,7 +385,7 @@ impl RouterService { Ok(RouterResponse { response, context }) } else { - self.count_error_codes(vec![Some("INVALID_ACCEPT_HEADER")], &context); + self.count_error_codes(vec!["INVALID_ACCEPT_HEADER"], &context); // Useful for selector in spans/instruments/events context.insert_json_value( @@ -818,15 +820,7 @@ impl RouterService { Ok(graphql_requests) } - fn count_errors(&self, errors: &[graphql::Error], context: &Context) { - let codes = errors - .iter() - .map(|e| e.extensions.get("code").and_then(|c| c.as_str())) - .collect(); - self.count_error_codes(codes, context); - } - - fn count_error_codes(&self, codes: Vec>, context: &Context) { + fn count_errors(&self, errors: &Vec, context: &Context) { let unwrap_context_string = |context_key: &str| -> String { context .get::<_, String>(context_key) @@ -841,7 +835,8 @@ impl RouterService { let client_version = unwrap_context_string(CLIENT_VERSION); let mut map = HashMap::new(); - for code in &codes { + for error in errors { + let code = error.extensions.get("code").and_then(|c| c.as_str()); let entry = map.entry(code).or_insert(0u64); *entry += 1; @@ -881,6 +876,24 @@ impl RouterService { } } } + + fn count_error_codes(&self, codes: Vec<&str>, context: &Context) { + let errors = codes + .iter() + .map(|c| { + let mut extensions = Object::new(); + extensions.insert("code", Value::String((*c).into())); + graphql::Error { + message: "".into(), + locations: vec![], + path: None, + extensions, + } + }) + .collect(); + + self.count_errors(&errors, context); + } } struct TranslateError<'a> { From 3a09ee8bbe779f07e4e1325e8270864387973e0c Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Thu, 23 Jan 2025 15:49:21 +1000 Subject: [PATCH 13/17] Update changeset --- .changesets/exp_njm_operation_error_metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changesets/exp_njm_operation_error_metrics.md b/.changesets/exp_njm_operation_error_metrics.md index 1eaf7f2dad..98c780f053 100644 --- a/.changesets/exp_njm_operation_error_metrics.md +++ b/.changesets/exp_njm_operation_error_metrics.md @@ -8,6 +8,6 @@ Adds a new experimental OpenTelemetry metric that includes error counts at a per * Client version * Error code -This metric is currently only sent to GraphOS and is not available in 3rd-party OTel destinations. +This metric is currently only sent to GraphOS and is not available in 3rd-party OTel destinations. The metric can be enabled using the configuration `telemetry.apollo.errors.experimental_otlp_error_metrics: enabled`. By [@bonnici](https://github.com/bonnici) in https://github.com/apollographql/router/pull/6443 From 872a8f3a2aabcb629fa763e13c79d4256c0e4f42 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 24 Jan 2025 12:23:33 +0100 Subject: [PATCH 14/17] opentelemetry_api::KeyValue is now in opentelemetry:: --- apollo-router/src/services/router/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo-router/src/services/router/tests.rs b/apollo-router/src/services/router/tests.rs index 052a3c0819..f04d9f9bb8 100644 --- a/apollo-router/src/services/router/tests.rs +++ b/apollo-router/src/services/router/tests.rs @@ -9,7 +9,7 @@ use http::HeaderValue; use http::Method; use http::Uri; use mime::APPLICATION_JSON; -use opentelemetry_api::KeyValue; +use opentelemetry::KeyValue; use serde_json_bytes::json; use tower::ServiceExt; use tower_service::Service; From 72b404990ef3ff04ef318d6a9c36217a912c6ab4 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Tue, 28 Jan 2025 13:58:51 +1000 Subject: [PATCH 15/17] Fix self references --- apollo-router/src/services/router/service.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index 066f7e9c56..413b392730 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -267,6 +267,8 @@ impl RouterService { request_res = self.apq_layer.supergraph_request(supergraph_request).await; } + let oltp_error_metrics_mode = self.oltp_error_metrics_mode.clone(); + let SupergraphResponse { response, context } = match request_res { Err(response) => response, Ok(request) => match self.query_analysis_layer.supergraph_request(request).await { @@ -343,7 +345,7 @@ impl RouterService { && (accepts_json || accepts_wildcard) { if !response.errors.is_empty() { - // self.count_errors(&response.errors, &context); + Self::count_errors(&response.errors, &context, &oltp_error_metrics_mode); } parts @@ -380,7 +382,7 @@ impl RouterService { } if !response.errors.is_empty() { - self.count_errors(&response.errors, &context); + Self::count_errors(&response.errors, &context, &oltp_error_metrics_mode); } // Useful when you're using a proxy like nginx which enable proxy_buffering by default (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) @@ -407,7 +409,7 @@ impl RouterService { Ok(RouterResponse { response, context }) } else { - self.count_error_codes(vec!["INVALID_ACCEPT_HEADER"], &context); + Self::count_error_codes(vec!["INVALID_ACCEPT_HEADER"], &context, &oltp_error_metrics_mode); // this should be unreachable due to a previous check, but just to be sure... Ok(router::Response::error_builder() @@ -837,7 +839,7 @@ impl RouterService { Ok(graphql_requests) } - fn count_errors(&self, errors: &Vec, context: &Context) { + fn count_errors(errors: &Vec, context: &Context, oltp_error_metrics_mode: &OtlpErrorMetricsMode) { let unwrap_context_string = |context_key: &str| -> String { context .get::<_, String>(context_key) @@ -857,7 +859,7 @@ impl RouterService { let entry = map.entry(code).or_insert(0u64); *entry += 1; - if matches!(self.oltp_error_metrics_mode, OtlpErrorMetricsMode::Enabled) { + if matches!(oltp_error_metrics_mode, OtlpErrorMetricsMode::Enabled) { let code_str = code.unwrap_or_default().to_string(); u64_counter!( "apollo.router.operations.error", @@ -894,7 +896,7 @@ impl RouterService { } } - fn count_error_codes(&self, codes: Vec<&str>, context: &Context) { + fn count_error_codes(codes: Vec<&str>, context: &Context, oltp_error_metrics_mode: &OtlpErrorMetricsMode) { let errors = codes .iter() .map(|c| { @@ -909,7 +911,7 @@ impl RouterService { }) .collect(); - self.count_errors(&errors, context); + Self::count_errors(&errors, context, oltp_error_metrics_mode); } } @@ -984,7 +986,7 @@ impl RouterCreator { // For now just call activate to make the gauges work on the happy path. apq_layer.activate(); - let oltp_error_metrics_mode = match configuration.apollo_plugins.plugins.get("telemetry") { + let oltp_error_metrics_mode: OtlpErrorMetricsMode = match configuration.apollo_plugins.plugins.get("telemetry") { Some(telemetry_config) => { match serde_json::from_value::(telemetry_config.clone()) { Ok(conf) => conf.apollo.errors.experimental_otlp_error_metrics, From a4a33e4be483dfcaf389655d34aa484c62e84325 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Tue, 28 Jan 2025 14:01:10 +1000 Subject: [PATCH 16/17] Format --- apollo-router/src/services/router/service.rs | 39 +++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index 413b392730..da671ea47a 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -44,10 +44,10 @@ use crate::configuration::Batching; use crate::configuration::BatchingMode; use crate::context::OPERATION_KIND; use crate::context::OPERATION_NAME; -use crate::json_ext::Object; -use crate::json_ext::Value; use crate::graphql; use crate::http_ext; +use crate::json_ext::Object; +use crate::json_ext::Value; use crate::layers::ServiceBuilderExt; use crate::layers::DEFAULT_BUFFER_SIZE; #[cfg(test)] @@ -409,7 +409,11 @@ impl RouterService { Ok(RouterResponse { response, context }) } else { - Self::count_error_codes(vec!["INVALID_ACCEPT_HEADER"], &context, &oltp_error_metrics_mode); + Self::count_error_codes( + vec!["INVALID_ACCEPT_HEADER"], + &context, + &oltp_error_metrics_mode, + ); // this should be unreachable due to a previous check, but just to be sure... Ok(router::Response::error_builder() @@ -839,7 +843,11 @@ impl RouterService { Ok(graphql_requests) } - fn count_errors(errors: &Vec, context: &Context, oltp_error_metrics_mode: &OtlpErrorMetricsMode) { + fn count_errors( + errors: &Vec, + context: &Context, + oltp_error_metrics_mode: &OtlpErrorMetricsMode, + ) { let unwrap_context_string = |context_key: &str| -> String { context .get::<_, String>(context_key) @@ -896,7 +904,11 @@ impl RouterService { } } - fn count_error_codes(codes: Vec<&str>, context: &Context, oltp_error_metrics_mode: &OtlpErrorMetricsMode) { + fn count_error_codes( + codes: Vec<&str>, + context: &Context, + oltp_error_metrics_mode: &OtlpErrorMetricsMode, + ) { let errors = codes .iter() .map(|c| { @@ -986,15 +998,16 @@ impl RouterCreator { // For now just call activate to make the gauges work on the happy path. apq_layer.activate(); - let oltp_error_metrics_mode: OtlpErrorMetricsMode = match configuration.apollo_plugins.plugins.get("telemetry") { - Some(telemetry_config) => { - match serde_json::from_value::(telemetry_config.clone()) { - Ok(conf) => conf.apollo.errors.experimental_otlp_error_metrics, - _ => OtlpErrorMetricsMode::default(), + let oltp_error_metrics_mode: OtlpErrorMetricsMode = + match configuration.apollo_plugins.plugins.get("telemetry") { + Some(telemetry_config) => { + match serde_json::from_value::(telemetry_config.clone()) { + Ok(conf) => conf.apollo.errors.experimental_otlp_error_metrics, + _ => OtlpErrorMetricsMode::default(), + } } - } - _ => OtlpErrorMetricsMode::default(), - }; + _ => OtlpErrorMetricsMode::default(), + }; let router_service = content_negotiation::RouterLayer::default().layer(RouterService::new( supergraph_creator.create(), From 7151ef089d538f40965eaa7a0427129f66818fc2 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Tue, 28 Jan 2025 14:19:49 +1000 Subject: [PATCH 17/17] Remove test --- apollo-router/src/services/router/service.rs | 16 ++- apollo-router/src/services/router/tests.rs | 107 ------------------- 2 files changed, 11 insertions(+), 112 deletions(-) diff --git a/apollo-router/src/services/router/service.rs b/apollo-router/src/services/router/service.rs index da671ea47a..922deb4263 100644 --- a/apollo-router/src/services/router/service.rs +++ b/apollo-router/src/services/router/service.rs @@ -267,8 +267,6 @@ impl RouterService { request_res = self.apq_layer.supergraph_request(supergraph_request).await; } - let oltp_error_metrics_mode = self.oltp_error_metrics_mode.clone(); - let SupergraphResponse { response, context } = match request_res { Err(response) => response, Ok(request) => match self.query_analysis_layer.supergraph_request(request).await { @@ -345,7 +343,11 @@ impl RouterService { && (accepts_json || accepts_wildcard) { if !response.errors.is_empty() { - Self::count_errors(&response.errors, &context, &oltp_error_metrics_mode); + Self::count_errors( + &response.errors, + &context, + &self.oltp_error_metrics_mode, + ); } parts @@ -382,7 +384,11 @@ impl RouterService { } if !response.errors.is_empty() { - Self::count_errors(&response.errors, &context, &oltp_error_metrics_mode); + Self::count_errors( + &response.errors, + &context, + &self.oltp_error_metrics_mode, + ); } // Useful when you're using a proxy like nginx which enable proxy_buffering by default (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) @@ -412,7 +418,7 @@ impl RouterService { Self::count_error_codes( vec!["INVALID_ACCEPT_HEADER"], &context, - &oltp_error_metrics_mode, + &self.oltp_error_metrics_mode, ); // this should be unreachable due to a previous check, but just to be sure... diff --git a/apollo-router/src/services/router/tests.rs b/apollo-router/src/services/router/tests.rs index fca2564ce3..b018196dac 100644 --- a/apollo-router/src/services/router/tests.rs +++ b/apollo-router/src/services/router/tests.rs @@ -9,21 +9,13 @@ use http::HeaderValue; use http::Method; use http::Uri; use mime::APPLICATION_JSON; -use opentelemetry::KeyValue; use serde_json_bytes::json; use tower::ServiceExt; use tower_service::Service; -use crate::context::OPERATION_KIND; -use crate::context::OPERATION_NAME; use crate::graphql; -use crate::metrics::FutureMetricsExt; -use crate::plugins::telemetry::CLIENT_NAME; -use crate::plugins::telemetry::CLIENT_VERSION; -use crate::query_planner::APOLLO_OPERATION_ID; use crate::services::router; use crate::services::router::service::from_supergraph_mock_callback; -use crate::services::router::service::from_supergraph_mock_callback_and_configuration; use crate::services::router::service::process_vary_header; use crate::services::subgraph; use crate::services::supergraph; @@ -31,7 +23,6 @@ use crate::services::SupergraphRequest; use crate::services::SupergraphResponse; use crate::services::MULTIPART_DEFER_CONTENT_TYPE; use crate::test_harness::make_fake_batch; -use crate::Configuration; use crate::Context; // Test Vary processing @@ -608,101 +599,3 @@ async fn escaped_quotes_in_string_literal() { // The string literal made it through unchanged: assert!(subgraph_query.contains(r#"reviewsForAuthor(authorID: "\"1\"")"#)); } - -#[tokio::test] -async fn it_stores_operation_error_when_config_is_enabled() { - async { - let query = "query operationName { __typename }"; - let operation_name = "operationName"; - let operation_type = "query"; - let operation_id = "opId"; - let client_name = "client"; - let client_version = "version"; - - let mut config = Configuration::default(); - config.apollo_plugins.plugins.insert( - "telemetry".to_string(), - serde_json::json!({ - "apollo": { - "errors": { - "experimental_otlp_error_metrics": "enabled" - } - } - }), - ); - - let mut router_service = from_supergraph_mock_callback_and_configuration( - move |req| { - let example_response = graphql::Response::builder() - .data(json!({"data": null})) - .errors(vec![ - graphql::Error::builder() - .message("some error") - .extension_code("SOME_ERROR_CODE") - .build(), - graphql::Error::builder() - .message("some other error") - .extension_code("SOME_OTHER_ERROR_CODE") - .build(), - ]) - .build(); - - Ok(SupergraphResponse::new_from_graphql_response( - example_response, - req.context, - )) - }, - Arc::new(config), - ) - .await; - - let context = Context::new(); - context.insert_json_value(APOLLO_OPERATION_ID, operation_id.into()); - context.insert_json_value(OPERATION_NAME, operation_name.into()); - context.insert_json_value(OPERATION_KIND, query.into()); - context.insert_json_value(CLIENT_NAME, client_name.into()); - context.insert_json_value(CLIENT_VERSION, client_version.into()); - - let post_request = supergraph::Request::builder() - .query(query) - .operation_name(operation_name) - .header(CONTENT_TYPE, APPLICATION_JSON.essence_str()) - .uri(Uri::from_static("/")) - .method(Method::POST) - .context(context) - .build() - .unwrap(); - - router_service - .call(post_request.try_into().unwrap()) - .await - .unwrap(); - - assert_counter!( - "apollo.router.operations.error", - 1, - &[ - KeyValue::new("apollo.operation.id", operation_id), - KeyValue::new("graphql.operation.name", operation_name), - KeyValue::new("graphql.operation.type", operation_type), - KeyValue::new("apollo.client.name", client_name), - KeyValue::new("apollo.client.version", client_version), - KeyValue::new("graphql.error.extensions.code", "SOME_ERROR_CODE"), - ] - ); - assert_counter!( - "apollo.router.operations.error", - 1, - &[ - KeyValue::new("apollo.operation.id", operation_id), - KeyValue::new("graphql.operation.name", operation_name), - KeyValue::new("graphql.operation.type", operation_type), - KeyValue::new("apollo.client.name", client_name), - KeyValue::new("apollo.client.version", client_version), - KeyValue::new("graphql.error.extensions.code", "SOME_OTHER_ERROR_CODE"), - ] - ); - } - .with_metrics() - .await; -}