diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e9333d66c..d600429163 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 24.10.0 **Breaking Changes:** @@ -30,6 +30,8 @@ - Use custom wildcard matching instead of regular expressions. ([#4073](https://github.com/getsentry/relay/pull/4073)) - Allowlist the SentryUptimeBot user-agent. ([#4068](https://github.com/getsentry/relay/pull/4068)) - Feature flags of graduated features are now hard-coded in Relay so they can be removed from Sentry. ([#4076](https://github.com/getsentry/relay/pull/4076), [#4080](https://github.com/getsentry/relay/pull/4080)) +- Add parallelization in Redis commands. ([#4118](https://github.com/getsentry/relay/pull/4118)) +- Extract user ip for spans. ([#4144](https://github.com/getsentry/relay/pull/4144)) ## 24.9.0 diff --git a/Cargo.lock b/Cargo.lock index 20f71c802e..b5703e7f5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,28 +197,6 @@ dependencies = [ "zstd-safe", ] -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.77", -] - [[package]] name = "async-trait" version = "0.1.68" @@ -419,7 +397,7 @@ dependencies = [ "bitflags 2.4.1", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -1547,7 +1525,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.2.5", + "indexmap", "slab", "tokio", "tokio-util", @@ -1573,12 +1551,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.5" @@ -1595,7 +1567,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "hashbrown 0.14.5", + "hashbrown", ] [[package]] @@ -1815,19 +1787,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-timeout" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "hyper-tls" version = "0.6.0" @@ -1914,16 +1873,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "indexmap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.2.5" @@ -1931,7 +1880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown", ] [[package]] @@ -2216,7 +2165,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ - "hashbrown 0.14.5", + "hashbrown", ] [[package]] @@ -2707,12 +2656,10 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "glob", "once_cell", "opentelemetry", "percent-encoding", "rand", - "serde_json", "thiserror", ] @@ -2975,7 +2922,7 @@ checksum = "70c501afe3a2e25c9bd219aa56ec1e04cdb3fcdd763055be268778c13fa82c1f" dependencies = [ "autocfg", "equivalent", - "indexmap 2.2.5", + "indexmap", ] [[package]] @@ -3037,7 +2984,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.77", @@ -3187,8 +3134,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.27.2" -source = "git+https://github.com/getsentry/redis-rs.git?rev=fc7d98cc10c16fa7c0c31de64dc1b713354a4384#fc7d98cc10c16fa7c0c31de64dc1b713354a4384" +version = "0.27.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6baebe319ef5e4b470f248335620098d1c2e9261e995be05f56f719ca4bdb2" dependencies = [ "arc-swap", "combine", @@ -3266,7 +3214,7 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "relay" -version = "24.9.0" +version = "24.10.0" dependencies = [ "anyhow", "clap", @@ -3285,7 +3233,7 @@ dependencies = [ [[package]] name = "relay-auth" -version = "24.9.0" +version = "24.10.0" dependencies = [ "chrono", "data-encoding", @@ -3302,7 +3250,7 @@ dependencies = [ [[package]] name = "relay-base-schema" -version = "24.9.0" +version = "24.10.0" dependencies = [ "regex", "relay-common", @@ -3341,11 +3289,11 @@ dependencies = [ [[package]] name = "relay-cardinality" -version = "24.9.0" +version = "24.10.0" dependencies = [ "criterion", "hash32", - "hashbrown 0.14.5", + "hashbrown", "parking_lot", "relay-base-schema", "relay-common", @@ -3360,11 +3308,11 @@ dependencies = [ [[package]] name = "relay-cogs" -version = "24.9.0" +version = "24.10.0" [[package]] name = "relay-common" -version = "24.9.0" +version = "24.10.0" dependencies = [ "chrono", "criterion", @@ -3378,7 +3326,7 @@ dependencies = [ [[package]] name = "relay-config" -version = "24.9.0" +version = "24.10.0" dependencies = [ "anyhow", "human-size", @@ -3400,7 +3348,7 @@ dependencies = [ [[package]] name = "relay-crash" -version = "24.9.0" +version = "24.10.0" dependencies = [ "bindgen", "cmake", @@ -3408,7 +3356,7 @@ dependencies = [ [[package]] name = "relay-dynamic-config" -version = "24.9.0" +version = "24.10.0" dependencies = [ "anyhow", "relay-auth", @@ -3431,7 +3379,7 @@ dependencies = [ [[package]] name = "relay-event-derive" -version = "24.9.0" +version = "24.10.0" dependencies = [ "proc-macro2", "quote", @@ -3441,7 +3389,7 @@ dependencies = [ [[package]] name = "relay-event-normalization" -version = "24.9.0" +version = "24.10.0" dependencies = [ "bytecount", "chrono", @@ -3475,7 +3423,7 @@ dependencies = [ [[package]] name = "relay-event-schema" -version = "24.9.0" +version = "24.10.0" dependencies = [ "bytecount", "chrono", @@ -3498,7 +3446,7 @@ dependencies = [ [[package]] name = "relay-ffi" -version = "24.9.0" +version = "24.10.0" dependencies = [ "anyhow", "relay-ffi-macros", @@ -3506,7 +3454,7 @@ dependencies = [ [[package]] name = "relay-ffi-macros" -version = "24.9.0" +version = "24.10.0" dependencies = [ "quote", "syn 1.0.109", @@ -3514,9 +3462,9 @@ dependencies = [ [[package]] name = "relay-filter" -version = "24.9.0" +version = "24.10.0" dependencies = [ - "indexmap 2.2.5", + "indexmap", "insta", "ipnetwork", "once_cell", @@ -3533,7 +3481,7 @@ dependencies = [ [[package]] name = "relay-kafka" -version = "24.9.0" +version = "24.10.0" dependencies = [ "rdkafka", "rdkafka-sys", @@ -3549,7 +3497,7 @@ dependencies = [ [[package]] name = "relay-log" -version = "24.9.0" +version = "24.10.0" dependencies = [ "console", "relay-common", @@ -3563,14 +3511,14 @@ dependencies = [ [[package]] name = "relay-metrics" -version = "24.9.0" +version = "24.10.0" dependencies = [ "bytecount", "chrono", "criterion", "fnv", "hash32", - "hashbrown 0.14.5", + "hashbrown", "insta", "itertools 0.13.0", "priority-queue", @@ -3595,7 +3543,7 @@ dependencies = [ [[package]] name = "relay-monitors" -version = "24.9.0" +version = "24.10.0" dependencies = [ "relay-base-schema", "serde", @@ -3607,7 +3555,7 @@ dependencies = [ [[package]] name = "relay-pattern" -version = "24.9.0" +version = "24.10.0" dependencies = [ "criterion", "memchr", @@ -3617,7 +3565,7 @@ dependencies = [ [[package]] name = "relay-pattern-fuzz" -version = "0.0.0" +version = "24.10.0" dependencies = [ "libfuzzer-sys", "relay-pattern", @@ -3625,7 +3573,7 @@ dependencies = [ [[package]] name = "relay-pii" -version = "24.9.0" +version = "24.10.0" dependencies = [ "hmac", "insta", @@ -3652,7 +3600,7 @@ dependencies = [ [[package]] name = "relay-profiling" -version = "24.9.0" +version = "24.10.0" dependencies = [ "android_trace_log", "chrono", @@ -3672,7 +3620,7 @@ dependencies = [ [[package]] name = "relay-protocol" -version = "24.9.0" +version = "24.10.0" dependencies = [ "insta", "num-traits", @@ -3689,7 +3637,7 @@ dependencies = [ [[package]] name = "relay-protocol-derive" -version = "24.9.0" +version = "24.10.0" dependencies = [ "proc-macro2", "quote", @@ -3699,9 +3647,9 @@ dependencies = [ [[package]] name = "relay-quotas" -version = "24.9.0" +version = "24.10.0" dependencies = [ - "hashbrown 0.14.5", + "hashbrown", "insta", "itertools 0.13.0", "relay-base-schema", @@ -3717,7 +3665,7 @@ dependencies = [ [[package]] name = "relay-redis" -version = "24.9.0" +version = "24.10.0" dependencies = [ "r2d2", "redis", @@ -3729,7 +3677,7 @@ dependencies = [ [[package]] name = "relay-replays" -version = "24.9.0" +version = "24.10.0" dependencies = [ "criterion", "flate2", @@ -3746,7 +3694,7 @@ dependencies = [ [[package]] name = "relay-sampling" -version = "24.9.0" +version = "24.10.0" dependencies = [ "anyhow", "chrono", @@ -3765,7 +3713,7 @@ dependencies = [ [[package]] name = "relay-server" -version = "24.9.0" +version = "24.10.0" dependencies = [ "anyhow", "arc-swap", @@ -3782,7 +3730,7 @@ dependencies = [ "flate2", "fnv", "futures", - "hashbrown 0.14.5", + "hashbrown", "http", "hyper-util", "insta", @@ -3849,7 +3797,7 @@ dependencies = [ [[package]] name = "relay-spans" -version = "24.9.0" +version = "24.10.0" dependencies = [ "chrono", "hex", @@ -3863,7 +3811,7 @@ dependencies = [ [[package]] name = "relay-statsd" -version = "24.9.0" +version = "24.10.0" dependencies = [ "cadence", "crossbeam-channel", @@ -3875,7 +3823,7 @@ dependencies = [ [[package]] name = "relay-system" -version = "24.9.0" +version = "24.10.0" dependencies = [ "futures", "once_cell", @@ -3886,7 +3834,7 @@ dependencies = [ [[package]] name = "relay-test" -version = "24.9.0" +version = "24.10.0" dependencies = [ "relay-log", "relay-system", @@ -3895,7 +3843,7 @@ dependencies = [ [[package]] name = "relay-ua" -version = "24.9.0" +version = "24.10.0" dependencies = [ "once_cell", "uaparser", @@ -4431,7 +4379,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.5", + "indexmap", "itoa", "ryu", "serde", @@ -4659,10 +4607,10 @@ dependencies = [ "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.14.5", + "hashbrown", "hashlink", "hex", - "indexmap 2.2.5", + "indexmap", "log", "memchr", "once_cell", @@ -5179,9 +5127,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -5213,36 +5161,27 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.5", + "indexmap", "toml_datetime", "winnow", ] [[package]] name = "tonic" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum", "base64 0.22.1", "bytes", - "h2", "http", "http-body", "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", "percent-encoding", "pin-project", "prost", - "socket2", - "tokio", "tokio-stream", - "tower", "tower-layer", "tower-service", "tracing", @@ -5256,13 +5195,9 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap 1.9.2", "pin-project", "pin-project-lite", - "rand", - "slab", "tokio", - "tokio-util", "tower-layer", "tower-service", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 100fca3242..2865dccce0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,7 +118,7 @@ multer = "3.1.0" num-traits = "0.2.18" num_cpus = "1.13.0" once_cell = "1.13.1" -opentelemetry-proto = "0.7.0" +opentelemetry-proto = { version = "0.7.0", default-features = false } parking_lot = "0.12.1" path-slash = "0.2.1" pest = "2.1.3" @@ -135,8 +135,7 @@ rand_pcg = "0.3.1" rayon = "1.10" rdkafka = "0.36.2" rdkafka-sys = "4.3.0" -# Git revision until https://github.com/redis-rs/redis-rs/pull/1097 (merged) and https://github.com/redis-rs/redis-rs/pull/1290 are released. -redis = { git = "https://github.com/getsentry/redis-rs.git", rev = "fc7d98cc10c16fa7c0c31de64dc1b713354a4384", default-features = false } +redis = { version = "0.27.4", default-features = false } regex = "1.10.2" regex-lite = "0.1.6" reqwest = "0.12.7" diff --git a/relay-auth/Cargo.toml b/relay-auth/Cargo.toml index 9b597bb1df..d74f9f1148 100644 --- a/relay-auth/Cargo.toml +++ b/relay-auth/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Authentication and crypto for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-base-schema/Cargo.toml b/relay-base-schema/Cargo.toml index 9dd1e5b86f..769d01348b 100644 --- a/relay-base-schema/Cargo.toml +++ b/relay-base-schema/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Basic types for Relay's API schema used across multiple services" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-cardinality/Cargo.toml b/relay-cardinality/Cargo.toml index 7f6d5818af..b66eefb00f 100644 --- a/relay-cardinality/Cargo.toml +++ b/relay-cardinality/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Metrics Cardinality Limiter" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE" publish = false diff --git a/relay-cogs/Cargo.toml b/relay-cogs/Cargo.toml index 3b69068489..2897455b68 100644 --- a/relay-cogs/Cargo.toml +++ b/relay-cogs/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Break down the cost of Relay by its features" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE" publish = false diff --git a/relay-common/Cargo.toml b/relay-common/Cargo.toml index 5420ee7338..1967e8578c 100644 --- a/relay-common/Cargo.toml +++ b/relay-common/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Common utilities and crate re-exports for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-config/Cargo.toml b/relay-config/Cargo.toml index 642cdc251e..2faba0abbc 100644 --- a/relay-config/Cargo.toml +++ b/relay-config/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Configuration for the Relay CLI and server" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-config/src/redis.rs b/relay-config/src/redis.rs index c6350d5a21..90d29617c9 100644 --- a/relay-config/src/redis.rs +++ b/relay-config/src/redis.rs @@ -208,6 +208,7 @@ pub enum RedisConfigRef<'a> { } /// Helper struct bundling connections and options for the various Redis pools. +#[allow(clippy::large_enum_variant)] #[derive(Clone, Debug)] pub enum RedisPoolConfigs<'a> { /// Use one pool for everything. @@ -232,7 +233,7 @@ fn build_redis_config_options( let max_connections = options.max_connections.unwrap_or(default_connections); let min_idle = options .min_idle - .unwrap_or_else(|| max_connections.div_ceil(crate::redis::DEFAULT_MIN_IDLE_RATIO)); + .unwrap_or_else(|| max_connections.div_ceil(DEFAULT_MIN_IDLE_RATIO)); RedisConfigOptions { max_connections, @@ -253,7 +254,6 @@ pub(super) fn create_redis_pool( RedisConfig::Cluster { cluster_nodes, options, - .. } => RedisConfigRef::Cluster { cluster_nodes, options: build_redis_config_options(options, default_connections), @@ -280,10 +280,8 @@ pub(super) fn create_redis_pool( pub(super) fn create_redis_pools(configs: &RedisConfigs, cpu_concurrency: u32) -> RedisPoolConfigs { // Default `max_connections` for the `project_configs` pool. // In a unified config, this is used for all pools. - let project_configs_default_connections = std::cmp::max( - cpu_concurrency * 2, - crate::redis::DEFAULT_MIN_MAX_CONNECTIONS, - ); + let project_configs_default_connections = + std::cmp::max(cpu_concurrency * 2, DEFAULT_MIN_MAX_CONNECTIONS); match configs { RedisConfigs::Unified(cfg) => { diff --git a/relay-crash/Cargo.toml b/relay-crash/Cargo.toml index 382b7e8e1c..fa88b70855 100644 --- a/relay-crash/Cargo.toml +++ b/relay-crash/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Native crash reporting for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" build = "build.rs" license-file = "../LICENSE.md" diff --git a/relay-dynamic-config/Cargo.toml b/relay-dynamic-config/Cargo.toml index eebd57a1c2..cb2eb773b6 100644 --- a/relay-dynamic-config/Cargo.toml +++ b/relay-dynamic-config/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Dynamic configuration passed down from sentry" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-event-derive/Cargo.toml b/relay-event-derive/Cargo.toml index 2a69890564..6be12e0275 100644 --- a/relay-event-derive/Cargo.toml +++ b/relay-event-derive/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Derive for visitor traits on the Event schema" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-event-normalization/Cargo.toml b/relay-event-normalization/Cargo.toml index 9ee4a7d69e..4ad7d6edef 100644 --- a/relay-event-normalization/Cargo.toml +++ b/relay-event-normalization/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Event normalization and processing" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-event-normalization/src/normalize/span/tag_extraction.rs b/relay-event-normalization/src/normalize/span/tag_extraction.rs index fa634b31ad..9e6555cbd4 100644 --- a/relay-event-normalization/src/normalize/span/tag_extraction.rs +++ b/relay-event-normalization/src/normalize/span/tag_extraction.rs @@ -35,6 +35,7 @@ pub enum SpanTagKey { Release, User, UserID, + UserIP, UserUsername, UserEmail, Environment, @@ -100,6 +101,7 @@ impl SpanTagKey { SpanTagKey::Release => "release", SpanTagKey::User => "user", SpanTagKey::UserID => "user.id", + SpanTagKey::UserIP => "user.ip", SpanTagKey::UserUsername => "user.username", SpanTagKey::UserEmail => "user.email", SpanTagKey::UserCountryCode => "user.geo.country_code", @@ -300,6 +302,9 @@ fn extract_shared_tags(event: &Event) -> BTreeMap { if let Some(user_id) = user.id.value() { tags.insert(SpanTagKey::UserID, user_id.as_str().to_owned()); } + if let Some(user_ip) = user.ip_address.value() { + tags.insert(SpanTagKey::UserIP, user_ip.as_str().to_owned()); + } if let Some(user_username) = user.username.value() { tags.insert(SpanTagKey::UserUsername, user_username.as_str().to_owned()); } @@ -2647,6 +2652,7 @@ LIMIT 1 }, "user": { "id": "1", + "ip_address": "127.0.0.1", "email": "admin@sentry.io", "username": "admin", "geo": { @@ -2680,6 +2686,7 @@ LIMIT 1 assert_eq!(get_value!(span.sentry_tags["user"]!), "id:1"); assert_eq!(get_value!(span.sentry_tags["user.id"]!), "1"); + assert_eq!(get_value!(span.sentry_tags["user.ip"]!), "127.0.0.1"); assert_eq!(get_value!(span.sentry_tags["user.username"]!), "admin"); assert_eq!( get_value!(span.sentry_tags["user.email"]!), diff --git a/relay-event-schema/Cargo.toml b/relay-event-schema/Cargo.toml index e3d7a66b4f..017fec7f4d 100644 --- a/relay-event-schema/Cargo.toml +++ b/relay-event-schema/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Event schema (Error, Transaction, Security) and types for event processing" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-ffi-macros/Cargo.toml b/relay-ffi-macros/Cargo.toml index 74d80079c2..a25c01930a 100644 --- a/relay-ffi-macros/Cargo.toml +++ b/relay-ffi-macros/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Macros for error handling in FFI bindings" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-ffi/Cargo.toml b/relay-ffi/Cargo.toml index 37f6c25d55..61a5b85b4e 100644 --- a/relay-ffi/Cargo.toml +++ b/relay-ffi/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Utilities for error handling in FFI bindings" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-ffi/src/lib.rs b/relay-ffi/src/lib.rs index 3a2552a6ac..0b0c8b9b2a 100644 --- a/relay-ffi/src/lib.rs +++ b/relay-ffi/src/lib.rs @@ -231,7 +231,7 @@ pub fn take_last_error() -> Option { pub struct Panic(String); impl Panic { - fn new(info: &panic::PanicInfo) -> Self { + fn new(info: &panic::PanicHookInfo) -> Self { let thread = thread::current(); let thread = thread.name().unwrap_or("unnamed"); diff --git a/relay-filter/Cargo.toml b/relay-filter/Cargo.toml index e61b9c6a24..e1a5e98126 100644 --- a/relay-filter/Cargo.toml +++ b/relay-filter/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Inbound data filters for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-kafka/Cargo.toml b/relay-kafka/Cargo.toml index f64f7ac51f..e313c9a564 100644 --- a/relay-kafka/Cargo.toml +++ b/relay-kafka/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Kafka related functionality for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-log/Cargo.toml b/relay-log/Cargo.toml index 005357491e..c0e2734ca4 100644 --- a/relay-log/Cargo.toml +++ b/relay-log/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Error reporting and logging for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-metrics/Cargo.toml b/relay-metrics/Cargo.toml index 95e75669b5..62c18b0936 100644 --- a/relay-metrics/Cargo.toml +++ b/relay-metrics/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Metrics protocol and processing" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-monitors/Cargo.toml b/relay-monitors/Cargo.toml index 6985ad06ac..1d35c06df3 100644 --- a/relay-monitors/Cargo.toml +++ b/relay-monitors/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Monitors processing for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-pattern/Cargo.toml b/relay-pattern/Cargo.toml index e2a0ca2ae9..0b070b50bb 100644 --- a/relay-pattern/Cargo.toml +++ b/relay-pattern/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "A glob like pattern used throughout Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-pattern/fuzz/Cargo.toml b/relay-pattern/fuzz/Cargo.toml index 8d0750a09b..d525c55658 100644 --- a/relay-pattern/fuzz/Cargo.toml +++ b/relay-pattern/fuzz/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "relay-pattern-fuzz" -version = "0.0.0" +version = "24.10.0" publish = false edition = "2021" diff --git a/relay-pii/Cargo.toml b/relay-pii/Cargo.toml index 6dbe93551d..d15e2ce1dd 100644 --- a/relay-pii/Cargo.toml +++ b/relay-pii/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Scrubbing of personally identifiable information (PII) from events" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-profiling/Cargo.toml b/relay-profiling/Cargo.toml index 5e42825e2b..75485d3bca 100644 --- a/relay-profiling/Cargo.toml +++ b/relay-profiling/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Profiling processing for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-protocol-derive/Cargo.toml b/relay-protocol-derive/Cargo.toml index dacfdf3712..36dc11b15a 100644 --- a/relay-protocol-derive/Cargo.toml +++ b/relay-protocol-derive/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Derives for Relay's protocol traits" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-protocol/Cargo.toml b/relay-protocol/Cargo.toml index c231e19148..dde64725e6 100644 --- a/relay-protocol/Cargo.toml +++ b/relay-protocol/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Types and traits for building JSON-based protocols and schemas" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-quotas/Cargo.toml b/relay-quotas/Cargo.toml index 3a4ddc78e9..4ac6cdb070 100644 --- a/relay-quotas/Cargo.toml +++ b/relay-quotas/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Sentry quotas and rate limiting" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-redis/Cargo.toml b/relay-redis/Cargo.toml index 2694e078a2..5dd94c19b7 100644 --- a/relay-redis/Cargo.toml +++ b/relay-redis/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Pooled Redis and Redis cluster abstraction for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-redis/src/real.rs b/relay-redis/src/real.rs index e535c43125..4f6e3cfa8c 100644 --- a/relay-redis/src/real.rs +++ b/relay-redis/src/real.rs @@ -1,10 +1,10 @@ -use std::error::Error; -use std::fmt; -use std::time::Duration; - use r2d2::{Builder, ManageConnection, Pool, PooledConnection}; pub use redis; use redis::ConnectionLike; +use std::error::Error; +use std::thread::Scope; +use std::time::Duration; +use std::{fmt, thread}; use thiserror::Error; use crate::config::RedisConfigOptions; @@ -25,6 +25,30 @@ pub enum RedisError { Redis(#[source] redis::RedisError), } +fn log_secondary_redis_error(result: redis::RedisResult) { + if let Err(error) = result { + relay_log::error!( + error = &error as &dyn Error, + "sending cmds to the secondary Redis instance failed", + ); + } +} + +fn spawn_secondary_thread<'scope, 'env: 'scope, T>( + scope: &'scope Scope<'scope, 'env>, + block: impl FnOnce() -> redis::RedisResult + Send + 'scope, +) { + let result = thread::Builder::new().spawn_scoped(scope, move || { + log_secondary_redis_error(block()); + }); + if let Err(error) = result { + relay_log::error!( + error = &error as &dyn Error, + "spawning the thread for the secondary Redis connection failed", + ); + } +} + enum ConnectionInner<'a> { Cluster(&'a mut redis::cluster::ClusterConnection), MultiWrite { @@ -37,24 +61,17 @@ enum ConnectionInner<'a> { impl ConnectionLike for ConnectionInner<'_> { fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult { match self { - ConnectionInner::Cluster(ref mut con) => con.req_packed_command(cmd), + ConnectionInner::Cluster(con) => con.req_packed_command(cmd), + ConnectionInner::Single(con) => con.req_packed_command(cmd), ConnectionInner::MultiWrite { - primary: primary_connection, - secondaries: secondary_connections, - } => { - let primary_result = primary_connection.req_packed_command(cmd); - for secondary_connection in secondary_connections.iter_mut() { - if let Err(error) = secondary_connection.req_packed_command(cmd) { - relay_log::error!( - error = &error as &dyn Error, - "sending cmd to the secondary Redis instance failed", - ); - } + primary, + secondaries, + } => thread::scope(|s| { + for connection in secondaries { + spawn_secondary_thread(s, || connection.req_packed_command(cmd)) } - - primary_result - } - ConnectionInner::Single(ref mut con) => con.req_packed_command(cmd), + primary.req_packed_command(cmd) + }), } } @@ -65,58 +82,50 @@ impl ConnectionLike for ConnectionInner<'_> { count: usize, ) -> redis::RedisResult> { match self { - ConnectionInner::Cluster(ref mut con) => con.req_packed_commands(cmd, offset, count), + ConnectionInner::Cluster(con) => con.req_packed_commands(cmd, offset, count), + ConnectionInner::Single(con) => con.req_packed_commands(cmd, offset, count), ConnectionInner::MultiWrite { - primary: primary_connection, - secondaries: secondary_connections, - } => { - let primary_result = primary_connection.req_packed_commands(cmd, offset, count); - for secondary_connection in secondary_connections.iter_mut() { - if let Err(error) = secondary_connection.req_packed_commands(cmd, offset, count) - { - relay_log::error!( - error = &error as &dyn Error, - "sending cmds to the secondary Redis instance failed", - ); - } + primary, + secondaries, + } => thread::scope(|s| { + for connection in secondaries { + spawn_secondary_thread(s, || connection.req_packed_commands(cmd, offset, count)) } - - primary_result - } - ConnectionInner::Single(ref mut con) => con.req_packed_commands(cmd, offset, count), + primary.req_packed_commands(cmd, offset, count) + }), } } fn get_db(&self) -> i64 { match self { - ConnectionInner::Cluster(ref con) => con.get_db(), + ConnectionInner::Cluster(con) => con.get_db(), ConnectionInner::MultiWrite { primary: primary_connection, .. } => primary_connection.get_db(), - ConnectionInner::Single(ref con) => con.get_db(), + ConnectionInner::Single(con) => con.get_db(), } } fn check_connection(&mut self) -> bool { match self { - ConnectionInner::Cluster(ref mut con) => con.check_connection(), + ConnectionInner::Cluster(con) => con.check_connection(), ConnectionInner::MultiWrite { primary: primary_connection, .. } => primary_connection.check_connection(), - ConnectionInner::Single(ref mut con) => con.check_connection(), + ConnectionInner::Single(con) => con.check_connection(), } } fn is_open(&self) -> bool { match self { - ConnectionInner::Cluster(ref con) => con.is_open(), + ConnectionInner::Cluster(con) => con.is_open(), ConnectionInner::MultiWrite { primary: primary_connection, .. } => primary_connection.is_open(), - ConnectionInner::Single(ref con) => con.is_open(), + ConnectionInner::Single(con) => con.is_open(), } } } @@ -185,7 +194,7 @@ impl PooledClient { /// Recursively computes the [`ConnectionInner`] from a [`PooledClient`]. fn connection_inner(&mut self) -> Result, RedisError> { let inner = match self { - PooledClient::Cluster(ref mut connection, opts) => { + PooledClient::Cluster(connection, opts) => { connection .set_read_timeout(Some(Duration::from_secs(opts.read_timeout))) .map_err(RedisError::Redis)?; @@ -211,7 +220,7 @@ impl PooledClient { secondaries: secondary_connections, } } - PooledClient::Single(ref mut connection, opts) => { + PooledClient::Single(connection, opts) => { connection .set_read_timeout(Some(Duration::from_secs(opts.read_timeout))) .map_err(RedisError::Redis)?; @@ -286,7 +295,9 @@ impl RedisPool { /// Creates a [`RedisPool`] in single-node configuration. pub fn single(server: &str, opts: RedisConfigOptions) -> Result { - let pool = Self::client_pool(server, &opts)?; + let pool = Self::base_pool_builder(&opts) + .build(redis::Client::open(server).map_err(RedisError::Redis)?) + .map_err(RedisError::Pool)?; Ok(RedisPool::Single(pool, opts)) } @@ -294,7 +305,7 @@ impl RedisPool { /// Returns a pooled connection to a client. pub fn client(&self) -> Result { let pool = match self { - RedisPool::Cluster(ref pool, opts) => PooledClient::Cluster( + RedisPool::Cluster(pool, opts) => PooledClient::Cluster( Box::new(pool.get().map_err(RedisError::Pool)?), opts.clone(), ), @@ -314,7 +325,7 @@ impl RedisPool { secondaries: secondary_clients, } } - RedisPool::Single(ref pool, opts) => PooledClient::Single( + RedisPool::Single(pool, opts) => PooledClient::Single( Box::new(pool.get().map_err(RedisError::Pool)?), opts.clone(), ), @@ -332,16 +343,6 @@ impl RedisPool { } } - /// Returns a [`Pool`] with a [`redis::Client`]. - fn client_pool( - server: &str, - opts: &RedisConfigOptions, - ) -> Result, RedisError> { - Self::base_pool_builder(opts) - .build(redis::Client::open(server).map_err(RedisError::Redis)?) - .map_err(RedisError::Pool) - } - /// Returns the base builder for the pool with the options applied. fn base_pool_builder(opts: &RedisConfigOptions) -> Builder { Pool::builder() diff --git a/relay-replays/Cargo.toml b/relay-replays/Cargo.toml index 3e727ccbb6..2dc8ab0b81 100644 --- a/relay-replays/Cargo.toml +++ b/relay-replays/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Session replay functionality for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-sampling/Cargo.toml b/relay-sampling/Cargo.toml index 7c7c949cc2..a366b3e933 100644 --- a/relay-sampling/Cargo.toml +++ b/relay-sampling/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Dynamic sampling functionality for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-server/Cargo.toml b/relay-server/Cargo.toml index e817a203d7..dbec10dc95 100644 --- a/relay-server/Cargo.toml +++ b/relay-server/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Endpoints and services for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" build = "build.rs" license-file = "../LICENSE.md" diff --git a/relay-server/src/endpoints/batch_metrics.rs b/relay-server/src/endpoints/batch_metrics.rs index 77abf88833..d068e59fde 100644 --- a/relay-server/src/endpoints/batch_metrics.rs +++ b/relay-server/src/endpoints/batch_metrics.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use crate::extractors::{SignedBytes, StartTime}; use crate::service::ServiceState; use crate::services::processor::ProcessBatchedMetrics; -use crate::services::project_cache::BucketSource; +use crate::services::projects::cache::BucketSource; #[derive(Debug, Serialize, Deserialize)] struct SendMetricsResponse {} diff --git a/relay-server/src/endpoints/common.rs b/relay-server/src/endpoints/common.rs index 9d5e9c50fc..5a124b84b7 100644 --- a/relay-server/src/endpoints/common.rs +++ b/relay-server/src/endpoints/common.rs @@ -13,7 +13,7 @@ use crate::service::ServiceState; use crate::services::buffer::EnvelopeBuffer; use crate::services::outcome::{DiscardReason, Outcome}; use crate::services::processor::{MetricData, ProcessMetricMeta, ProcessingGroup}; -use crate::services::project_cache::{CheckEnvelope, ProcessMetrics, ValidateEnvelope}; +use crate::services::projects::cache::{CheckEnvelope, ProcessMetrics, ValidateEnvelope}; use crate::statsd::{RelayCounters, RelayHistograms}; use crate::utils::{self, ApiErrorResponse, FormDataIter, ManagedEnvelope}; diff --git a/relay-server/src/endpoints/project_configs.rs b/relay-server/src/endpoints/project_configs.rs index 2ede7027ec..391687a31f 100644 --- a/relay-server/src/endpoints/project_configs.rs +++ b/relay-server/src/endpoints/project_configs.rs @@ -16,8 +16,10 @@ use crate::endpoints::forward; use crate::extractors::SignedJson; use crate::service::ServiceState; use crate::services::global_config::{self, StatusResponse}; -use crate::services::project::{LimitedParsedProjectState, ParsedProjectState, ProjectState}; -use crate::services::project_cache::{GetCachedProjectState, GetProjectState}; +use crate::services::projects::cache::{GetCachedProjectState, GetProjectState}; +use crate::services::projects::project::{ + LimitedParsedProjectState, ParsedProjectState, ProjectState, +}; /// V2 version of this endpoint. /// @@ -89,7 +91,7 @@ struct GetProjectStatesResponseWrapper { /// Request payload of the project config endpoint. /// -/// This is a replica of [`GetProjectStates`](crate::services::project_upstream::GetProjectStates) +/// This is a replica of [`GetProjectStates`](crate::services::projects::source::upstream::GetProjectStates) /// which allows skipping invalid project keys. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] diff --git a/relay-server/src/service.rs b/relay-server/src/service.rs index 69b20af3f4..d7e954fe86 100644 --- a/relay-server/src/service.rs +++ b/relay-server/src/service.rs @@ -12,7 +12,7 @@ use crate::services::metrics::{Aggregator, RouterService}; use crate::services::outcome::{OutcomeProducer, OutcomeProducerService, TrackOutcome}; use crate::services::outcome_aggregator::OutcomeAggregator; use crate::services::processor::{self, EnvelopeProcessor, EnvelopeProcessorService}; -use crate::services::project_cache::{ProjectCache, ProjectCacheService, Services}; +use crate::services::projects::cache::{ProjectCache, ProjectCacheService, Services}; use crate::services::relays::{RelayCache, RelayCacheService}; use crate::services::stats::RelayStats; #[cfg(feature = "processing")] diff --git a/relay-server/src/services/buffer/envelope_buffer/mod.rs b/relay-server/src/services/buffer/envelope_buffer/mod.rs index 0ff0bc2b7e..02871016c4 100644 --- a/relay-server/src/services/buffer/envelope_buffer/mod.rs +++ b/relay-server/src/services/buffer/envelope_buffer/mod.rs @@ -14,6 +14,7 @@ use relay_config::Config; use tokio::time::{timeout, Instant}; use crate::envelope::Envelope; +use crate::envelope::Item; use crate::services::buffer::common::ProjectKeyPair; use crate::services::buffer::envelope_stack::sqlite::SqliteEnvelopeStackError; use crate::services::buffer::envelope_stack::EnvelopeStack; @@ -78,6 +79,11 @@ impl PolymorphicEnvelopeBuffer { /// Adds an envelope to the buffer. pub async fn push(&mut self, envelope: Box) -> Result<(), EnvelopeBufferError> { + relay_statsd::metric!( + histogram(RelayHistograms::BufferEnvelopeBodySize) = + envelope.items().map(Item::len).sum::() as u64 + ); + relay_statsd::metric!(timer(RelayTimers::BufferPush), { match self { Self::Sqlite(buffer) => buffer.push(envelope).await, diff --git a/relay-server/src/services/buffer/envelope_stack/sqlite.rs b/relay-server/src/services/buffer/envelope_stack/sqlite.rs index 703569ddab..97accdf3cb 100644 --- a/relay-server/src/services/buffer/envelope_stack/sqlite.rs +++ b/relay-server/src/services/buffer/envelope_stack/sqlite.rs @@ -97,7 +97,9 @@ impl SqliteEnvelopeStack { // We convert envelopes into a format which simplifies insertion in the store. If an // envelope can't be serialized, we will not insert it. - let envelopes = envelopes.iter().filter_map(|e| e.as_ref().try_into().ok()); + let envelopes = relay_statsd::metric!(timer(RelayTimers::BufferEnvelopesSerialization), { + envelopes.iter().filter_map(|e| e.as_ref().try_into().ok()) + }); // When early return here, we are acknowledging that the elements that we popped from // the buffer are lost in case of failure. We are doing this on purposes, since if we were diff --git a/relay-server/src/services/buffer/mod.rs b/relay-server/src/services/buffer/mod.rs index d732452763..3bcc0c5fea 100644 --- a/relay-server/src/services/buffer/mod.rs +++ b/relay-server/src/services/buffer/mod.rs @@ -21,7 +21,7 @@ use crate::services::outcome::DiscardReason; use crate::services::outcome::Outcome; use crate::services::outcome::TrackOutcome; use crate::services::processor::ProcessingGroup; -use crate::services::project_cache::{DequeuedEnvelope, ProjectCache, UpdateProject}; +use crate::services::projects::cache::{DequeuedEnvelope, ProjectCache, UpdateProject}; use crate::services::test_store::TestStore; use crate::statsd::{RelayCounters, RelayHistograms}; diff --git a/relay-server/src/services/mod.rs b/relay-server/src/services/mod.rs index be7b542b06..8c06c1d0af 100644 --- a/relay-server/src/services/mod.rs +++ b/relay-server/src/services/mod.rs @@ -7,7 +7,7 @@ //! and dispatches the graceful shutdown signal. Internally, it creates several other services //! comprising the service state: //! -//! - [`ProjectCache`](project_cache::ProjectCache): A cache that serves queries for project +//! - [`ProjectCache`](projects::cache::ProjectCache): A cache that serves queries for project //! configurations. Its requests are debounced and batched based on a configured interval (100ms //! by default). Also, missing projects are cached for some time. //! - [`EnvelopeProcessor`](processor::EnvelopeProcessor): A worker pool for CPU-intensive tasks. @@ -35,10 +35,7 @@ pub mod metrics; pub mod outcome; pub mod outcome_aggregator; pub mod processor; -pub mod project; -pub mod project_cache; -pub mod project_local; -pub mod project_upstream; +pub mod projects; pub mod relays; pub mod server; pub mod spooler; @@ -46,7 +43,5 @@ pub mod stats; pub mod test_store; pub mod upstream; -#[cfg(feature = "processing")] -pub mod project_redis; #[cfg(feature = "processing")] pub mod store; diff --git a/relay-server/src/services/processor.rs b/relay-server/src/services/processor.rs index 2a3248e909..67b70bee72 100644 --- a/relay-server/src/services/processor.rs +++ b/relay-server/src/services/processor.rs @@ -72,10 +72,10 @@ use crate::services::global_config::GlobalConfigHandle; use crate::services::metrics::{Aggregator, MergeBuckets}; use crate::services::outcome::{DiscardReason, Outcome, TrackOutcome}; use crate::services::processor::event::FiltersStatus; -use crate::services::project::{ProjectInfo, ProjectState}; -use crate::services::project_cache::{ +use crate::services::projects::cache::{ AddMetricMeta, BucketSource, ProcessMetrics, ProjectCache, UpdateRateLimits, }; +use crate::services::projects::project::{ProjectInfo, ProjectState}; use crate::services::test_store::{Capture, TestStore}; use crate::services::upstream::{ SendRequest, UpstreamRelay, UpstreamRequest, UpstreamRequestError, @@ -1710,10 +1710,6 @@ impl EnvelopeProcessorService { let profile_id = profile::filter(state); profile::transfer_id(state, profile_id); - if_processing!(self.inner.config, { - attachment::create_placeholders(state); - }); - event::finalize(state, &self.inner.config)?; self.normalize_event(state)?; diff --git a/relay-server/src/services/processor/attachment.rs b/relay-server/src/services/processor/attachment.rs index 21814def5e..c45f3fbbc5 100644 --- a/relay-server/src/services/processor/attachment.rs +++ b/relay-server/src/services/processor/attachment.rs @@ -12,7 +12,7 @@ use crate::statsd::RelayTimers; #[cfg(feature = "processing")] use { - crate::services::processor::EventProcessing, crate::utils, relay_event_schema::protocol::Event, + crate::services::processor::ErrorGroup, crate::utils, relay_event_schema::protocol::Event, relay_protocol::Annotated, }; @@ -23,7 +23,7 @@ use { /// /// If the event payload was empty before, it is created. #[cfg(feature = "processing")] -pub fn create_placeholders(state: &mut ProcessEnvelopeState) { +pub fn create_placeholders(state: &mut ProcessEnvelopeState) { let envelope = state.managed_envelope.envelope(); let minidump_attachment = envelope.get_item_by(|item| item.attachment_type() == Some(&AttachmentType::Minidump)); diff --git a/relay-server/src/services/processor/dynamic_sampling.rs b/relay-server/src/services/processor/dynamic_sampling.rs index 1267015444..b75a4162af 100644 --- a/relay-server/src/services/processor/dynamic_sampling.rs +++ b/relay-server/src/services/processor/dynamic_sampling.rs @@ -252,7 +252,7 @@ mod tests { use crate::services::processor::{ ProcessEnvelope, ProcessingExtractedMetrics, ProcessingGroup, SpanGroup, }; - use crate::services::project::ProjectInfo; + use crate::services::projects::project::ProjectInfo; use crate::testutils::{ self, create_test_processor, new_envelope, state_with_rule_and_condition, }; diff --git a/relay-server/src/services/processor/metrics.rs b/relay-server/src/services/processor/metrics.rs index 8c6ac4fcc4..0d28285e77 100644 --- a/relay-server/src/services/processor/metrics.rs +++ b/relay-server/src/services/processor/metrics.rs @@ -5,8 +5,8 @@ use relay_quotas::Scoping; use crate::metrics::MetricOutcomes; use crate::services::outcome::Outcome; -use crate::services::project::ProjectInfo; -use crate::services::project_cache::BucketSource; +use crate::services::projects::cache::BucketSource; +use crate::services::projects::project::ProjectInfo; /// Checks if the namespace of the passed bucket is valid. /// diff --git a/relay-server/src/services/processor/profile.rs b/relay-server/src/services/processor/profile.rs index be93112a02..6f0565afca 100644 --- a/relay-server/src/services/processor/profile.rs +++ b/relay-server/src/services/processor/profile.rs @@ -152,7 +152,7 @@ mod tests { use crate::envelope::Envelope; use crate::extractors::RequestMeta; use crate::services::processor::{ProcessEnvelope, ProcessingGroup}; - use crate::services::project::ProjectInfo; + use crate::services::projects::project::ProjectInfo; use crate::testutils::create_test_processor; use crate::utils::ManagedEnvelope; diff --git a/relay-server/src/services/processor/report.rs b/relay-server/src/services/processor/report.rs index 002a0753de..d0c101755b 100644 --- a/relay-server/src/services/processor/report.rs +++ b/relay-server/src/services/processor/report.rs @@ -274,7 +274,7 @@ mod tests { use crate::extractors::RequestMeta; use crate::services::outcome::RuleCategory; use crate::services::processor::{ProcessEnvelope, ProcessingGroup}; - use crate::services::project::ProjectInfo; + use crate::services::projects::project::ProjectInfo; use crate::testutils::{self, create_test_processor}; use crate::utils::ManagedEnvelope; diff --git a/relay-server/src/services/processor/span/processing.rs b/relay-server/src/services/processor/span/processing.rs index 4f905e6fe4..cc56ee1d8d 100644 --- a/relay-server/src/services/processor/span/processing.rs +++ b/relay-server/src/services/processor/span/processing.rs @@ -786,7 +786,7 @@ mod tests { use crate::envelope::Envelope; use crate::services::processor::{ProcessingExtractedMetrics, ProcessingGroup}; - use crate::services::project::ProjectInfo; + use crate::services::projects::project::ProjectInfo; use crate::utils::ManagedEnvelope; use super::*; diff --git a/relay-server/src/services/project_cache.rs b/relay-server/src/services/projects/cache.rs similarity index 89% rename from relay-server/src/services/project_cache.rs rename to relay-server/src/services/projects/cache.rs index 815d265226..dc29f20bc2 100644 --- a/relay-server/src/services/project_cache.rs +++ b/relay-server/src/services/projects/cache.rs @@ -9,31 +9,23 @@ use crate::services::global_config; use crate::services::processor::{ EncodeMetrics, EnvelopeProcessor, MetricData, ProcessEnvelope, ProcessingGroup, ProjectMetrics, }; -use crate::services::project::state::UpstreamProjectState; use crate::Envelope; use chrono::{DateTime, Utc}; use hashbrown::HashSet; use relay_base_schema::project::ProjectKey; -#[cfg(feature = "processing")] -use relay_config::RedisConfigRef; -use relay_config::{Config, RelayMode}; +use relay_config::Config; use relay_metrics::{Bucket, MetricMeta}; use relay_quotas::RateLimits; use relay_redis::RedisPool; use relay_statsd::metric; use relay_system::{Addr, FromMessage, Interface, Sender, Service}; -#[cfg(feature = "processing")] -use tokio::sync::Semaphore; use tokio::sync::{mpsc, watch}; use tokio::time::Instant; use crate::services::metrics::{Aggregator, FlushBuckets}; use crate::services::outcome::{DiscardReason, Outcome, TrackOutcome}; -use crate::services::project::{Project, ProjectFetchState, ProjectSender, ProjectState}; -use crate::services::project_local::{LocalProjectSource, LocalProjectSourceService}; -#[cfg(feature = "processing")] -use crate::services::project_redis::RedisProjectSource; -use crate::services::project_upstream::{UpstreamProjectSource, UpstreamProjectSourceService}; +use crate::services::projects::project::{Project, ProjectFetchState, ProjectSender, ProjectState}; +use crate::services::projects::source::ProjectSource; use crate::services::spooler::{ self, Buffer, BufferService, DequeueMany, Enqueue, QueueKey, RemoveMany, RestoreIndex, UnspooledEnvelope, BATCH_KEY_COUNT, @@ -44,10 +36,6 @@ use crate::services::upstream::UpstreamRelay; use crate::statsd::{RelayCounters, RelayGauges, RelayHistograms, RelayTimers}; use crate::utils::{GarbageDisposal, ManagedEnvelope, MemoryChecker, RetryBackoff, SleepHandle}; -/// Default value of maximum connections to Redis. This value was arbitrarily determined. -#[cfg(feature = "processing")] -const DEFAULT_REDIS_MAX_CONNECTIONS: u32 = 10; - /// Requests a refresh of a project state from one of the available sources. /// /// The project state is resolved in the following precedence: @@ -421,146 +409,6 @@ impl FromMessage for ProjectCache { } } -/// Helper type that contains all configured sources for project cache fetching. -/// -/// See [`RequestUpdate`] for a description on how project states are fetched. -#[derive(Clone, Debug)] -struct ProjectSource { - config: Arc, - local_source: Addr, - upstream_source: Addr, - #[cfg(feature = "processing")] - redis_source: Option, - #[cfg(feature = "processing")] - redis_semaphore: Arc, -} - -impl ProjectSource { - /// Starts all project source services in the current runtime. - pub fn start( - config: Arc, - upstream_relay: Addr, - _redis: Option, - ) -> Self { - let local_source = LocalProjectSourceService::new(config.clone()).start(); - let upstream_source = - UpstreamProjectSourceService::new(config.clone(), upstream_relay).start(); - - #[cfg(feature = "processing")] - let redis_max_connections = config - .redis() - .map(|configs| { - let config = match configs { - relay_config::RedisPoolConfigs::Unified(config) => config, - relay_config::RedisPoolConfigs::Individual { - project_configs: config, - .. - } => config, - }; - Self::compute_max_connections(config).unwrap_or(DEFAULT_REDIS_MAX_CONNECTIONS) - }) - .unwrap_or(DEFAULT_REDIS_MAX_CONNECTIONS); - #[cfg(feature = "processing")] - let redis_source = _redis.map(|pool| RedisProjectSource::new(config.clone(), pool)); - - Self { - config, - local_source, - upstream_source, - #[cfg(feature = "processing")] - redis_source, - #[cfg(feature = "processing")] - redis_semaphore: Arc::new(Semaphore::new(redis_max_connections.try_into().unwrap())), - } - } - - #[cfg(feature = "processing")] - fn compute_max_connections(config: RedisConfigRef) -> Option { - match config { - RedisConfigRef::Cluster { options, .. } => Some(options.max_connections), - RedisConfigRef::MultiWrite { configs } => configs - .into_iter() - .filter_map(|c| Self::compute_max_connections(c)) - .max(), - RedisConfigRef::Single { options, .. } => Some(options.max_connections), - } - } - - async fn fetch( - self, - project_key: ProjectKey, - no_cache: bool, - cached_state: ProjectFetchState, - ) -> Result { - let state_opt = self - .local_source - .send(FetchOptionalProjectState { project_key }) - .await - .map_err(|_| ())?; - - if let Some(state) = state_opt { - return Ok(ProjectFetchState::new(state)); - } - - match self.config.relay_mode() { - RelayMode::Proxy => return Ok(ProjectFetchState::allowed()), - RelayMode::Static => return Ok(ProjectFetchState::disabled()), - RelayMode::Capture => return Ok(ProjectFetchState::allowed()), - RelayMode::Managed => (), // Proceed with loading the config from redis or upstream - } - - let current_revision = cached_state.revision().map(String::from); - #[cfg(feature = "processing")] - if let Some(redis_source) = self.redis_source { - let current_revision = current_revision.clone(); - - let redis_permit = self.redis_semaphore.acquire().await.map_err(|_| ())?; - let state_fetch_result = tokio::task::spawn_blocking(move || { - redis_source.get_config_if_changed(project_key, current_revision.as_deref()) - }) - .await - .map_err(|_| ())?; - drop(redis_permit); - - match state_fetch_result { - // New state fetched from Redis, possibly pending. - Ok(UpstreamProjectState::New(state)) => { - let state = state.sanitized(); - if !state.is_pending() { - return Ok(ProjectFetchState::new(state)); - } - } - // Redis reported that we're holding an up-to-date version of the state already, - // refresh the state and return the old cached state again. - Ok(UpstreamProjectState::NotModified) => { - return Ok(ProjectFetchState::refresh(cached_state)) - } - Err(error) => { - relay_log::error!( - error = &error as &dyn Error, - "failed to fetch project from Redis", - ); - } - }; - }; - - let state = self - .upstream_source - .send(FetchProjectState { - project_key, - current_revision, - no_cache, - }) - .await - .map_err(|_| ())?; - - match state { - UpstreamProjectState::New(state) => Ok(ProjectFetchState::new(state.sanitized())), - UpstreamProjectState::NotModified => Ok(ProjectFetchState::refresh(cached_state)), - } - } -} - /// Updates the cache with new project state information. struct UpdateProjectState { /// The public key to fetch the project by. @@ -669,7 +517,7 @@ impl ProjectCacheBroker { /// Ideally, we would use `check_expiry` to determine expiry here. /// However, for eviction, we want to add an additional delay, such that we do not delete /// a project that has expired recently and for which a fetch is already underway in - /// [`super::project_upstream`]. + /// [`super::source::upstream`]. fn evict_stale_project_caches(&mut self) { let eviction_start = Instant::now(); let delta = 2 * self.config.project_cache_expiry() + self.config.project_grace_period(); @@ -775,7 +623,16 @@ impl ProjectCacheBroker { let state = source .fetch(project_key, no_cache, cached_state) .await - .unwrap_or_else(|()| ProjectFetchState::disabled()); + .unwrap_or_else(|e| { + relay_log::error!( + error = &e as &dyn Error, + tags.project_key = %project_key, + "Failed to fetch project from source" + ); + // TODO: change this to ProjectFetchState::pending() once we consider it safe to do so. + // see https://github.com/getsentry/relay/pull/4140. + ProjectFetchState::disabled() + }); let message = UpdateProjectState { project_key, @@ -1503,36 +1360,6 @@ impl Service for ProjectCacheService { } } -#[derive(Clone, Debug)] -pub struct FetchProjectState { - /// The public key to fetch the project by. - pub project_key: ProjectKey, - - /// Currently cached revision if available. - /// - /// The upstream is allowed to omit full project configs - /// for requests for which the requester already has the most - /// recent revision. - /// - /// Settings this to `None` will essentially always re-fetch - /// the project config. - pub current_revision: Option, - - /// If true, all caches should be skipped and a fresh state should be computed. - pub no_cache: bool, -} - -#[derive(Clone, Debug)] -pub struct FetchOptionalProjectState { - project_key: ProjectKey, -} - -impl FetchOptionalProjectState { - pub fn project_key(&self) -> ProjectKey { - self.project_key - } -} - /// Sum type for all objects which need to be discareded through the [`GarbageDisposal`]. #[derive(Debug)] #[allow(dead_code)] // Fields are never read, only used for discarding/dropping data. diff --git a/relay-server/src/services/projects/mod.rs b/relay-server/src/services/projects/mod.rs new file mode 100644 index 0000000000..bc8f253eee --- /dev/null +++ b/relay-server/src/services/projects/mod.rs @@ -0,0 +1,3 @@ +pub mod cache; +pub mod project; +pub mod source; diff --git a/relay-server/src/services/project.rs b/relay-server/src/services/projects/project/mod.rs similarity index 98% rename from relay-server/src/services/project.rs rename to relay-server/src/services/projects/project/mod.rs index 7e5ab2bb97..a553ebd89b 100644 --- a/relay-server/src/services/project.rs +++ b/relay-server/src/services/projects/project/mod.rs @@ -16,8 +16,7 @@ use crate::envelope::ItemType; use crate::services::metrics::{Aggregator, MergeBuckets}; use crate::services::outcome::{DiscardReason, Outcome}; use crate::services::processor::{EncodeMetricMeta, EnvelopeProcessor, ProcessProjectMetrics}; -use crate::services::project::state::ExpiryState; -use crate::services::project_cache::{ +use crate::services::projects::cache::{ CheckedEnvelope, ProcessMetrics, ProjectCache, RequestUpdate, }; use crate::utils::{Enforcement, SeqCount}; @@ -28,7 +27,8 @@ use crate::utils::{CheckLimits, EnvelopeLimiter, ManagedEnvelope, RetryBackoff}; pub mod state; pub use state::{ - LimitedParsedProjectState, ParsedProjectState, ProjectFetchState, ProjectInfo, ProjectState, + ExpiryState, LimitedParsedProjectState, ParsedProjectState, ProjectFetchState, ProjectInfo, + ProjectState, }; /// Sender type for messages that respond with project states. @@ -433,7 +433,7 @@ impl Project { /// `no_cache` should be passed from the requesting call. Updates with `no_cache` will always /// take precedence. /// - /// [`ValidateEnvelope`]: crate::services::project_cache::ValidateEnvelope + /// [`ValidateEnvelope`]: crate::services::projects::cache::ValidateEnvelope pub fn update_state( &mut self, project_cache: &Addr, @@ -515,7 +515,7 @@ impl Project { /// Runs the checks on incoming envelopes. /// - /// See, [`crate::services::project_cache::CheckEnvelope`] for more information + /// See, [`crate::services::projects::cache::CheckEnvelope`] for more information /// /// * checks the rate limits /// * validates the envelope meta in `check_request` - determines whether the given request diff --git a/relay-server/src/services/project/state/fetch_state.rs b/relay-server/src/services/projects/project/state/fetch_state.rs similarity index 95% rename from relay-server/src/services/project/state/fetch_state.rs rename to relay-server/src/services/projects/project/state/fetch_state.rs index abc4a9f79e..020150f303 100644 --- a/relay-server/src/services/project/state/fetch_state.rs +++ b/relay-server/src/services/projects/project/state/fetch_state.rs @@ -5,8 +5,8 @@ use tokio::time::Instant; use relay_config::Config; use relay_dynamic_config::ProjectConfig; -use crate::services::project::state::info::ProjectInfo; -use crate::services::project::ProjectState; +use crate::services::projects::project::state::info::ProjectInfo; +use crate::services::projects::project::ProjectState; /// Hides a cached project state and only exposes it if it has not expired. #[derive(Clone, Debug)] @@ -65,7 +65,7 @@ impl ProjectFetchState { /// Create a config that immediately counts as expired. /// - /// This is what [`Project`](crate::services::project::Project) initializes itself with. + /// This is what [`Project`](crate::services::projects::project::Project) initializes itself with. pub fn expired() -> Self { Self { // Make sure the state immediately qualifies as expired: diff --git a/relay-server/src/services/project/state/info.rs b/relay-server/src/services/projects/project/state/info.rs similarity index 98% rename from relay-server/src/services/project/state/info.rs rename to relay-server/src/services/projects/project/state/info.rs index 7b8b9e3975..eab9b8fb62 100644 --- a/relay-server/src/services/project/state/info.rs +++ b/relay-server/src/services/projects/project/state/info.rs @@ -16,7 +16,7 @@ use url::Url; use crate::envelope::Envelope; use crate::extractors::RequestMeta; use crate::services::outcome::DiscardReason; -use crate::services::project::PublicKeyConfig; +use crate::services::projects::project::PublicKeyConfig; /// Information about an enabled project. /// @@ -189,7 +189,7 @@ impl ProjectInfo { /// scoping. /// /// To get the own scoping of this ProjectKey without amending request information, use - /// [`Project::scoping`](crate::services::project::Project::scoping) instead. + /// [`Project::scoping`](crate::services::projects::project::Project::scoping) instead. pub fn scope_request(&self, meta: &RequestMeta) -> Scoping { let mut scoping = meta.get_partial_scoping(); diff --git a/relay-server/src/services/project/state.rs b/relay-server/src/services/projects/project/state/mod.rs similarity index 97% rename from relay-server/src/services/project/state.rs rename to relay-server/src/services/projects/project/state/mod.rs index 70616d4a6e..868ad5ed4a 100644 --- a/relay-server/src/services/project/state.rs +++ b/relay-server/src/services/projects/project/state/mod.rs @@ -22,7 +22,7 @@ pub enum ProjectState { Disabled, /// A project to which one of the following conditions apply: /// - The project has not yet been fetched. - /// - The upstream returned "pending" for this project (see [`crate::services::project_upstream`]). + /// - The upstream returned "pending" for this project (see [`crate::services::projects::source::upstream`]). /// - The upstream returned an unparsable project so we have to try again. /// - The project has expired and must be treated as "has not been fetched". Pending, @@ -95,7 +95,7 @@ pub struct ParsedProjectState { /// Project info. /// /// This contains no information when `disabled` is `true`, except for - /// public keys in static project configs (see [`crate::services::project_local`]). + /// public keys in static project configs (see [`crate::services::projects::source::local`]). #[serde(flatten)] pub info: ProjectInfo, } @@ -109,7 +109,7 @@ pub struct LimitedParsedProjectState { /// Limited project info for external Relays. /// /// This contains no information when `disabled` is `true`, except for - /// public keys in static project configs (see [`crate::services::project_local`]). + /// public keys in static project configs (see [`crate::services::projects::source::local`]). #[serde(with = "LimitedProjectInfo")] #[serde(flatten)] pub info: ProjectInfo, diff --git a/relay-server/src/services/project_local.rs b/relay-server/src/services/projects/source/local.rs similarity index 97% rename from relay-server/src/services/project_local.rs rename to relay-server/src/services/projects/source/local.rs index 745203780a..9bffd851ef 100644 --- a/relay-server/src/services/project_local.rs +++ b/relay-server/src/services/projects/source/local.rs @@ -9,8 +9,8 @@ use relay_system::{AsyncResponse, FromMessage, Interface, Receiver, Sender, Serv use tokio::sync::mpsc; use tokio::time::Instant; -use crate::services::project::{ParsedProjectState, ProjectState}; -use crate::services::project_cache::FetchOptionalProjectState; +use crate::services::projects::project::{ParsedProjectState, ProjectState}; +use crate::services::projects::source::FetchOptionalProjectState; /// Service interface of the local project source. #[derive(Debug)] @@ -196,7 +196,7 @@ mod tests { use std::str::FromStr; use super::*; - use crate::services::project::{ProjectInfo, PublicKeyConfig}; + use crate::services::projects::project::{ProjectInfo, PublicKeyConfig}; /// Tests that we can follow the symlinks and read the project file properly. #[tokio::test] diff --git a/relay-server/src/services/projects/source/mod.rs b/relay-server/src/services/projects/source/mod.rs new file mode 100644 index 0000000000..5b208ef3ae --- /dev/null +++ b/relay-server/src/services/projects/source/mod.rs @@ -0,0 +1,210 @@ +use std::convert::Infallible; +use std::sync::Arc; + +use relay_base_schema::project::ProjectKey; +#[cfg(feature = "processing")] +use relay_config::RedisConfigRef; +use relay_config::{Config, RelayMode}; +use relay_redis::RedisPool; +use relay_system::{Addr, Service as _}; +#[cfg(feature = "processing")] +use tokio::sync::Semaphore; + +pub mod local; +#[cfg(feature = "processing")] +pub mod redis; +pub mod upstream; + +use crate::services::projects::project::state::UpstreamProjectState; +use crate::services::projects::project::ProjectFetchState; +use crate::services::upstream::UpstreamRelay; + +use self::local::{LocalProjectSource, LocalProjectSourceService}; +#[cfg(feature = "processing")] +use self::redis::RedisProjectSource; +use self::upstream::{UpstreamProjectSource, UpstreamProjectSourceService}; + +/// Default value of maximum connections to Redis. This value was arbitrarily determined. +#[cfg(feature = "processing")] +const DEFAULT_REDIS_MAX_CONNECTIONS: u32 = 10; + +/// Helper type that contains all configured sources for project cache fetching. +#[derive(Clone, Debug)] +pub struct ProjectSource { + config: Arc, + local_source: Addr, + upstream_source: Addr, + #[cfg(feature = "processing")] + redis_source: Option, + #[cfg(feature = "processing")] + redis_semaphore: Arc, +} + +impl ProjectSource { + /// Starts all project source services in the current runtime. + pub fn start( + config: Arc, + upstream_relay: Addr, + _redis: Option, + ) -> Self { + let local_source = LocalProjectSourceService::new(config.clone()).start(); + let upstream_source = + UpstreamProjectSourceService::new(config.clone(), upstream_relay).start(); + + #[cfg(feature = "processing")] + let redis_max_connections = config + .redis() + .map(|configs| { + let config = match configs { + relay_config::RedisPoolConfigs::Unified(config) => config, + relay_config::RedisPoolConfigs::Individual { + project_configs: config, + .. + } => config, + }; + Self::compute_max_connections(config).unwrap_or(DEFAULT_REDIS_MAX_CONNECTIONS) + }) + .unwrap_or(DEFAULT_REDIS_MAX_CONNECTIONS); + #[cfg(feature = "processing")] + let redis_source = _redis.map(|pool| RedisProjectSource::new(config.clone(), pool)); + + Self { + config, + local_source, + upstream_source, + #[cfg(feature = "processing")] + redis_source, + #[cfg(feature = "processing")] + redis_semaphore: Arc::new(Semaphore::new(redis_max_connections.try_into().unwrap())), + } + } + + #[cfg(feature = "processing")] + fn compute_max_connections(config: RedisConfigRef) -> Option { + match config { + RedisConfigRef::Cluster { options, .. } => Some(options.max_connections), + RedisConfigRef::MultiWrite { configs } => configs + .into_iter() + .filter_map(|c| Self::compute_max_connections(c)) + .max(), + RedisConfigRef::Single { options, .. } => Some(options.max_connections), + } + } + + pub async fn fetch( + self, + project_key: ProjectKey, + no_cache: bool, + cached_state: ProjectFetchState, + ) -> Result { + let state_opt = self + .local_source + .send(FetchOptionalProjectState { project_key }) + .await?; + + if let Some(state) = state_opt { + return Ok(ProjectFetchState::new(state)); + } + + match self.config.relay_mode() { + RelayMode::Proxy => return Ok(ProjectFetchState::allowed()), + RelayMode::Static => return Ok(ProjectFetchState::disabled()), + RelayMode::Capture => return Ok(ProjectFetchState::allowed()), + RelayMode::Managed => (), // Proceed with loading the config from redis or upstream + } + + let current_revision = cached_state.revision().map(String::from); + #[cfg(feature = "processing")] + if let Some(redis_source) = self.redis_source { + let current_revision = current_revision.clone(); + + let redis_permit = self.redis_semaphore.acquire().await?; + let state_fetch_result = tokio::task::spawn_blocking(move || { + redis_source.get_config_if_changed(project_key, current_revision.as_deref()) + }) + .await?; + drop(redis_permit); + + match state_fetch_result { + // New state fetched from Redis, possibly pending. + Ok(UpstreamProjectState::New(state)) => { + let state = state.sanitized(); + if !state.is_pending() { + return Ok(ProjectFetchState::new(state)); + } + } + // Redis reported that we're holding an up-to-date version of the state already, + // refresh the state and return the old cached state again. + Ok(UpstreamProjectState::NotModified) => { + return Ok(ProjectFetchState::refresh(cached_state)) + } + Err(error) => { + relay_log::error!( + error = &error as &dyn std::error::Error, + "failed to fetch project from Redis", + ); + } + }; + }; + + let state = self + .upstream_source + .send(FetchProjectState { + project_key, + current_revision, + no_cache, + }) + .await?; + + match state { + UpstreamProjectState::New(state) => Ok(ProjectFetchState::new(state.sanitized())), + UpstreamProjectState::NotModified => Ok(ProjectFetchState::refresh(cached_state)), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ProjectSourceError { + #[error("redis permit error {0}")] + RedisPermit(#[from] tokio::sync::AcquireError), + #[error("redis join error {0}")] + RedisJoin(#[from] tokio::task::JoinError), + #[error("upstream error {0}")] + Upstream(#[from] relay_system::SendError), +} + +impl From for ProjectSourceError { + fn from(value: Infallible) -> Self { + match value {} + } +} + +#[derive(Clone, Debug)] +pub struct FetchProjectState { + /// The public key to fetch the project by. + pub project_key: ProjectKey, + + /// Currently cached revision if available. + /// + /// The upstream is allowed to omit full project configs + /// for requests for which the requester already has the most + /// recent revision. + /// + /// Settings this to `None` will essentially always re-fetch + /// the project config. + pub current_revision: Option, + + /// If true, all caches should be skipped and a fresh state should be computed. + pub no_cache: bool, +} + +#[derive(Clone, Debug)] +pub struct FetchOptionalProjectState { + project_key: ProjectKey, +} + +impl FetchOptionalProjectState { + pub fn project_key(&self) -> ProjectKey { + self.project_key + } +} diff --git a/relay-server/src/services/project_redis.rs b/relay-server/src/services/projects/source/redis.rs similarity index 97% rename from relay-server/src/services/project_redis.rs rename to relay-server/src/services/projects/source/redis.rs index 18e8dd0e53..e2a520f4e7 100644 --- a/relay-server/src/services/project_redis.rs +++ b/relay-server/src/services/projects/source/redis.rs @@ -5,8 +5,8 @@ use relay_config::Config; use relay_redis::{RedisError, RedisPool}; use relay_statsd::metric; -use crate::services::project::state::UpstreamProjectState; -use crate::services::project::{ParsedProjectState, ProjectState}; +use crate::services::projects::project::state::UpstreamProjectState; +use crate::services::projects::project::{ParsedProjectState, ProjectState}; use crate::statsd::{RelayCounters, RelayHistograms, RelayTimers}; #[derive(Debug, Clone)] diff --git a/relay-server/src/services/project_upstream.rs b/relay-server/src/services/projects/source/upstream.rs similarity index 99% rename from relay-server/src/services/project_upstream.rs rename to relay-server/src/services/projects/source/upstream.rs index 0b54cc17e6..bcbfd5a279 100644 --- a/relay-server/src/services/project_upstream.rs +++ b/relay-server/src/services/projects/source/upstream.rs @@ -18,10 +18,10 @@ use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::Instant; -use crate::services::project::state::UpstreamProjectState; -use crate::services::project::ParsedProjectState; -use crate::services::project::ProjectState; -use crate::services::project_cache::FetchProjectState; +use crate::services::projects::project::state::UpstreamProjectState; +use crate::services::projects::project::ParsedProjectState; +use crate::services::projects::project::ProjectState; +use crate::services::projects::source::FetchProjectState; use crate::services::upstream::{ Method, RequestPriority, SendQuery, UpstreamQuery, UpstreamRelay, UpstreamRequestError, }; diff --git a/relay-server/src/services/spooler/mod.rs b/relay-server/src/services/spooler/mod.rs index 36debfc9bf..54c2530550 100644 --- a/relay-server/src/services/spooler/mod.rs +++ b/relay-server/src/services/spooler/mod.rs @@ -13,7 +13,7 @@ //! be happening. //! //! The initial state is always [`InMemory`], and if the Relay can properly fetch all the -//! [`crate::services::project::ProjectState`] it continues to use the memory as temporary spool. +//! [`crate::services::projects::project::ProjectState`] it continues to use the memory as temporary spool. //! //! Keeping the envelopes in memory as long as we can, we ensure the fast unspool operations and //! fast processing times. @@ -55,7 +55,7 @@ use crate::envelope::{Envelope, EnvelopeError}; use crate::extractors::StartTime; use crate::services::outcome::TrackOutcome; use crate::services::processor::ProcessingGroup; -use crate::services::project_cache::{ProjectCache, RefreshIndexCache, UpdateSpoolIndex}; +use crate::services::projects::cache::{ProjectCache, RefreshIndexCache, UpdateSpoolIndex}; use crate::services::test_store::TestStore; use crate::statsd::{RelayCounters, RelayGauges, RelayHistograms, RelayTimers}; use crate::utils::{ManagedEnvelope, MemoryChecker}; diff --git a/relay-server/src/statsd.rs b/relay-server/src/statsd.rs index fbae91d779..e7f3a498db 100644 --- a/relay-server/src/statsd.rs +++ b/relay-server/src/statsd.rs @@ -181,6 +181,11 @@ pub enum RelayHistograms { /// Number of envelopes in the backpressure buffer between the envelope buffer /// and the project cache. BufferBackpressureEnvelopesCount, + /// The amount of bytes in the item payloads of an envelope pushed to the envelope buffer. + /// + /// This is not quite the same as the actual size of a serialized envelope, because it ignores + /// the envelope header and item headers. + BufferEnvelopeBodySize, /// The number of batches emitted per partition. BatchesPerPartition, /// The number of buckets in a batch emitted. @@ -309,6 +314,7 @@ impl HistogramMetric for RelayHistograms { RelayHistograms::BufferBackpressureEnvelopesCount => { "buffer.backpressure_envelopes_count" } + RelayHistograms::BufferEnvelopeBodySize => "buffer.envelope_body_size", RelayHistograms::ProjectStatePending => "project_state.pending", RelayHistograms::ProjectStateAttempts => "project_state.attempts", RelayHistograms::ProjectStateRequestBatchSize => "project_state.request.batch_size", @@ -538,6 +544,8 @@ pub enum RelayTimers { BufferPop, /// Timing in milliseconds for the time it takes for the buffer to drain its envelopes. BufferDrain, + /// Timing in milliseconds for the time it takes for the envelopes to be serialized. + BufferEnvelopesSerialization, } impl TimerMetric for RelayTimers { @@ -586,6 +594,7 @@ impl TimerMetric for RelayTimers { RelayTimers::BufferPeek => "buffer.peek.duration", RelayTimers::BufferPop => "buffer.pop.duration", RelayTimers::BufferDrain => "buffer.drain.duration", + RelayTimers::BufferEnvelopesSerialization => "buffer.envelopes_serialization", } } } diff --git a/relay-server/src/testutils.rs b/relay-server/src/testutils.rs index 47e8f988fd..9f4c255d35 100644 --- a/relay-server/src/testutils.rs +++ b/relay-server/src/testutils.rs @@ -20,7 +20,7 @@ use crate::service::create_redis_pools; use crate::services::global_config::GlobalConfigHandle; use crate::services::outcome::TrackOutcome; use crate::services::processor::{self, EnvelopeProcessorService}; -use crate::services::project::ProjectInfo; +use crate::services::projects::project::ProjectInfo; use crate::services::test_store::TestStore; use crate::utils::{ThreadPool, ThreadPoolBuilder}; diff --git a/relay-spans/Cargo.toml b/relay-spans/Cargo.toml index ed65eed8a7..807b2e9acf 100644 --- a/relay-spans/Cargo.toml +++ b/relay-spans/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Event normalization and processing" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE" publish = false diff --git a/relay-statsd/Cargo.toml b/relay-statsd/Cargo.toml index f5ee35cdab..1b74fe65d4 100644 --- a/relay-statsd/Cargo.toml +++ b/relay-statsd/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "High-level StatsD metric client for internal measurements" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-system/Cargo.toml b/relay-system/Cargo.toml index 9a33e90d21..a2d3bf9f06 100644 --- a/relay-system/Cargo.toml +++ b/relay-system/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Foundational system components for Relay's services" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-test/Cargo.toml b/relay-test/Cargo.toml index 274a5d7310..77ea2231d8 100644 --- a/relay-test/Cargo.toml +++ b/relay-test/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "Test utilities for Relay" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay-ua/Cargo.toml b/relay-ua/Cargo.toml index b3da7547c2..4d2087729b 100644 --- a/relay-ua/Cargo.toml +++ b/relay-ua/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "User agent parser with built-in rules" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false diff --git a/relay/Cargo.toml b/relay/Cargo.toml index 90a255b35d..751d8f001e 100644 --- a/relay/Cargo.toml +++ b/relay/Cargo.toml @@ -4,7 +4,7 @@ authors = ["Sentry "] description = "The Relay binary, a proxy server for Sentry" homepage = "https://getsentry.github.io/relay/" repository = "https://github.com/getsentry/relay" -version = "24.9.0" +version = "24.10.0" edition = "2021" license-file = "../LICENSE.md" publish = false