diff --git a/docker-compose.yml b/docker-compose.yml index abf71a535a..892181dfb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -278,6 +278,18 @@ services: snuba-issue-occurrence-consumer: <<: *snuba_defaults command: consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + snuba-metrics-consumer: + <<: *snuba_defaults + command: consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + snuba-generic-metrics-distributions-consumer: + <<: *snuba_defaults + command: consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + snuba-generic-metrics-sets-consumer: + <<: *snuba_defaults + command: consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + snuba-generic-metrics-counters-consumer: + <<: *snuba_defaults + command: consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset @@ -290,6 +302,9 @@ services: snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --delay-seconds=60 --schedule-ttl=60 --stale-threshold-seconds=900 + snuba-subscription-consumer-metrics: + <<: *snuba_defaults + command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --delay-seconds=60 --schedule-ttl=60 --stale-threshold-seconds=900 snuba-profiling-profiles-consumer: <<: *snuba_defaults command: consumer --storage profiles --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset @@ -345,6 +360,15 @@ services: transactions-consumer: <<: *sentry_defaults command: run consumer ingest-transactions --consumer-group ingest-consumer + metrics-consumer: + <<: *sentry_defaults + command: run consumer ingest-metrics --consumer-group metrics-consumer + generic-metrics-consumer: + <<: *sentry_defaults + command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer + billing-metrics-consumer: + <<: *sentry_defaults + command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer ingest-replay-recordings: <<: *sentry_defaults command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings @@ -369,6 +393,12 @@ services: subscription-consumer-transactions: <<: *sentry_defaults command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer + subscription-consumer-metrics: + <<: *sentry_defaults + command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer + subscription-consumer-generic-metrics: + <<: *sentry_defaults + command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer sentry-cleanup: <<: *sentry_defaults image: sentry-cleanup-self-hosted-local diff --git a/install/create-kafka-topics.sh b/install/create-kafka-topics.sh index 7d8a6dd575..9c9b723e51 100644 --- a/install/create-kafka-topics.sh +++ b/install/create-kafka-topics.sh @@ -3,7 +3,7 @@ echo "${_group}Creating additional Kafka topics ..." # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) EXISTING_KAFKA_TOPICS=$($dcr -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) -NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles ingest-occurrences" +NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles ingest-occurrences ingest-metrics ingest-performance-metrics" for topic in $NEEDED_KAFKA_TOPICS; do if ! echo "$EXISTING_KAFKA_TOPICS" | grep -qE "(^| )$topic( |$)"; then $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 2988ecf76b..3f3fc01a9c 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -188,6 +188,13 @@ def get_internal_network(): SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend" +################### +# Metrics Backend # +################### + +SENTRY_RELEASE_HEALTH = "sentry.release_health.metrics.MetricsReleaseHealthBackend" +SENTRY_RELEASE_MONITOR = "sentry.release_health.release_monitor.metrics.MetricReleaseMonitorBackend" + ############## # Web Server # ############## @@ -271,6 +278,11 @@ def get_internal_network(): "organizations:session-replay", "organizations:issue-platform", "organizations:profiling", + "organizations:dashboards-mep", + "organizations:mep-rollout-flag", + "organizations:dashboards-rh-widget", + "organizations:metrics-extraction", + "organizations:transaction-metrics-extraction", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups",