From 6fe5270b6f0b1582f6353df93c2dc2a12157cfec Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Mon, 11 Mar 2024 09:34:40 +0000
Subject: [PATCH] services/horizon/internal/httpx: Add prometheus metrics for
 requests in flight and requests received (#5240)

---
 services/horizon/internal/httpx/middleware.go | 15 ++++++++++++---
 services/horizon/internal/httpx/server.go     | 17 +++++++++++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/services/horizon/internal/httpx/middleware.go b/services/horizon/internal/httpx/middleware.go
index cdcd7f4e3c..eaa49634a5 100644
--- a/services/horizon/internal/httpx/middleware.go
+++ b/services/horizon/internal/httpx/middleware.go
@@ -78,11 +78,21 @@ func loggerMiddleware(serverMetrics *ServerMetrics) func(next http.Handler) http
 			// is reset before sending the first event no Content-Type header is sent in a response.
 			acceptHeader := r.Header.Get("Accept")
 			streaming := strings.Contains(acceptHeader, render.MimeEventStream)
+			route := supportHttp.GetChiRoutePattern(r)
+
+			requestLabels := prometheus.Labels{
+				"route":     route,
+				"streaming": strconv.FormatBool(streaming),
+				"method":    r.Method,
+			}
+			serverMetrics.RequestsInFlightGauge.With(requestLabels).Inc()
+			defer serverMetrics.RequestsInFlightGauge.With(requestLabels).Dec()
+			serverMetrics.RequestsReceivedCounter.With(requestLabels).Inc()
 
 			then := time.Now()
 			next.ServeHTTP(mw, r.WithContext(ctx))
 			duration := time.Since(then)
-			logEndOfRequest(ctx, r, serverMetrics.RequestDurationSummary, duration, mw, streaming)
+			logEndOfRequest(ctx, r, route, serverMetrics.RequestDurationSummary, duration, mw, streaming)
 		})
 	}
 }
@@ -129,8 +139,7 @@ func getClientData(r *http.Request, headerName string) string {
 	return value
 }
 
-func logEndOfRequest(ctx context.Context, r *http.Request, requestDurationSummary *prometheus.SummaryVec, duration time.Duration, mw middleware.WrapResponseWriter, streaming bool) {
-	route := supportHttp.GetChiRoutePattern(r)
+func logEndOfRequest(ctx context.Context, r *http.Request, route string, requestDurationSummary *prometheus.SummaryVec, duration time.Duration, mw middleware.WrapResponseWriter, streaming bool) {
 
 	referer := r.Referer()
 	if referer == "" {
diff --git a/services/horizon/internal/httpx/server.go b/services/horizon/internal/httpx/server.go
index 65058c0ea5..262d51578d 100644
--- a/services/horizon/internal/httpx/server.go
+++ b/services/horizon/internal/httpx/server.go
@@ -23,6 +23,8 @@ import (
 type ServerMetrics struct {
 	RequestDurationSummary  *prometheus.SummaryVec
 	ReplicaLagErrorsCounter prometheus.Counter
+	RequestsInFlightGauge   *prometheus.GaugeVec
+	RequestsReceivedCounter *prometheus.CounterVec
 }
 
 type TLSConfig struct {
@@ -71,6 +73,19 @@ func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState
 			},
 			[]string{"status", "route", "streaming", "method"},
 		),
+		RequestsInFlightGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "horizon", Subsystem: "http", Name: "requests_in_flight",
+				Help: "HTTP requests in flight",
+			},
+			[]string{"route", "streaming", "method"},
+		),
+		RequestsReceivedCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: "horizon", Subsystem: "http", Name: "requests_received",
+			},
+			[]string{"route", "streaming", "method"},
+		),
 		ReplicaLagErrorsCounter: prometheus.NewCounter(
 			prometheus.CounterOpts{
 				Namespace: "horizon", Subsystem: "http", Name: "replica_lag_errors_count",
@@ -109,6 +124,8 @@ func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState
 func (s *Server) RegisterMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(s.Metrics.RequestDurationSummary)
 	registry.MustRegister(s.Metrics.ReplicaLagErrorsCounter)
+	registry.MustRegister(s.Metrics.RequestsInFlightGauge)
+	registry.MustRegister(s.Metrics.RequestsReceivedCounter)
 }
 
 func (s *Server) Serve() error {