services/horizon/internal/httpx: Add prometheus metrics for requests …

…in flight and requests received (stellar#5240)
chowbao · Mar 11, 2024 · 6fe5270 · 6fe5270
1 parent ab3a926
commit 6fe5270
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 3 deletions.
diff --git a/services/horizon/internal/httpx/middleware.go b/services/horizon/internal/httpx/middleware.go
@@ -78,11 +78,21 @@ func loggerMiddleware(serverMetrics *ServerMetrics) func(next http.Handler) http
 			// is reset before sending the first event no Content-Type header is sent in a response.
 			acceptHeader := r.Header.Get("Accept")
 			streaming := strings.Contains(acceptHeader, render.MimeEventStream)
+			route := supportHttp.GetChiRoutePattern(r)
+
+			requestLabels := prometheus.Labels{
+				"route":     route,
+				"streaming": strconv.FormatBool(streaming),
+				"method":    r.Method,
+			}
+			serverMetrics.RequestsInFlightGauge.With(requestLabels).Inc()
+			defer serverMetrics.RequestsInFlightGauge.With(requestLabels).Dec()
+			serverMetrics.RequestsReceivedCounter.With(requestLabels).Inc()
 
 			then := time.Now()
 			next.ServeHTTP(mw, r.WithContext(ctx))
 			duration := time.Since(then)
-			logEndOfRequest(ctx, r, serverMetrics.RequestDurationSummary, duration, mw, streaming)
+			logEndOfRequest(ctx, r, route, serverMetrics.RequestDurationSummary, duration, mw, streaming)
 		})
 	}
 }
@@ -129,8 +139,7 @@ func getClientData(r *http.Request, headerName string) string {
 	return value
 }
 
-func logEndOfRequest(ctx context.Context, r *http.Request, requestDurationSummary *prometheus.SummaryVec, duration time.Duration, mw middleware.WrapResponseWriter, streaming bool) {
-	route := supportHttp.GetChiRoutePattern(r)
+func logEndOfRequest(ctx context.Context, r *http.Request, route string, requestDurationSummary *prometheus.SummaryVec, duration time.Duration, mw middleware.WrapResponseWriter, streaming bool) {
 
 	referer := r.Referer()
 	if referer == "" {

diff --git a/services/horizon/internal/httpx/server.go b/services/horizon/internal/httpx/server.go
@@ -23,6 +23,8 @@ import (
 type ServerMetrics struct {
 	RequestDurationSummary  *prometheus.SummaryVec
 	ReplicaLagErrorsCounter prometheus.Counter
+	RequestsInFlightGauge   *prometheus.GaugeVec
+	RequestsReceivedCounter *prometheus.CounterVec
 }
 
 type TLSConfig struct {
@@ -71,6 +73,19 @@ func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState
 			},
 			[]string{"status", "route", "streaming", "method"},
 		),
+		RequestsInFlightGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "horizon", Subsystem: "http", Name: "requests_in_flight",
+				Help: "HTTP requests in flight",
+			},
+			[]string{"route", "streaming", "method"},
+		),
+		RequestsReceivedCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: "horizon", Subsystem: "http", Name: "requests_received",
+			},
+			[]string{"route", "streaming", "method"},
+		),
 		ReplicaLagErrorsCounter: prometheus.NewCounter(
 			prometheus.CounterOpts{
 				Namespace: "horizon", Subsystem: "http", Name: "replica_lag_errors_count",
@@ -109,6 +124,8 @@ func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState
 func (s *Server) RegisterMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(s.Metrics.RequestDurationSummary)
 	registry.MustRegister(s.Metrics.ReplicaLagErrorsCounter)
+	registry.MustRegister(s.Metrics.RequestsInFlightGauge)
+	registry.MustRegister(s.Metrics.RequestsReceivedCounter)
 }
 
 func (s *Server) Serve() error {