Add more prometheus metrics

go.mod

@@ -99,6 +99,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/otp v1.4.0
 	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/common v0.60.1
 	github.com/quasoft/websspi v1.1.2
 	github.com/redis/go-redis/v9 v9.7.0
 	github.com/robfig/cron/v3 v3.0.1
@@ -241,6 +242,7 @@ require (
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/libdns/libdns v0.2.2 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -268,7 +270,6 @@ require (
 	github.com/pjbgf/sha1cd v0.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.60.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rhysd/actionlint v1.7.3 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect

modules/cache/cache.go

@@ -10,10 +10,39 @@ import (
 
 	"code.gitea.io/gitea/modules/setting"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
 	_ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache"
 )
 
-var defaultCache StringCache
+var (
+	defaultCache StringCache
+
+	// TODO: Combine hit and miss into one
+	hitCounter = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace:   "gitea",
+		Help:        "Cache count",
+		Subsystem:   "cache",
+		Name:        "response",
+		ConstLabels: prometheus.Labels{"state": "hit"},
+	})
+	missCounter = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace:   "gitea",
+		Help:        "Cache count",
+		Subsystem:   "cache",
+		Name:        "response",
+		ConstLabels: prometheus.Labels{"state": "miss"},
+	})
+	latencyHistogram = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "gitea",
+			Help:      "Cache latency",
+			Subsystem: "cache",
+			Name:      "duration",
+		},
+	)
+)
 
 // Init start cache service
 func Init() error {

modules/cache/string_cache.go

@@ -6,6 +6,7 @@ package cache
 import (
 	"errors"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/setting"
@@ -63,10 +64,15 @@ func (sc *stringCache) Ping() error {
 }
 
 func (sc *stringCache) Get(key string) (string, bool) {
+	start := time.Now()
 	v := sc.chiCache.Get(key)
+	elapsed := time.Since(start).Seconds()
+	latencyHistogram.Observe(elapsed)
 	if v == nil {
+		missCounter.Add(1)
 		return "", false
 	}
+	hitCounter.Add(1)
 	s, ok := v.(string)
 	return s, ok
 }

modules/metrics/collector.go

@@ -11,10 +11,15 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/model"
 )
 
 const namespace = "gitea_"
 
+func init() {
+	model.NameValidationScheme = model.UTF8Validation
+}
+
 // Collector implements the prometheus.Collector interface and
 // exposes gitea metrics for prometheus
 type Collector struct {
@@ -89,7 +94,7 @@ func NewCollector() Collector {
 		Issues: prometheus.NewDesc(
 			namespace+"issues",
 			"Number of Issues",
-			nil, nil,
+			[]string{"state"}, nil,
 		),
 		IssuesByLabel: prometheus.NewDesc(
 			namespace+"issues_by_label",
@@ -103,12 +108,12 @@ func NewCollector() Collector {
 		),
 		IssuesOpen: prometheus.NewDesc(
 			namespace+"issues_open",
-			"Number of open Issues",
+			"DEPRECATED: Use Issues with state: open",
 			nil, nil,
 		),
 		IssuesClosed: prometheus.NewDesc(
 			namespace+"issues_closed",
-			"Number of closed Issues",
+			"DEPRECATED: Use Issues with state: closed",
 			nil, nil,
 		),
 		Labels: prometheus.NewDesc(
@@ -272,8 +277,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		c.Issues,
 		prometheus.GaugeValue,
-		float64(stats.Counter.Issue),
+		float64(stats.Counter.IssueOpen), "open",
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Issues,
+		prometheus.GaugeValue,
+		float64(stats.Counter.IssueClosed), "closed",
 	)
+
 	for _, il := range stats.Counter.IssueByLabel {
 		ch <- prometheus.MustNewConstMetric(
 			c.IssuesByLabel,

routers/common/middleware.go

@@ -6,7 +6,9 @@ package common
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/gtprof"
@@ -19,6 +21,45 @@ import (
 	"gitea.com/go-chi/session"
 	"github.com/chi-middleware/proxy"
 	"github.com/go-chi/chi/v5"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	httpRequestMethod      = "http_request_method"
+	httpResponseStatusCode = "http_response_status_code"
+	httpRoute              = "http_route"
+)
+
+var (
+	// reqInflightGauge tracks the amount of currently handled requests
+	reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "http",
+		Subsystem: "server",
+		Name:      "active_requests",
+		Help:      "Number of active HTTP server requests.",
+	}, []string{httpRequestMethod})
+	// reqDurationHistogram tracks the time taken by http request
+	reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "http",
+		Subsystem: "server",
+		Name:      "request_duration",
+		Help:      "Measures the latency of HTTP requests processed by the server",
+	}, []string{httpRequestMethod, httpResponseStatusCode, httpRoute})
+	// reqSizeHistogram tracks the size of request
+	reqSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "http",
+		Subsystem: "server_request",
+		Name:      "body_size",
+		Help:      "Size of HTTP server request bodies.",
+	}, []string{httpRequestMethod, httpResponseStatusCode, httpRoute})
+	// respSizeHistogram tracks the size of the response
+	respSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "http",
+		Subsystem: "server_response",
+		Name:      "body_size",
+		Help:      "Size of HTTP server response bodies.",
+	}, []string{httpRequestMethod, httpResponseStatusCode, httpRoute})
 )
 
 // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery
@@ -38,6 +79,9 @@ func ProtocolMiddlewares() (handlers []any) {
 	if setting.IsAccessLogEnabled() {
 		handlers = append(handlers, context.AccessLogger())
 	}
+	if setting.Metrics.Enabled {
+		handlers = append(handlers, RouteMetrics())
+	}
 
 	return handlers
 }
@@ -107,6 +151,30 @@ func ForwardedHeadersHandler(limit int, trustedProxies []string) func(h http.Han
 	return proxy.ForwardedHeaders(opt)
 }
 
+// RouteMetrics instruments http requests and responses
+func RouteMetrics() func(h http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+			inflight := reqInflightGauge.WithLabelValues(req.Method)
+			inflight.Inc()
+			defer inflight.Dec()
+			start := time.Now()
+
+			next.ServeHTTP(resp, req)
+			m := context.WrapResponseWriter(resp)
+			route := chi.RouteContext(req.Context()).RoutePattern()
+			code := strconv.Itoa(m.WrittenStatus())
+			reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(time.Since(start).Seconds())
+			respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.WrittenSize()))
+			size := req.ContentLength
+			if size < 0 {
+				size = 0
+			}
+			reqSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(size))
+		})
+	}
+}
+
 func Sessioner() func(next http.Handler) http.Handler {
 	return session.Sessioner(session.Options{
 		Provider:       setting.SessionConfig.Provider,

routers/common/middleware_test.go

@@ -0,0 +1,38 @@
+package common
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetricsMiddlewere(t *testing.T) {
+	middleware := RouteMetrics()
+	r := chi.NewRouter()
+	r.Use(middleware)
+	r.Get("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("test"))
+		time.Sleep(5 * time.Millisecond)
+	}))
+	testServer := httptest.NewServer(r)
+	// Check all defined metrics
+	verify := func(i int) {
+		assert.Equal(t, testutil.CollectAndCount(reqDurationHistogram, "http_server_request_duration"), i)
+		assert.Equal(t, testutil.CollectAndCount(reqSizeHistogram, "http_server_request_body_size"), i)
+		assert.Equal(t, testutil.CollectAndCount(respSizeHistogram, "http_server_response_body_size"), i)
+		assert.Equal(t, testutil.CollectAndCount(reqInflightGauge, "http_server_active_requests"), i)
+	}
+
+	// Check they don't exist before making a request
+	verify(0)
+	_, err := http.Get(testServer.URL)
+	require.NoError(t, err)
+	// Check they do exist after making the request
+	verify(1)
+}

routers/web/web.go

@@ -52,6 +52,7 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 )
 
 var GzipMinSize = 1400 // min size to compress for the body size of response
@@ -258,6 +259,7 @@ func Routes() *web.Router {
 	}
 
 	if setting.Metrics.Enabled {
+		prometheus.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{Namespace: "gitea"}))
 		prometheus.MustRegister(metrics.NewCollector())
 		routes.Get("/metrics", append(mid, Metrics)...)
 	}

services/migrations/migrate.go

@@ -21,6 +21,14 @@ import (
 	base "code.gitea.io/gitea/modules/migration"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	repoMigrationsInflightGauge = promauto.NewGauge(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "inflight_migrations", Help: "Number of inflight repository migrations"})
+	repoMigrationsCounter       = promauto.NewGaugeVec(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "migrations", Help: "Total migrations"}, []string{"result"})
 )
 
 // MigrateOptions is equal to base.MigrateOptions
@@ -124,6 +132,9 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str
 		return nil, err
 	}
 
+	repoMigrationsInflightGauge.Inc()
+	defer repoMigrationsInflightGauge.Dec()
+
 	uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
 	uploader.gitServiceType = opts.GitServiceType
 
@@ -134,8 +145,10 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str
 		if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
 			log.Error("create respotiry notice failed: ", err2)
 		}
+		repoMigrationsCounter.WithLabelValues("fail").Inc()
 		return nil, err
 	}
+	repoMigrationsCounter.WithLabelValues("success").Inc()
 	return uploader.repo, nil
 }