Skip to content

Commit

Permalink
feat: Support exporting metrics in Prometheus format (#255)
Browse files Browse the repository at this point in the history
  • Loading branch information
chaneytech authored Dec 18, 2024
1 parent ee1b726 commit 7c42e21
Show file tree
Hide file tree
Showing 23 changed files with 214 additions and 4 deletions.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ The example shows how to use opentelemetry-go-auto-instrumentation to report met

## Do hybrid compilation with otel

Do hybrid compilation with otel according to [README.md](../demo/README.md).
Do hybrid compilation with otel according to [README.md](../../demo/README.md).

## Start OpenTelemetry Collector and Prometheus server

Expand Down
File renamed without changes.
File renamed without changes
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ module example/metrics

go 1.22

replace github.com/alibaba/opentelemetry-go-auto-instrumentation => ../../../opentelemetry-go-auto-instrumentation
replace github.com/alibaba/opentelemetry-go-auto-instrumentation => ./../../..

replace github.com/alibaba/opentelemetry-go-auto-instrumentation/test/verifier => ../../../opentelemetry-go-auto-instrumentation/test/verifier
replace github.com/alibaba/opentelemetry-go-auto-instrumentation/test/verifier => ./../../../test/verifier
File renamed without changes.
File renamed without changes
7 changes: 7 additions & 0 deletions example/metrics/prometheus-exporter/.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Dependent images
PROMETHEUS_IMAGE=quay.io/prometheus/prometheus:v2.54.1

# Prometheus
PROMETHEUS_SERVICE_PORT=9090
PROMETHEUS_SERVICE_HOST=prometheus
PROMETHEUS_ADDR=${PROMETHEUS_SERVICE_HOST}:${PROMETHEUS_SERVICE_PORT}
44 changes: 44 additions & 0 deletions example/metrics/prometheus-exporter/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
## Use opentelemetry-go-auto-instrumentation to report metrics

The example shows how to use opentelemetry-go-auto-instrumentation to report metrics in prometheus format.

## Do hybrid compilation with otel

Do hybrid compilation with otel according to [README.md](../../demo/README.md).

## Start OpenTelemetry Collector and Prometheus server

You can use `docker-compose` to run the Prometheus server locally.

```shell
docker compose up --force-recreate --remove-orphans --detach
```

## Run Application

Set the endpoint and other otel-related environment variables according to
the [document](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and execute the
binary.

```shell
OTEL_METRICS_EXPORTER=prometheus OTEL_EXPORTER_PROMETHEUS_PORT=9464 ./metrics
```

And you can request to the server to generate some metrics:

```shell
# For Golang GC metrics
curl localhost:9000/gc-metrics
# For Golang Memory metrics
curl localhost:9000/mem-metrics
```

## Query Metrics

Open the http://localhost:9464/metrics to query metrics in local server:
![metrics.png](metrics.png)


Open the Prometheus UI at http://localhost:9090 to query metrics in prometheus server:
![gc_count.png](gc_count.png)
![mem_alloc.png](mem_alloc.png)
42 changes: 42 additions & 0 deletions example/metrics/prometheus-exporter/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

x-default-logging: &logging
driver: "json-file"
options:
max-size: "5m"
max-file: "2"
tag: "{{.Name}}"

networks:
default:
name: opentelemetry-demo
driver: bridge

services:
# Prometheus
prometheus:
image: ${PROMETHEUS_IMAGE}
container_name: prometheus
command:
- --web.console.templates=/etc/prometheus/consoles
- --web.console.libraries=/etc/prometheus/console_libraries
- --storage.tsdb.retention.time=1h
- --config.file=/etc/prometheus/prometheus-config.yaml
- --storage.tsdb.path=/prometheus
- --web.enable-lifecycle
- --web.route-prefix=/
- --enable-feature=exemplar-storage
- --enable-feature=otlp-write-receiver
volumes:
- ./src/prometheus/prometheus-config.yaml:/etc/prometheus/prometheus-config.yaml:ro
deploy:
resources:
limits:
memory: 300M
restart: unless-stopped
ports:
- "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
logging: *logging
extra_hosts:
- "host.docker.internal:host-gateway"
Binary file added example/metrics/prometheus-exporter/gc_count.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
7 changes: 7 additions & 0 deletions example/metrics/prometheus-exporter/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
module example/metrics

go 1.22

replace github.com/alibaba/opentelemetry-go-auto-instrumentation => ./../../..

replace github.com/alibaba/opentelemetry-go-auto-instrumentation/test/verifier => ./../../../test/verifier
42 changes: 42 additions & 0 deletions example/metrics/prometheus-exporter/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright (c) 2024 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"net/http"
"runtime"
)

func main() {
http.Handle("/gc-metrics", http.HandlerFunc(gcMetrics))
http.Handle("/mem-metrics", http.HandlerFunc(memMetrics))
err := http.ListenAndServe("0.0.0.0:9000", nil)
if err != nil {
panic(err)
}
}

func gcMetrics(w http.ResponseWriter, r *http.Request) {
runtime.GC()
w.Write([]byte("Get GC Metrics"))
}

func memMetrics(w http.ResponseWriter, r *http.Request) {
var bytes []byte
for i := 0; i < 10; i++ {
bytes = append(bytes, make([]byte, 1024*1024)...)
}
w.Write([]byte("Get Memory Metrics"))
}
Binary file added example/metrics/prometheus-exporter/mem_alloc.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added example/metrics/prometheus-exporter/metrics.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

global:
evaluation_interval: 30s
scrape_interval: 5s
storage:
tsdb:
out_of_order_time_window: 30m
scrape_configs:
- job_name: metrics-scrape
static_configs:
- targets:
- 'host.docker.internal:9464'
7 changes: 7 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/gomodule/redigo v1.9.0
github.com/gorilla/mux v1.8.1
github.com/labstack/echo/v4 v4.12.0
github.com/prometheus/client_golang v1.16.0
github.com/redis/go-redis/v9 v9.6.1
github.com/sirupsen/logrus v1.9.3
github.com/testcontainers/testcontainers-go v0.34.0
Expand All @@ -35,6 +36,7 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0
go.opentelemetry.io/otel/exporters/prometheus v0.42.0
go.opentelemetry.io/otel/metric v1.32.0
go.opentelemetry.io/otel/sdk v1.32.0
go.opentelemetry.io/otel/sdk/metric v1.32.0
Expand All @@ -51,6 +53,7 @@ require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bytedance/go-tagexpr/v2 v2.9.2 // indirect
github.com/bytedance/gopkg v0.1.0 // indirect
github.com/bytedance/sonic v1.12.0 // indirect
Expand Down Expand Up @@ -101,6 +104,7 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
Expand All @@ -117,6 +121,9 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
Expand Down
15 changes: 15 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
Expand Down Expand Up @@ -201,6 +203,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
Expand Down Expand Up @@ -249,6 +253,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
Expand Down Expand Up @@ -342,6 +354,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S2
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA=
go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA=
go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
Expand Down Expand Up @@ -384,6 +398,7 @@ golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfS
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down
32 changes: 31 additions & 1 deletion pkg/otel_setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,11 @@ package pkg
import (
"context"
"errors"
"fmt"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel/exporters/prometheus"
"log"
http2 "net/http"
"os"
"strings"

Expand Down Expand Up @@ -46,6 +50,9 @@ import (
const exec_name = "otel"
const report_protocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
const trace_report_protocol = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
const metrics_exporter = "OTEL_METRICS_EXPORTER"
const prometheus_exporter_port = "OTEL_EXPORTER_PROMETHEUS_PORT"
const default_prometheus_exporter_port = "9464"

func init() {
path, err := os.Executable()
Expand Down Expand Up @@ -115,7 +122,16 @@ func initMetrics() error {
metric.WithReader(verifier.ManualReader),
)
} else {
if os.Getenv(report_protocol) == "grpc" || os.Getenv(trace_report_protocol) == "grpc" {
if os.Getenv(metrics_exporter) == "prometheus" {
exporter, err := prometheus.New()
if err != nil {
log.Fatalf("new otlp metric prometheus exporter failed: %v", err)
}
mp = metric.NewMeterProvider(
metric.WithReader(exporter),
)
go serveMetrics()
} else if os.Getenv(report_protocol) == "grpc" || os.Getenv(trace_report_protocol) == "grpc" {
exporter, err := otlpmetricgrpc.New(ctx)
if err != nil {
log.Fatalf("new otlp metric grpc exporter failed: %v", err)
Expand Down Expand Up @@ -144,3 +160,17 @@ func initMetrics() error {
// DefaultMinimumReadMemStatsInterval is 15 second
return runtime.Start(runtime.WithMeterProvider(mp))
}

func serveMetrics() {
http2.Handle("/metrics", promhttp.Handler())
port := os.Getenv(prometheus_exporter_port)
if port == "" {
port = default_prometheus_exporter_port
}
log.Printf("serving serveMetrics at localhost:%s/metrics", port)
err := http2.ListenAndServe(fmt.Sprintf(":%s", port), nil)
if err != nil {
fmt.Printf("error serving serveMetrics: %v", err)
return
}
}
2 changes: 2 additions & 0 deletions tool/preprocess/preprocess.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ var fixedDeps = []struct {
"v1.32.0", true, false},
{"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp",
"v1.32.0", true, false},
{"go.opentelemetry.io/otel/exporters/prometheus",
"v0.42.0", true, false},
// otel contrib
{"go.opentelemetry.io/contrib/instrumentation/runtime",
"v0.57.0", false, false},
Expand Down

0 comments on commit 7c42e21

Please sign in to comment.