Skip to content

Commit

Permalink
conprof: support heap profiling for tikv (#199)
Browse files Browse the repository at this point in the history
close #197
  • Loading branch information
Connor1996 authored Nov 8, 2023
1 parent 1834b43 commit b5ea410
Show file tree
Hide file tree
Showing 9 changed files with 5,919 additions and 19 deletions.
26 changes: 19 additions & 7 deletions component/conprof/http/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

"github.com/pingcap/ng-monitoring/component/conprof"
"github.com/pingcap/ng-monitoring/component/conprof/jeprof"
"github.com/pingcap/ng-monitoring/component/conprof/meta"
"github.com/pingcap/ng-monitoring/component/topology"
"github.com/pingcap/ng-monitoring/config"
Expand Down Expand Up @@ -121,7 +122,8 @@ func getProfileEstimateSize(component topology.Component) int {
400*1024 + // heap size
30*1024 // mutex size
case topology.ComponentTiKV:
return 200 * 1024 // profile size
return 200*1024 + // profile size
200*1024 // heap size
case topology.ComponentTiFlash:
// TODO: remove this after TiFlash fix the profile bug.
return 0
Expand Down Expand Up @@ -307,7 +309,13 @@ func querySingleProfileView(c *gin.Context) ([]byte, error) {
if err != nil {
return nil, err
}
if param.DataFormat == meta.ProfileDataFormatSVG {
if param.Targets[0].Component == topology.ComponentTiKV && param.Targets[0].Kind == meta.ProfileKindHeap {
if param.DataFormat == meta.ProfileDataFormatSVG {
return jeprof.ConvertToSVG(profileData)
} else if param.DataFormat == meta.ProfileDataFormatText {
return jeprof.ConvertToText(profileData)
}
} else if param.DataFormat == meta.ProfileDataFormatSVG {
if svg, err := ConvertToSVG(profileData); err == nil {
return svg, nil
}
Expand Down Expand Up @@ -341,6 +349,8 @@ func queryAndDownload(c *gin.Context) error {
fileName = strings.ReplaceAll(fileName, ":", "_")
if pt.Kind == meta.ProfileKindGoroutine {
fileName += ".txt"
} else if pt.Kind == meta.ProfileKindHeap && pt.Component == topology.ComponentTiKV {
fileName += ".prof"
} else {
fileName += ".proto"
}
Expand Down Expand Up @@ -380,9 +390,11 @@ func queryAndDownload(c *gin.Context) error {
}

const downloadReadme = `
To review the profile data whose file name suffix is '.proto' interactively:
To review the go profile data whose file name suffix is '.proto' interactively:
$ go tool pprof --http=127.0.0.1:6060 profile_xxx.proto
To review the jemalloc profile data whose file name suffix is '.prof' interactively:
$ jeprof --web profile_xxx.prof
`

var (
Expand Down Expand Up @@ -454,11 +466,11 @@ func getParamFromRequest(r *http.Request, param *meta.BasicQueryParam, paramName
param.Limit = value
case dataFormatParamStr:
switch v {
case meta.ProfileDataFormatSVG, meta.ProfileDataFormatProtobuf:
case meta.ProfileDataFormatSVG, meta.ProfileDataFormatProtobuf, meta.ProfileDataFormatJeprof, meta.ProfileDataFormatText:
param.DataFormat = v
default:
return fmt.Errorf("invalid param %v value %v, expected: %v, %v",
dataFormatParamStr, v, meta.ProfileDataFormatSVG, meta.ProfileDataFormatProtobuf)
return fmt.Errorf("invalid param %v value %v, expected: %v, %v, %v, %v",
dataFormatParamStr, v, meta.ProfileDataFormatSVG, meta.ProfileDataFormatProtobuf, meta.ProfileDataFormatJeprof, meta.ProfileDataFormatText)
}
default:
return fmt.Errorf("unknow param %s", paramName)
Expand Down
19 changes: 11 additions & 8 deletions component/conprof/http/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,15 @@ func TestAPI(t *testing.T) {
ts.setup(t)
defer ts.close(t)

httpAddr := setupHTTPService(t)
mockServer := testutil.CreateMockProfileServer(t)
defer mockServer.Stop(t)

topoSubScribe := make(topology.Subscriber)
err := conprof.Init(ts.db, topoSubScribe)
require.NoError(t, err)
defer conprof.Stop()

httpAddr := setupHTTPService(t)

mockServer := testutil.CreateMockProfileServer(t)
defer mockServer.Stop(t)

addr := mockServer.Addr
port := mockServer.Port
components := []topology.Component{
Expand Down Expand Up @@ -191,7 +190,11 @@ func testAPIDownload(t *testing.T, httpAddr string, ts int64, components []topol
buf := make([]byte, len(fields[0]))
n, _ := reader.Read(buf)
require.Equal(t, len(fields[0]), n)
require.Equal(t, fields[0], string(buf))
if fields[1] == "tikv" && fields[0] == "heap" {
require.Equal(t, "--- ", string(buf))
} else {
require.Equal(t, fields[0], string(buf))
}

if idx == len(urls)-1 {
// test for download single profile
Expand Down Expand Up @@ -249,7 +252,7 @@ func testAPIEstimateSize(t *testing.T, httpAddr string, components []topology.Co
err = json.Unmarshal(body, &estimateSize)
require.NoError(t, err)
require.Equal(t, len(components), estimateSize.InstanceCount, string(body))
require.Equal(t, 88915968000, estimateSize.ProfileSize)
require.Equal(t, 106610688000, estimateSize.ProfileSize)
}

func testErrorRequest(t *testing.T, httpAddr string) {
Expand All @@ -273,7 +276,7 @@ func testErrorRequest(t *testing.T, httpAddr string) {
{"/single_profile/view?ts=x", `{"message":"invalid param ts value, error: strconv.ParseInt: parsing \"x\": invalid syntax","status":"error"}`},
{"/single_profile/view?ts=0", `{"message":"need param profile_type","status":"error"}`},
{"/single_profile/view?ts=0&data_format=svg", `{"message":"need param profile_type","status":"error"}`},
{"/single_profile/view?ts=0&data_format=unknown", `{"message":"invalid param data_format value unknown, expected: svg, protobuf","status":"error"}`},
{"/single_profile/view?ts=0&data_format=unknown", `{"message":"invalid param data_format value unknown, expected: svg, protobuf, jeprof, text","status":"error"}`},
{"/single_profile/view?ts=0&profile_type=heap", `{"message":"need param component","status":"error"}`},
{"/single_profile/view?ts=0&profile_type=heap&component=tidb", `{"message":"need param address","status":"error"}`},

Expand Down
106 changes: 106 additions & 0 deletions component/conprof/jeprof/jeprof.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package jeprof

import (
"bytes"
_ "embed"
"fmt"
"io"
"os"
"os/exec"
"strings"

graphviz "github.com/goccy/go-graphviz"
)

//go:embed jeprof.in
var jeprof string

func FetchRaw(url string) ([]byte, error) {
cmd := exec.Command("perl", "/dev/stdin", "--raw", url) //nolint:gosec
cmd.Stdin = strings.NewReader(jeprof)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
err = cmd.Start()
if err != nil {
return nil, err
}
data, err := io.ReadAll(stdout)
if err != nil {
return nil, err
}
errMsg, err := io.ReadAll(stderr)
if err != nil {
return nil, err
}
err = cmd.Wait()
if err != nil {
return nil, fmt.Errorf("failed to fetch tikv heap profile: %s", errMsg)
}
return data, nil
}

func ConvertToSVG(data []byte) ([]byte, error) {
f, err := os.CreateTemp("", "prof")
if err != nil {
return nil, err
}
defer os.Remove(f.Name())
_, err = f.Write(data)
if err != nil {
return nil, err
}

cmd := exec.Command("perl", "/dev/stdin", "--dot", f.Name()) //nolint:gosec
cmd.Stdin = strings.NewReader(jeprof)
dotContent, err := cmd.Output()
if err != nil {
return nil, err
}
svgContent, err := convertDotToSVG(dotContent)
if err != nil {
return nil, err
}
return svgContent, nil
}

func convertDotToSVG(dotData []byte) ([]byte, error) {
g := graphviz.New()
graph, err := graphviz.ParseBytes(dotData)
if err != nil {
return nil, err
}

buf := bytes.NewBuffer(nil)
err = g.Render(graph, graphviz.SVG, buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}

func ConvertToText(data []byte) ([]byte, error) {
f, err := os.CreateTemp("", "prof")
if err != nil {
return nil, err
}
defer os.Remove(f.Name())
_, err = f.Write(data)
if err != nil {
return nil, err
}

// Brendan Gregg's collapsed stack format
cmd := exec.Command("perl", "/dev/stdin", "--collapsed", f.Name()) //nolint:gosec
cmd.Stdin = strings.NewReader(jeprof)
textContent, err := cmd.Output()
if err != nil {
return nil, err
}
return textContent, nil
}
Loading

0 comments on commit b5ea410

Please sign in to comment.