profiler: Implement Delta Profiles (#842)

This patch implements delta profiles which will allow us to enable
allocation, mutex and block profiles in the aggregation and comparison
views of our web ui.

The internal google doc "RFC: Go Profiling: Delta Profiles" describes
this in great detail.

To simplify the code, a refactoring was done that attempts to increase
code sharing between similar profile types, in particular heap, mutex,
block and goroutine profiles (see type profileType).

Additionally the new profiler.enabledProfileTypes() method ensures that
profiles are collected in a deterministic order.

Testing is accomplished by the new pprofutils package
which allows converting profiles between protobuf and a simplified text
format. Since that package also contains a suitable delta profiling
implementation, it's used for the delta profiling itself as well.

In this iteration, delta profiles are uploaded in addition to the
original profiles using a "delta-" prefix, e.g. "delta-mutex.pprof".
This is done to avoid breaking things until the backend has made
corresponding changes as well. The plan for the next iteration is to
stop uploading the original profiles since they are redundant and a
waste of bandwidth and storage.

One particular complexity worth noting is that the "delta-heap.pprof"
contains 2 profiles alloc_ and inuse_. Only the alloc_ sample
types are subject to delta computation, the inuse_ ones are kept as-is
since they describe the current state of the heap's live set.

go.mod

@@ -6,6 +6,6 @@ require (
 	github.com/DataDog/datadog-go v4.4.0+incompatible
 	github.com/DataDog/gostackparse v0.5.0
 	github.com/DataDog/sketches-go v1.0.0
-	github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998
+	github.com/google/pprof v0.0.0-20210423192551-a2663126120b
 	github.com/tinylib/msgp v1.1.2
 )

go.sum

@@ -259,6 +259,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210423192551-a2663126120b h1:l2YRhr+YLzmSp7KJMswRVk/lO5SwoFIcCLzJsVj+YPc=
+github.com/google/pprof v0.0.0-20210423192551-a2663126120b/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
@@ -539,6 +541,7 @@ github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgh
 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=

profiler/internal/pprofutils/README.md

@@ -0,0 +1,7 @@
+# pprofutils
+
+Internal fork of https://github.com/felixge/pprofutils stripped to only include
+essential code and tests. It's used for delta profiles as well as testing.
+
+It'd be nice to keep this in sync with upstream, but no worries if not. We just
+need the delta profile stuff to work.

profiler/internal/pprofutils/delta.go

@@ -0,0 +1,66 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+package pprofutils
+
+import (
+	"errors"
+
+	"github.com/google/pprof/profile"
+)
+
+// Delta describes how to compute the delta between two profiles and implements
+// the conversion.
+type Delta struct {
+	// SampleTypes limits the delta calcultion to the given sample types. Other
+	// sample types will retain the values of profile b. The defined sample types
+	// must exist in the profile, otherwise derivation will fail with an error.
+	// If the slice is empty, all sample types are subject to delta profile
+	// derivation.
+	//
+	// The use case for this for this is to deal with the heap profile which
+	// contains alloc and inuse sample types, but delta profiling makes no sense
+	// for the latter.
+	SampleTypes []ValueType
+}
+
+// Convert computes the delta between all values b-a and returns them as a new
+// profile. Samples that end up with a delta of 0 are dropped. WARNING: Profile
+// a will be mutated by this function. You should pass a copy if that's
+// undesirable.
+func (d Delta) Convert(a, b *profile.Profile) (*profile.Profile, error) {
+	ratios := make([]float64, len(a.SampleType))
+
+	found := 0
+	for i, st := range a.SampleType {
+		// Empty c.SampleTypes means we calculate the delta for every st
+		if len(d.SampleTypes) == 0 {
+			ratios[i] = -1
+			continue
+		}
+
+		// Otherwise we only calcuate the delta for any st that is listed in
+		// c.SampleTypes. st's not listed in there will default to ratio 0, which
+		// means we delete them from pa, so only the pb values remain in the final
+		// profile.
+		for _, deltaSt := range d.SampleTypes {
+			if deltaSt.Type == st.Type && deltaSt.Unit == st.Unit {
+				ratios[i] = -1
+				found++
+			}
+		}
+	}
+	if found != len(d.SampleTypes) {
+		return nil, errors.New("one or more sample type(s) was not found in the profile")
+	}
+
+	a.ScaleN(ratios)
+
+	delta, err := profile.Merge([]*profile.Profile{a, b})
+	if err != nil {
+		return nil, err
+	}
+	return delta, delta.CheckValid()
+}

profiler/internal/pprofutils/delta_test.go

@@ -0,0 +1,85 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+package pprofutils
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDelta(t *testing.T) {
+	t.Run("simple", func(t *testing.T) {
+		var deltaText bytes.Buffer
+
+		profA, err := Text{}.Convert(strings.NewReader(strings.TrimSpace(`
+main;foo 5
+main;foo;bar 3
+main;foobar 4
+`)))
+		require.NoError(t, err)
+
+		profB, err := Text{}.Convert(strings.NewReader(strings.TrimSpace(`
+main;foo 8
+main;foo;bar 3
+main;foobar 5
+`)))
+		require.NoError(t, err)
+
+		delta, err := Delta{}.Convert(profA, profB)
+		require.NoError(t, err)
+
+		require.NoError(t, Protobuf{}.Convert(delta, &deltaText))
+		require.Equal(t, deltaText.String(), strings.TrimSpace(`
+main;foo 3
+main;foobar 1
+`)+"\n")
+	})
+
+	t.Run("sampleTypes", func(t *testing.T) {
+		profA, err := Text{}.Convert(strings.NewReader(strings.TrimSpace(`
+x/count y/count
+main;foo 5 10
+main;foo;bar 3 6
+main;foo;baz 9 0
+main;foobar 4 8
+`)))
+		require.NoError(t, err)
+
+		profB, err := Text{}.Convert(strings.NewReader(strings.TrimSpace(`
+x/count y/count
+main;foo 8 16
+main;foo;bar 3 6
+main;foo;baz 9 0
+main;foobar 5 10
+`)))
+		require.NoError(t, err)
+
+		t.Run("happyPath", func(t *testing.T) {
+			var deltaText bytes.Buffer
+
+			deltaConfig := Delta{SampleTypes: []ValueType{{Type: "x", Unit: "count"}}}
+			delta, err := deltaConfig.Convert(profA, profB)
+			require.NoError(t, err)
+
+			require.NoError(t, Protobuf{SampleTypes: true}.Convert(delta, &deltaText))
+			require.Equal(t, deltaText.String(), strings.TrimSpace(`
+x/count y/count
+main;foo 3 16
+main;foobar 1 10
+main;foo;bar 0 6
+`)+"\n")
+		})
+
+		t.Run("unknownSampleType", func(t *testing.T) {
+			deltaConfig := Delta{SampleTypes: []ValueType{{Type: "foo", Unit: "count"}}}
+			_, err := deltaConfig.Convert(profA, profB)
+			require.Equal(t, "one or more sample type(s) was not found in the profile", err.Error())
+		})
+	})
+}

profiler/internal/pprofutils/pprofutils.go

@@ -0,0 +1,13 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+// Package pprofutils is a fork of github.com/felixge/pprofutils, see README.
+package pprofutils
+
+// ValueType describes the type and unit of a value.
+type ValueType struct {
+	Type string
+	Unit string
+}

profiler/internal/pprofutils/protobuf.go

@@ -0,0 +1,67 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+package pprofutils
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/google/pprof/profile"
+)
+
+// Protobuf converts from pprof's protobuf to folded text format.
+type Protobuf struct {
+	// SampleTypes causes the text output to begin with a header line listing
+	// the sample types found in the profile. This is a custom extension to the
+	// folded text format.
+	SampleTypes bool
+}
+
+// Convert marshals the given protobuf profile into folded text format.
+func (p Protobuf) Convert(protobuf *profile.Profile, text io.Writer) error {
+	w := bufio.NewWriter(text)
+	if p.SampleTypes {
+		var sampleTypes []string
+		for _, sampleType := range protobuf.SampleType {
+			sampleTypes = append(sampleTypes, sampleType.Type+"/"+sampleType.Unit)
+		}
+		w.WriteString(strings.Join(sampleTypes, " ") + "\n")
+	}
+	if err := protobuf.Aggregate(true, true, false, false, false); err != nil {
+		return err
+	}
+	protobuf = protobuf.Compact()
+	sort.Slice(protobuf.Sample, func(i, j int) bool {
+		return protobuf.Sample[i].Value[0] > protobuf.Sample[j].Value[0]
+	})
+	for _, sample := range protobuf.Sample {
+		var frames []string
+		for i := range sample.Location {
+			loc := sample.Location[len(sample.Location)-i-1]
+			for j := range loc.Line {
+				line := loc.Line[len(loc.Line)-j-1]
+				frames = append(frames, line.Function.Name)
+			}
+		}
+		var values []string
+		for _, val := range sample.Value {
+			values = append(values, fmt.Sprintf("%d", val))
+			if !p.SampleTypes {
+				break
+			}
+		}
+		fmt.Fprintf(
+			w,
+			"%s %s\n",
+			strings.Join(frames, ";"),
+			strings.Join(values, " "),
+		)
+	}
+	return w.Flush()
+}

profiler/internal/pprofutils/protobuf_test.go

@@ -0,0 +1,59 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+package pprofutils
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/pprof/profile"
+	"github.com/stretchr/testify/require"
+)
+
+func TestProtobufConvert(t *testing.T) {
+	t.Run("basic", func(t *testing.T) {
+		data, err := ioutil.ReadFile(filepath.Join("test-fixtures", "pprof.samples.cpu.001.pb.gz"))
+		require.NoError(t, err)
+
+		proto, err := profile.Parse(bytes.NewReader(data))
+		require.NoError(t, err)
+
+		out := bytes.Buffer{}
+		require.NoError(t, Protobuf{}.Convert(proto, &out))
+		want := strings.TrimSpace(`
+golang.org/x/sync/errgroup.(*Group).Go.func1;main.run.func2;main.computeSum 19
+runtime.mcall;runtime.park_m;runtime.resetForSleep;runtime.resettimer;runtime.modtimer;runtime.wakeNetPoller;runtime.netpollBreak;runtime.write;runtime.write1 7
+golang.org/x/sync/errgroup.(*Group).Go.func1;main.run.func2;main.computeSum;runtime.asyncPreempt 5
+runtime.mstart;runtime.mstart1;runtime.sysmon;runtime.usleep 3
+runtime.mcall;runtime.park_m;runtime.schedule;runtime.findrunnable;runtime.stopm;runtime.notesleep;runtime.semasleep;runtime.pthread_cond_wait 2
+runtime.mcall;runtime.gopreempt_m;runtime.goschedImpl;runtime.schedule;runtime.findrunnable;runtime.stopm;runtime.notesleep;runtime.semasleep;runtime.pthread_cond_wait 1
+runtime.mcall;runtime.park_m;runtime.schedule;runtime.findrunnable;runtime.checkTimers;runtime.nanotime;runtime.nanotime1 1
+`) + "\n"
+		require.Equal(t, out.String(), want)
+	})
+
+	t.Run("differentLinesPerFunction", func(t *testing.T) {
+		data, err := ioutil.ReadFile(filepath.Join("test-fixtures", "pprof.lines.pb.gz"))
+		require.NoError(t, err)
+
+		proto, err := profile.Parse(bytes.NewReader(data))
+		require.NoError(t, err)
+
+		out := bytes.Buffer{}
+		require.NoError(t, Protobuf{}.Convert(proto, &out))
+		want := strings.TrimSpace(`
+main.run.func1;main.threadKind.Run;main.goGo1;main.goHog 85
+main.run.func1;main.threadKind.Run;main.goGo2;main.goHog 78
+main.run.func1;main.threadKind.Run;main.goGo3;main.goHog 72
+main.run.func1;main.threadKind.Run;main.goGo0;main.goHog 72
+main.run.func1;main.threadKind.Run;main.goGo0;main.goHog;runtime.asyncPreempt 1
+`) + "\n"
+		require.Equal(t, out.String(), want)
+	})
+}